Merge pull request #2111 from ASFHyP3/develop

Release v6.2.0

.github/actions/deploy-hyp3/action.yml

@@ -41,6 +41,9 @@ inputs:
   RESET_CREDITS_MONTHLY:
     description: "Whether to reset each user's remaining credits each month"
     required: true
+  COST_PROFILE:
+    description: "Job spec cost profile"
+    required: true
   JOB_FILES:
     description: "Space separated list of job spec YAMLs to include"
     required: true
@@ -83,7 +86,7 @@ runs:
         run: |
           pip install --upgrade pip
           make install
-          make files='${{ inputs.JOB_FILES }}' security_environment='${{ inputs.SECURITY_ENVIRONMENT }}' api_name='${{ inputs.API_NAME }}' build
+          make files='${{ inputs.JOB_FILES }}' security_environment='${{ inputs.SECURITY_ENVIRONMENT }}' api_name='${{ inputs.API_NAME }}' cost_profile='${{ inputs.COST_PROFILE }}' build
       - name: Package and deploy
         shell: bash
         run: |

.github/workflows/deploy-credits-sandbox.yml

@@ -21,6 +21,7 @@ jobs:
             product_lifetime_in_days: 14
             default_credits_per_user: 0
             reset_credits_monthly: true
+            cost_profile: EDC
             deploy_ref: refs/heads/credits-sandbox
             job_files: job_spec/AUTORIFT.yml job_spec/INSAR_GAMMA.yml job_spec/RTC_GAMMA.yml job_spec/INSAR_ISCE_BURST.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
@@ -64,6 +65,7 @@ jobs:
           CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }}
           DEFAULT_CREDITS_PER_USER: ${{ matrix.default_credits_per_user }}
           RESET_CREDITS_MONTHLY: ${{ matrix.reset_credits_monthly }}
+          COST_PROFILE: ${{ matrix.cost_profile }}
           JOB_FILES: ${{ matrix.job_files }}
           DEFAULT_MAX_VCPUS: ${{ matrix.default_max_vcpus }}
           EXPANDED_MAX_VCPUS: ${{ matrix.expanded_max_vcpus }}

.github/workflows/deploy-daac.yml

@@ -23,6 +23,7 @@ jobs:
             product_lifetime_in_days: 14
             default_credits_per_user: 1000
             reset_credits_monthly: true
+            cost_profile: DEFAULT
             deploy_ref: refs/heads/main
             job_files: job_spec/AUTORIFT.yml job_spec/INSAR_GAMMA.yml job_spec/RTC_GAMMA.yml job_spec/INSAR_ISCE_BURST.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
@@ -39,8 +40,9 @@ jobs:
             template_bucket: cf-templates-118ylv0o6jp2n-us-west-2
             image_tag: test
             product_lifetime_in_days: 14
-            default_credits_per_user: 1000
+            default_credits_per_user: 10000
             reset_credits_monthly: true
+            cost_profile: EDC
             deploy_ref: refs/heads/develop
             job_files: >-
               job_spec/AUTORIFT.yml
@@ -90,6 +92,7 @@ jobs:
           CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }}
           DEFAULT_CREDITS_PER_USER: ${{ matrix.default_credits_per_user }}
           RESET_CREDITS_MONTHLY: ${{ matrix.reset_credits_monthly }}
+          COST_PROFILE: ${{ matrix.cost_profile }}
           JOB_FILES: ${{ matrix.job_files }}
           DEFAULT_MAX_VCPUS: ${{ matrix.default_max_vcpus }}
           EXPANDED_MAX_VCPUS: ${{ matrix.expanded_max_vcpus }}

.github/workflows/deploy-enterprise-test.yml

@@ -21,6 +21,7 @@ jobs:
             product_lifetime_in_days: 14
             default_credits_per_user: 0
             reset_credits_monthly: false
+            cost_profile: DEFAULT
             deploy_ref: refs/heads/develop
             job_files: >-
               job_spec/AUTORIFT_ITS_LIVE.yml
@@ -72,6 +73,7 @@ jobs:
           CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }}
           DEFAULT_CREDITS_PER_USER: ${{ matrix.default_credits_per_user }}
           RESET_CREDITS_MONTHLY: ${{ matrix.reset_credits_monthly }}
+          COST_PROFILE: ${{ matrix.cost_profile }}
           JOB_FILES: ${{ matrix.job_files }}
           DEFAULT_MAX_VCPUS: ${{ matrix.default_max_vcpus }}
           EXPANDED_MAX_VCPUS: ${{ matrix.expanded_max_vcpus }}

.github/workflows/deploy-enterprise.yml

@@ -21,6 +21,7 @@ jobs:
             product_lifetime_in_days: 45
             default_credits_per_user: 0
             reset_credits_monthly: true
+            cost_profile: DEFAULT
             job_files: >-
               job_spec/AUTORIFT_ITS_LIVE.yml
               job_spec/AUTORIFT_ITS_LIVE_TEST.yml
@@ -40,10 +41,11 @@ jobs:
             product_lifetime_in_days: 180
             default_credits_per_user: 0
             reset_credits_monthly: true
+            cost_profile: DEFAULT
             job_files: job_spec/INSAR_ISCE.yml job_spec/INSAR_ISCE_TEST.yml
             instance_types: c6id.xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge
-            default_max_vcpus: 10000
-            expanded_max_vcpus: 10000
+            default_max_vcpus: 1000
+            expanded_max_vcpus: 1000
             required_surplus: 0
             security_environment: JPL-public
             ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
@@ -56,10 +58,11 @@ jobs:
             product_lifetime_in_days: 30
             default_credits_per_user: 0
             reset_credits_monthly: true
+            cost_profile: DEFAULT
             job_files: job_spec/INSAR_ISCE.yml job_spec/INSAR_ISCE_TEST.yml
             instance_types: c6id.xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge
-            default_max_vcpus: 10000
-            expanded_max_vcpus: 10000
+            default_max_vcpus: 1000
+            expanded_max_vcpus: 1000
             required_surplus: 0
             security_environment: JPL-public
             ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
@@ -72,6 +75,7 @@ jobs:
             product_lifetime_in_days: 14
             default_credits_per_user: 0
             reset_credits_monthly: true
+            cost_profile: DEFAULT
             job_files: job_spec/INSAR_ISCE.yml job_spec/INSAR_ISCE_TEST.yml
             instance_types: c6id.xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge
             default_max_vcpus: 1600
@@ -88,6 +92,7 @@ jobs:
             product_lifetime_in_days: 365000
             default_credits_per_user: 0
             reset_credits_monthly: true
+            cost_profile: DEFAULT
             job_files: job_spec/INSAR_GAMMA.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
             default_max_vcpus: 640
@@ -104,6 +109,7 @@ jobs:
             product_lifetime_in_days: 14
             default_credits_per_user: 0
             reset_credits_monthly: true
+            cost_profile: DEFAULT
             job_files: job_spec/RTC_GAMMA.yml job_spec/WATER_MAP.yml job_spec/WATER_MAP_EQ.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
             default_max_vcpus: 640
@@ -120,6 +126,7 @@ jobs:
             product_lifetime_in_days: 90
             default_credits_per_user: 0
             reset_credits_monthly: true
+            cost_profile: DEFAULT
             job_files: job_spec/RTC_GAMMA.yml job_spec/WATER_MAP.yml job_spec/WATER_MAP_EQ.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
             default_max_vcpus: 1600
@@ -136,6 +143,7 @@ jobs:
             product_lifetime_in_days: 30
             default_credits_per_user: 0
             reset_credits_monthly: true
+            cost_profile: DEFAULT
             job_files: job_spec/INSAR_GAMMA.yml job_spec/INSAR_ISCE_BURST.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
             default_max_vcpus: 640
@@ -152,6 +160,7 @@ jobs:
             product_lifetime_in_days: 14
             default_credits_per_user: 0
             reset_credits_monthly: true
+            cost_profile: DEFAULT
             job_files: job_spec/INSAR_GAMMA.yml job_spec/RTC_GAMMA.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
             default_max_vcpus: 640
@@ -168,6 +177,7 @@ jobs:
             product_lifetime_in_days: 14
             default_credits_per_user: 0
             reset_credits_monthly: true
+            cost_profile: DEFAULT
             job_files: job_spec/INSAR_GAMMA.yml job_spec/RTC_GAMMA.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
             default_max_vcpus: 640
@@ -184,6 +194,7 @@ jobs:
             product_lifetime_in_days: 30
             default_credits_per_user: 0
             reset_credits_monthly: true
+            cost_profile: DEFAULT
             job_files: job_spec/INSAR_GAMMA.yml job_spec/RTC_GAMMA.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
             default_max_vcpus: 640
@@ -202,6 +213,7 @@ jobs:
             product_lifetime_in_days: 14
             default_credits_per_user: 0
             reset_credits_monthly: true
+            cost_profile: DEFAULT
             job_files: job_spec/WATER_MAP.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
             default_max_vcpus: 640
@@ -244,6 +256,7 @@ jobs:
           CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }}
           DEFAULT_CREDITS_PER_USER: ${{ matrix.default_credits_per_user }}
           RESET_CREDITS_MONTHLY: ${{ matrix.reset_credits_monthly }}
+          COST_PROFILE: ${{ matrix.cost_profile }}
           JOB_FILES: ${{ matrix.job_files }}
           DEFAULT_MAX_VCPUS: ${{ matrix.default_max_vcpus }}
           EXPANDED_MAX_VCPUS: ${{ matrix.expanded_max_vcpus }}

.gitignore

@@ -6,6 +6,7 @@ apps/api/src/hyp3_api/job_validation_map.yml
 apps/step-function.json
 apps/**/*-cf.yml
 lib/dynamo/dynamo/*.json
+lib/dynamo/dynamo/*.yml
 
 
 # Byte-compiled / optimized / DLL files

CHANGELOG.md

@@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [6.2.0]
+
+HyP3 is in the process of transitioning from a monthly job quota to a credits system. [HyP3 v6.0.0](https://github.com/ASFHyP3/hyp3/releases/tag/v6.0.0) implemented the new credits system without changing the number of jobs that users can run per month. This release implements the capability to assign a different credit cost to each type of job, again without actually changing the number of jobs that users can run per month.
+
+Beginning on April 1st, the production API at <https://hyp3-api.asf.alaska.edu> will assign a different cost to each type of job and users will be given an allotment of 10,000 credits per month. Visit the [credits announcement page](https://hyp3-docs.asf.alaska.edu/using/credits/) for full details.
+
+### Added
+- `/costs` API endpoint that returns a table that can be used to look up the credit costs for different types of jobs.
+
+### Changed
+- The <https://hyp3-test-api.asf.alaska.edu> API now implements the credit costs displayed on the [credits announcement page](https://hyp3-docs.asf.alaska.edu/using/credits/).
+- `hyp3-a19-jpl` and `hyp3-tibet-jpl` deployments max vCPUs have been reduced to 1,000 from 10,000 because of persistent spot interruptions.
+
 ## [6.1.1]
 
 ### Changed

Makefile

@@ -37,8 +37,9 @@ install:
 files ?= job_spec/*.yml
 security_environment ?= ASF
 api_name ?= local
+cost_profile ?= DEFAULT
 render:
-	@echo rendering $(files) for API $(api_name) and security environment $(security_environment); python apps/render_cf.py -j $(files) -s $(security_environment) -n $(api_name)
+	@echo rendering $(files) for API $(api_name) and security environment $(security_environment); python apps/render_cf.py -j $(files) -s $(security_environment) -n $(api_name) -c $(cost_profile)
 
 static: flake8 openapi-validate cfn-lint
 

apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2

@@ -16,6 +16,17 @@ servers:
 
 paths:
 
+  /costs:
+    get:
+      description: Get table of job costs.
+      responses:
+        "200":
+          description: 200 response
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/costs"
+
   /jobs:
 
     post:
@@ -106,6 +117,10 @@ paths:
 components:
   schemas:
 
+    costs:
+      description: Table of job costs.
+      type: object
+
     post_jobs_body:
       description: List for new jobs to submit for processing.
       type: object

apps/api/src/hyp3_api/routes.py

@@ -11,6 +11,7 @@
 from openapi_core.contrib.flask.decorators import FlaskOpenAPIViewDecorator
 from openapi_core.contrib.flask.handlers import FlaskOpenAPIErrorsHandler
 
+import dynamo
 from hyp3_api import app, auth, handlers
 from hyp3_api.openapi import get_spec_yaml
 
@@ -111,6 +112,11 @@ def __call__(self, errors):
 )
 
 
+@app.route('/costs', methods=['GET'])
+def costs_get():
+    return jsonify(dynamo.jobs.COSTS)
+
+
 @app.route('/jobs', methods=['POST'])
 @openapi
 def jobs_post():

apps/render_cf.py

@@ -47,11 +47,21 @@ def render_default_params_by_job_type(job_types: dict) -> None:
         json.dump(default_params_by_job_type, f, indent=2)
 
 
+def render_costs(job_types: dict, cost_profile: str) -> None:
+    costs = {
+        job_type: job_spec['cost_profiles'][cost_profile]
+        for job_type, job_spec in job_types.items()
+    }
+    with open(Path('lib') / 'dynamo' / 'dynamo' / 'costs.yml', 'w') as f:
+        yaml.safe_dump(costs, f)
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('-j', '--job-spec-files', required=True, nargs='+', type=Path)
     parser.add_argument('-s', '--security-environment', default='ASF', choices=['ASF', 'EDC', 'JPL', 'JPL-public'])
     parser.add_argument('-n', '--api-name', required=True)
+    parser.add_argument('-c', '--cost-profile', default='DEFAULT', choices=['DEFAULT', 'EDC'])
     args = parser.parse_args()
 
     job_types = {}
@@ -63,6 +73,7 @@ def main():
             task['name'] = job_type + '_' + task['name'] if task['name'] else job_type
 
     render_default_params_by_job_type(job_types)
+    render_costs(job_types, args.cost_profile)
     render_templates(job_types, args.security_environment, args.api_name)
 
 

job_spec/AUTORIFT.yml

@@ -33,6 +33,11 @@ AUTORIFT:
               example: LC08_L1GT_118112_20210107_20210107_02_T2
     bucket_prefix:
       default:  '""'
+  cost_profiles:
+    EDC:
+      cost: 25.0
+    DEFAULT:
+      cost: 1.0
   validators: []
   tasks:
     - name: ''

job_spec/AUTORIFT_ITS_LIVE.yml

@@ -38,6 +38,9 @@ AUTORIFT:
         description: Shapefile for determining the correct search parameters by geographic location. Path to shapefile must be understood by GDAL.
         type: string
         default: '/vsicurl/http://its-live-data.s3.amazonaws.com/autorift_parameters/v001/autorift_landice_0120m.shp'
+  cost_profiles:
+    DEFAULT:
+      cost: 1.0
   validators: []
   tasks:
     - name: ''

job_spec/AUTORIFT_ITS_LIVE_TEST.yml

@@ -38,6 +38,9 @@ AUTORIFT_TEST:
         description: Shapefile for determining the correct search parameters by geographic location. Path to shapefile must be understood by GDAL.
         type: string
         default: '/vsicurl/http://its-live-data.s3.amazonaws.com/autorift_parameters/v001/autorift_landice_0120m.shp'
+  cost_profiles:
+    DEFAULT:
+      cost: 1.0
   validators: []
   tasks:
     - name: ''

job_spec/INSAR_GAMMA.yml

@@ -70,6 +70,14 @@ INSAR_GAMMA:
         minimum: 0.0
         maximum: 1.0
         default: 0.6
+  cost_profiles:
+    EDC:
+      cost_parameter: looks
+      cost_table:
+        20x4: 10.0
+        10x2: 15.0
+    DEFAULT:
+      cost: 1.0
   validators:
     - check_dem_coverage
   tasks:

job_spec/INSAR_ISCE.yml

@@ -64,6 +64,9 @@ INSAR_ISCE:
           - HRRR
     bucket_prefix:
       default:  '""'
+  cost_profiles:
+    DEFAULT:
+      cost: 1.0
   validators: []
   tasks:
     - name: ''

job_spec/INSAR_ISCE_BURST.yml

@@ -34,6 +34,11 @@ INSAR_ISCE_BURST:
           - 20x4
           - 10x2
           - 5x1
+  cost_profiles:
+    EDC:
+      cost: 1.0
+    DEFAULT:
+      cost: 1.0
   validators:
     - check_dem_coverage
     - check_valid_polarizations

job_spec/INSAR_ISCE_TEST.yml

@@ -88,6 +88,11 @@ INSAR_ISCE_TEST:
           - HRRR
     bucket_prefix:
       default:  '""'
+  cost_profiles:
+    EDC:
+      cost: 1.0
+    DEFAULT:
+      cost: 1.0
   validators: []
   tasks:
     - name: ''