diff --git a/.github/actions/deploy-hyp3/action.yml b/.github/actions/deploy-hyp3/action.yml index 946b9faa3..f1c5649b4 100644 --- a/.github/actions/deploy-hyp3/action.yml +++ b/.github/actions/deploy-hyp3/action.yml @@ -35,11 +35,14 @@ inputs: CLOUDFORMATION_ROLE_ARN: description: "The CloudFormation role to use for this deployment" required: true - MONTHLY_JOB_QUOTA_PER_USER: - description: "The default number of jobs any user with an Earthdata Login can run per month" + DEFAULT_CREDITS_PER_USER: + description: "The default number of credits given to a new user" + required: true + RESET_CREDITS_MONTHLY: + description: "Whether to reset each user's remaining credits each month" required: true JOB_FILES: - description: "Space seperated list of job spec YAMLs to include" + description: "Space separated list of job spec YAMLs to include" required: true DEFAULT_MAX_VCPUS: description: "Default maximum size for the AWS Batch compute environment" @@ -119,7 +122,8 @@ runs: $CERTIFICATE_ARN \ $ORIGIN_ACCESS_IDENTITY_ID \ $DISTRIBUTION_URL \ - MonthlyJobQuotaPerUser='${{ inputs.MONTHLY_JOB_QUOTA_PER_USER }}' \ + DefaultCreditsPerUser='${{ inputs.DEFAULT_CREDITS_PER_USER }}' \ + ResetCreditsMonthly='${{ inputs.RESET_CREDITS_MONTHLY }}' \ DefaultMaxvCpus='${{ inputs.DEFAULT_MAX_VCPUS }}' \ ExpandedMaxvCpus='${{ inputs.EXPANDED_MAX_VCPUS }}' \ MonthlyBudget='${{ inputs.MONTHLY_BUDGET }}' \ diff --git a/.github/workflows/deploy-credits-sandbox.yml b/.github/workflows/deploy-credits-sandbox.yml new file mode 100644 index 000000000..4bed2ae9a --- /dev/null +++ b/.github/workflows/deploy-credits-sandbox.yml @@ -0,0 +1,77 @@ +name: Deploy Credits Sandbox Stack to AWS + +on: + push: + branches: + - credits-sandbox + +concurrency: ${{ github.workflow }}-${{ github.ref }} + +jobs: + deploy: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - environment: hyp3-credits-sandbox + domain: hyp3-credits-sandbox.asf.alaska.edu + template_bucket: cf-templates-1hz9ldhhl4ahu-us-west-2 + image_tag: test + product_lifetime_in_days: 14 + default_credits_per_user: 0 + reset_credits_monthly: true + deploy_ref: refs/heads/credits-sandbox + job_files: job_spec/AUTORIFT.yml job_spec/INSAR_GAMMA.yml job_spec/RTC_GAMMA.yml job_spec/INSAR_ISCE_BURST.yml + instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge + default_max_vcpus: 640 + expanded_max_vcpus: 640 + required_surplus: 0 + security_environment: ASF + ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id + distribution_url: '' + + environment: + name: ${{ matrix.environment }} + url: https://${{ matrix.domain }} + + steps: + - uses: actions/checkout@v4.1.1 + + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.V2_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.V2_AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ secrets.V2_AWS_SESSION_TOKEN }} + aws-region: ${{ secrets.AWS_REGION }} + + - uses: actions/setup-python@v5 + with: + python-version: 3.9 + + - uses: ./.github/actions/deploy-hyp3 + with: + TEMPLATE_BUCKET: ${{ matrix.template_bucket }} + STACK_NAME: ${{ matrix.environment }} + DOMAIN_NAME: ${{ matrix.domain }} + API_NAME: ${{ matrix.environment }} + CERTIFICATE_ARN: ${{ secrets.CERTIFICATE_ARN }} + IMAGE_TAG: ${{ matrix.image_tag }} + PRODUCT_LIFETIME: ${{ matrix.product_lifetime_in_days }} + VPC_ID: ${{ secrets.VPC_ID }} + SUBNET_IDS: ${{ secrets.SUBNET_IDS }} + SECRET_ARN: ${{ secrets.SECRET_ARN }} + CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }} + DEFAULT_CREDITS_PER_USER: ${{ matrix.default_credits_per_user }} + RESET_CREDITS_MONTHLY: ${{ matrix.reset_credits_monthly }} + JOB_FILES: ${{ matrix.job_files }} + DEFAULT_MAX_VCPUS: ${{ matrix.default_max_vcpus }} + EXPANDED_MAX_VCPUS: ${{ matrix.expanded_max_vcpus }} + MONTHLY_BUDGET: ${{ secrets.MONTHLY_BUDGET }} + REQUIRED_SURPLUS: ${{ matrix.required_surplus }} + ORIGIN_ACCESS_IDENTITY_ID: ${{ secrets.ORIGIN_ACCESS_IDENTITY_ID }} + SECURITY_ENVIRONMENT: ${{ matrix.security_environment }} + AMI_ID: ${{ matrix.ami_id }} + INSTANCE_TYPES: ${{ matrix.instance_types }} + DISTRIBUTION_URL: ${{ matrix.distribution_url }} + AUTH_PUBLIC_KEY: ${{ secrets.AUTH_PUBLIC_KEY }} diff --git a/.github/workflows/deploy-daac.yml b/.github/workflows/deploy-daac.yml index 1a20afb45..749dfde28 100644 --- a/.github/workflows/deploy-daac.yml +++ b/.github/workflows/deploy-daac.yml @@ -21,7 +21,8 @@ jobs: template_bucket: cf-templates-118mtzosmrltk-us-west-2 image_tag: latest product_lifetime_in_days: 14 - quota: 1000 + default_credits_per_user: 1000 + reset_credits_monthly: true deploy_ref: refs/heads/main job_files: job_spec/AUTORIFT.yml job_spec/INSAR_GAMMA.yml job_spec/RTC_GAMMA.yml job_spec/INSAR_ISCE_BURST.yml instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge @@ -38,7 +39,8 @@ jobs: template_bucket: cf-templates-118ylv0o6jp2n-us-west-2 image_tag: test product_lifetime_in_days: 14 - quota: 1000 + default_credits_per_user: 1000 + reset_credits_monthly: true deploy_ref: refs/heads/develop job_files: >- job_spec/AUTORIFT.yml @@ -86,7 +88,8 @@ jobs: SUBNET_IDS: ${{ secrets.SUBNET_IDS }} SECRET_ARN: ${{ secrets.SECRET_ARN }} CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }} - MONTHLY_JOB_QUOTA_PER_USER: ${{ matrix.quota }} + DEFAULT_CREDITS_PER_USER: ${{ matrix.default_credits_per_user }} + RESET_CREDITS_MONTHLY: ${{ matrix.reset_credits_monthly }} JOB_FILES: ${{ matrix.job_files }} DEFAULT_MAX_VCPUS: ${{ matrix.default_max_vcpus }} EXPANDED_MAX_VCPUS: ${{ matrix.expanded_max_vcpus }} diff --git a/.github/workflows/deploy-enterprise-test.yml b/.github/workflows/deploy-enterprise-test.yml index eb64d691f..0bd80a138 100644 --- a/.github/workflows/deploy-enterprise-test.yml +++ b/.github/workflows/deploy-enterprise-test.yml @@ -19,7 +19,8 @@ jobs: template_bucket: cf-templates-1iw894v4yzqya-us-west-2 image_tag: test product_lifetime_in_days: 14 - quota: 0 + default_credits_per_user: 0 + reset_credits_monthly: false deploy_ref: refs/heads/develop job_files: >- job_spec/AUTORIFT_ITS_LIVE.yml @@ -69,7 +70,8 @@ jobs: SUBNET_IDS: ${{ secrets.SUBNET_IDS }} SECRET_ARN: ${{ secrets.SECRET_ARN }} CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }} - MONTHLY_JOB_QUOTA_PER_USER: ${{ matrix.quota }} + DEFAULT_CREDITS_PER_USER: ${{ matrix.default_credits_per_user }} + RESET_CREDITS_MONTHLY: ${{ matrix.reset_credits_monthly }} JOB_FILES: ${{ matrix.job_files }} DEFAULT_MAX_VCPUS: ${{ matrix.default_max_vcpus }} EXPANDED_MAX_VCPUS: ${{ matrix.expanded_max_vcpus }} diff --git a/.github/workflows/deploy-enterprise.yml b/.github/workflows/deploy-enterprise.yml index 4c1ce215e..516f6f202 100644 --- a/.github/workflows/deploy-enterprise.yml +++ b/.github/workflows/deploy-enterprise.yml @@ -19,7 +19,8 @@ jobs: template_bucket: cf-templates-3o5lnspmwmzg-us-west-2 image_tag: latest product_lifetime_in_days: 45 - quota: 0 + default_credits_per_user: 0 + reset_credits_monthly: true job_files: >- job_spec/AUTORIFT_ITS_LIVE.yml job_spec/AUTORIFT_ITS_LIVE_TEST.yml @@ -37,7 +38,8 @@ jobs: template_bucket: cf-templates-v4pvone059de-us-west-2 image_tag: latest product_lifetime_in_days: 180 - quota: 0 + default_credits_per_user: 0 + reset_credits_monthly: true job_files: job_spec/INSAR_ISCE.yml job_spec/INSAR_ISCE_TEST.yml instance_types: c6id.xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge default_max_vcpus: 10000 @@ -52,7 +54,8 @@ jobs: template_bucket: cf-templates-1or0efwqffkgd-us-west-2 image_tag: latest product_lifetime_in_days: 60 - quota: 0 + default_credits_per_user: 0 + reset_credits_monthly: true job_files: job_spec/INSAR_ISCE.yml job_spec/INSAR_ISCE_TEST.yml instance_types: c6id.xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge default_max_vcpus: 0 @@ -67,7 +70,8 @@ jobs: template_bucket: cf-templates-gdeyr9hh8rzs-us-west-2 image_tag: latest product_lifetime_in_days: 14 - quota: 0 + default_credits_per_user: 0 + reset_credits_monthly: true job_files: job_spec/INSAR_ISCE.yml job_spec/INSAR_ISCE_TEST.yml instance_types: c6id.xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge default_max_vcpus: 1600 @@ -82,7 +86,8 @@ jobs: template_bucket: cf-templates-1x4a21iq1cba7-us-west-2 image_tag: latest product_lifetime_in_days: 365000 - quota: 0 + default_credits_per_user: 0 + reset_credits_monthly: true job_files: job_spec/INSAR_GAMMA.yml instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge default_max_vcpus: 640 @@ -97,7 +102,8 @@ jobs: template_bucket: cf-templates-1217di08q7vwl-us-west-2 image_tag: latest product_lifetime_in_days: 14 - quota: 0 + default_credits_per_user: 0 + reset_credits_monthly: true job_files: job_spec/RTC_GAMMA.yml job_spec/WATER_MAP.yml job_spec/WATER_MAP_EQ.yml instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge default_max_vcpus: 640 @@ -112,7 +118,8 @@ jobs: template_bucket: cf-templates-15gmiot9prm67-us-west-2 image_tag: latest product_lifetime_in_days: 90 - quota: 0 + default_credits_per_user: 0 + reset_credits_monthly: true job_files: job_spec/RTC_GAMMA.yml job_spec/WATER_MAP.yml job_spec/WATER_MAP_EQ.yml instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge default_max_vcpus: 1600 @@ -127,7 +134,8 @@ jobs: template_bucket: cf-templates-xlga17noink6-us-west-2 image_tag: latest product_lifetime_in_days: 30 - quota: 0 + default_credits_per_user: 0 + reset_credits_monthly: true job_files: job_spec/INSAR_GAMMA.yml job_spec/INSAR_ISCE_BURST.yml instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge default_max_vcpus: 640 @@ -142,7 +150,8 @@ jobs: template_bucket: cf-templates-j4kd746vpsuv-us-east-1 image_tag: latest product_lifetime_in_days: 14 - quota: 0 + default_credits_per_user: 0 + reset_credits_monthly: true job_files: job_spec/INSAR_GAMMA.yml job_spec/RTC_GAMMA.yml instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge default_max_vcpus: 640 @@ -157,7 +166,8 @@ jobs: template_bucket: cf-templates-ez0805f6vy20-us-west-2 image_tag: latest product_lifetime_in_days: 14 - quota: 0 + default_credits_per_user: 0 + reset_credits_monthly: true job_files: job_spec/INSAR_GAMMA.yml job_spec/RTC_GAMMA.yml instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge default_max_vcpus: 640 @@ -172,7 +182,8 @@ jobs: template_bucket: cf-templates-1qx2mwia5g4kh-us-west-2 image_tag: latest product_lifetime_in_days: 30 - quota: 0 + default_credits_per_user: 0 + reset_credits_monthly: true job_files: job_spec/INSAR_GAMMA.yml job_spec/RTC_GAMMA.yml instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge default_max_vcpus: 640 @@ -189,7 +200,8 @@ jobs: # TODO product lifetime could be much shorter since they all get transferred to a separate # S3 bucket, but maybe we want to allow for a backlog of products-to-be-transferred? product_lifetime_in_days: 14 - quota: 0 + default_credits_per_user: 0 + reset_credits_monthly: true job_files: job_spec/WATER_MAP.yml instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge default_max_vcpus: 640 @@ -230,7 +242,8 @@ jobs: SUBNET_IDS: ${{ secrets.SUBNET_IDS }} SECRET_ARN: ${{ secrets.SECRET_ARN }} CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }} - MONTHLY_JOB_QUOTA_PER_USER: ${{ matrix.quota }} + DEFAULT_CREDITS_PER_USER: ${{ matrix.default_credits_per_user }} + RESET_CREDITS_MONTHLY: ${{ matrix.reset_credits_monthly }} JOB_FILES: ${{ matrix.job_files }} DEFAULT_MAX_VCPUS: ${{ matrix.default_max_vcpus }} EXPANDED_MAX_VCPUS: ${{ matrix.expanded_max_vcpus }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 30772e450..7681319a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,21 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [6.0.0] + +HyP3's monthly quota system has been replaced by a credits system. Previously, HyP3 provided each user with a certain number of jobs per month. Now, each job costs a particular number of credits, and users spend credits when they submit jobs. This release assigns every job a cost of 1 credit, but future releases will assign a different credit cost to each job type. Additionally, the main production deployment (`https://hyp3-api.asf.alaska.edu`) resets each user's balance to 1,000 credits each month, effectively granting each user 1,000 jobs per month. Therefore, users should not notice any difference when ordering jobs via ASF's On Demand service at . + +### Added +- The `job` object returned by the `/jobs` API endpoint now includes a `credit_cost` attribute, which represents the job's cost in credits. +- A `DAR` tag is now included in Earthdata Cloud deployments for each S3 bucket to communicate which contain objects + that required to be encrypted at rest. + +### Changed +- The `quota` attribute of the `user` object returned by the `/user` API endpoint has been replaced by a `remaining_credits` attribute, which represents the user's remaining credits. + +### Removed +- The non-functional CloudWatch alarm for API 5xx errors has been removed from the `monitoring` module. See [#2044](https://github.com/ASFHyP3/hyp3/issues/2044). + ## [5.0.4] ### Added - `INSAR_ISCE_BURST` jobs are now available in the azdwr-hyp3 deployment. diff --git a/apps/api/api-cf.yml.j2 b/apps/api/api-cf.yml.j2 index db2cc819e..94f28cc8f 100644 --- a/apps/api/api-cf.yml.j2 +++ b/apps/api/api-cf.yml.j2 @@ -12,9 +12,12 @@ Parameters: AuthAlgorithm: Type: String - MonthlyJobQuotaPerUser: + DefaultCreditsPerUser: Type: Number + ResetCreditsMonthly: + Type: String + SystemAvailable: Type: String @@ -41,9 +44,6 @@ Outputs: Url: Value: {{ '!Sub "https://${RestApi}.execute-api.${AWS::Region}.amazonaws.com/${Stage}/ui"' if security_environment == 'EDC' else '!Sub "https://${CustomDomainName}/ui"' }} - ApiId: - Value: !Ref RestApi - Resources: RestApi: @@ -161,13 +161,15 @@ Resources: Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/*" - Effect: Allow Action: - - dynamodb:PutItem + - dynamodb:BatchWriteItem - dynamodb:Query - dynamodb:GetItem Resource: !Sub "arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${JobsTable}*" - Effect: Allow Action: - dynamodb:GetItem + - dynamodb:PutItem + - dynamodb:UpdateItem Resource: !Sub "arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${UsersTable}*" Lambda: @@ -179,7 +181,8 @@ Resources: USERS_TABLE_NAME: !Ref UsersTable AUTH_PUBLIC_KEY: !Ref AuthPublicKey AUTH_ALGORITHM: !Ref AuthAlgorithm - MONTHLY_JOB_QUOTA_PER_USER: !Ref MonthlyJobQuotaPerUser + DEFAULT_CREDITS_PER_USER: !Ref DefaultCreditsPerUser + RESET_CREDITS_MONTHLY: !Ref ResetCreditsMonthly SYSTEM_AVAILABLE: !Ref SystemAvailable Code: src/ Handler: hyp3_api.lambda_handler.handler diff --git a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 index 26135d69c..b9ee46a0a 100644 --- a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 +++ b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 @@ -133,36 +133,24 @@ components: $ref: "#/components/schemas/next_url" user: - description: Information about a user (quota, user id) + description: Information about a user type: object required: - user_id - - quota + - remaining_credits additionalProperties: false properties: user_id: $ref: "#/components/schemas/user_id" - quota: - $ref: "#/components/schemas/quota" + remaining_credits: + $ref: "#/components/schemas/credits" job_names: $ref: "#/components/schemas/job_names_list" - quota: - description: Containes the limit of jobs per month and the amount remaining for a user. - type: object - required: - - max_jobs_per_month - - remaining - additionalProperties: false - properties: - max_jobs_per_month: - type: integer - minimum: 0 - nullable: true - remaining: - type: integer - minimum: 0 - nullable: true + credits: + description: Processing credits for running jobs. + type: number + minimum: 0 job_names_list: type: array @@ -287,6 +275,8 @@ components: $ref: "#/components/schemas/processing_times" priority: $ref: "#/components/schemas/priority" + credit_cost: + $ref: "#/components/schemas/credits" validate_only: type: boolean diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index 929346ab9..fd873a67e 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -41,12 +41,11 @@ def post_jobs(body, user): except GranuleValidationError as e: abort(problem_format(400, str(e))) - if not body.get('validate_only'): - try: - body['jobs'] = dynamo.jobs.put_jobs(user, body['jobs']) - except dynamo.jobs.QuotaError as e: - abort(problem_format(400, str(e))) - return body + try: + body['jobs'] = dynamo.jobs.put_jobs(user, body['jobs'], dry_run=body.get('validate_only')) + except dynamo.jobs.InsufficientCreditsError as e: + abort(problem_format(400, str(e))) + return body def get_jobs(user, start=None, end=None, status_code=None, name=None, job_type=None, start_token=None): @@ -80,13 +79,10 @@ def get_names_for_user(user): def get_user(user): - max_jobs, _, remaining_jobs = dynamo.jobs.get_quota_status(user) - + user_record = dynamo.user.get_or_create_user(user) return { 'user_id': user, - 'quota': { - 'max_jobs_per_month': max_jobs, - 'remaining': remaining_jobs, - }, + 'remaining_credits': user_record['remaining_credits'], + # TODO: count this as jobs are submitted, not every time `/user` is queried 'job_names': get_names_for_user(user) } diff --git a/apps/main-cf.yml.j2 b/apps/main-cf.yml.j2 index ce1d9abbf..df7d00fd5 100644 --- a/apps/main-cf.yml.j2 +++ b/apps/main-cf.yml.j2 @@ -31,11 +31,18 @@ Parameters: Type: String Default: RS256 - MonthlyJobQuotaPerUser: - Description: Number of jobs each user is allowed per month. + DefaultCreditsPerUser: + Description: The default number of credits given to a new user. Type: Number MinValue: 0 + ResetCreditsMonthly: + Description: "Whether to reset each user's remaining credits each month" + Type: String + AllowedValues: + - false + - true + SystemAvailable: Description: Set to false to shutdown system, API will run and provide errors to users, but will not accept jobs. Type: String @@ -115,7 +122,8 @@ Resources: UsersTable: !Ref UsersTable AuthPublicKey: !Ref AuthPublicKey AuthAlgorithm: !Ref AuthAlgorithm - MonthlyJobQuotaPerUser: !Ref MonthlyJobQuotaPerUser + DefaultCreditsPerUser: !Ref DefaultCreditsPerUser + ResetCreditsMonthly: !Ref ResetCreditsMonthly SystemAvailable: !Ref SystemAvailable {% if security_environment == 'EDC' %} VpcId: !Ref VpcId @@ -189,7 +197,6 @@ Resources: Properties: Parameters: StepFunctionArn: !GetAtt StepFunction.Outputs.StepFunctionArn - ApiId: !GetAtt Api.Outputs.ApiId TemplateURL: monitoring-cf.yml LogBucket: @@ -208,6 +215,11 @@ Resources: OwnershipControls: Rules: - ObjectOwnership: BucketOwnerEnforced + {% if security_environment == 'EDC' %} + Tags: + - Key: DAR + Value: "YES" + {% endif %} LogBucketPolicy: Type: AWS::S3::BucketPolicy @@ -265,6 +277,11 @@ Resources: OwnershipControls: Rules: - ObjectOwnership: BucketOwnerEnforced + {% if security_environment == 'EDC' %} + Tags: + - Key: DAR + Value: "NO" + {% endif %} {% if security_environment != 'JPL' %} BucketPolicy: diff --git a/apps/monitoring-cf.yml.j2 b/apps/monitoring-cf.yml.j2 index d140130fe..9e1a35a27 100644 --- a/apps/monitoring-cf.yml.j2 +++ b/apps/monitoring-cf.yml.j2 @@ -4,9 +4,6 @@ Parameters: StepFunctionArn: Type: String - ApiId: - Type: String - Resources: AlarmTopic: @@ -30,21 +27,3 @@ Resources: Statistic: Sum Unit: Count TreatMissingData: notBreaching - - ApiAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmActions: - - !Ref AlarmTopic - AlarmDescription: Hyp3 api HTTP 500 errors - ComparisonOperator: GreaterThanThreshold - Period: 60 - EvaluationPeriods: 1 - Threshold: 0 - Dimensions: - - Name: ApiId - Value: !Ref ApiId - MetricName: 5xx - Namespace: AWS/ApiGateway - Statistic: Sum - TreatMissingData: notBreaching diff --git a/docs/deployments/ASF-deployment.md b/docs/deployments/ASF-deployment.md index 5714ceac2..11c74e6e4 100644 --- a/docs/deployments/ASF-deployment.md +++ b/docs/deployments/ASF-deployment.md @@ -93,5 +93,6 @@ aws cloudformation deploy \ DomainName= \ CertificateArn= \ SecretArn= \ - MonthlyJobQuotaPerUser=0 + DefaultCreditsPerUser=0 \ + ResetCreditsMonthly=true ``` diff --git a/docs/deployments/JPL-deployment.md b/docs/deployments/JPL-deployment.md index 341a95e56..eca192a3b 100644 --- a/docs/deployments/JPL-deployment.md +++ b/docs/deployments/JPL-deployment.md @@ -93,7 +93,8 @@ aws cloudformation deploy \ DomainName= \ CertificateArn= \ SecretArn= \ - MonthlyJobQuotaPerUser=0 + DefaultCreditsPerUser=0 \ + ResetCreditsMonthly=true ``` ## 5. Post deployment diff --git a/lib/dynamo/dynamo/jobs.py b/lib/dynamo/dynamo/jobs.py index 7281990eb..3214136c2 100644 --- a/lib/dynamo/dynamo/jobs.py +++ b/lib/dynamo/dynamo/jobs.py @@ -1,6 +1,6 @@ from datetime import datetime, timezone from os import environ -from typing import Callable, List, Optional, Union +from typing import List, Optional from uuid import uuid4 from boto3.dynamodb.conditions import Attr, Key @@ -9,93 +9,80 @@ from dynamo.util import DYNAMODB_RESOURCE, convert_floats_to_decimals, format_time, get_request_time_expression -class QuotaError(Exception): - """Raised when trying to submit more jobs that user has remaining""" +class InsufficientCreditsError(Exception): + """Raised when trying to submit jobs whose total cost exceeds the user's remaining credits.""" -def _get_job_count_for_month(user) -> int: - now = datetime.now(timezone.utc) - start_of_month = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - job_count_for_month = count_jobs(user, format_time(start_of_month)) - return job_count_for_month +def _get_credit_cost(job: dict) -> float: + return 1.0 -def get_quota_status(user) -> Union[tuple[int, int, int], tuple[None, None, None]]: - max_jobs = dynamo.user.get_max_jobs_per_month(user) - - if max_jobs is not None: - previous_jobs = _get_job_count_for_month(user) - remaining_jobs = max(max_jobs - previous_jobs, 0) - else: - previous_jobs = None - remaining_jobs = None - - return max_jobs, previous_jobs, remaining_jobs - - -def put_jobs(user_id: str, jobs: List[dict], fail_when_over_quota=True) -> List[dict]: +def put_jobs(user_id: str, jobs: List[dict], dry_run=False) -> List[dict]: table = DYNAMODB_RESOURCE.Table(environ['JOBS_TABLE_NAME']) request_time = format_time(datetime.now(timezone.utc)) - max_jobs, previous_jobs, remaining_jobs = get_quota_status(user_id) - has_quota = max_jobs is not None - if has_quota: - assert previous_jobs is not None - assert remaining_jobs is not None - - if has_quota and len(jobs) > remaining_jobs: - if fail_when_over_quota: - raise QuotaError(f'Your monthly quota is {max_jobs} jobs. You have {remaining_jobs} jobs remaining.') - jobs = jobs[:remaining_jobs] - - priority_override = dynamo.user.get_priority(user_id) - priority = _get_job_priority(priority_override, has_quota) - - prepared_jobs = [ - { - 'job_id': str(uuid4()), - 'user_id': user_id, - 'status_code': 'PENDING', - 'execution_started': False, - 'request_time': request_time, - 'priority': priority(previous_jobs, index), - **job, - } for index, job in enumerate(jobs) - ] - - for prepared_job in prepared_jobs: - table.put_item(Item=convert_floats_to_decimals(prepared_job)) + user_record = dynamo.user.get_or_create_user(user_id) + + remaining_credits = user_record['remaining_credits'] + if remaining_credits is not None: + remaining_credits = float(remaining_credits) + + priority_override = user_record.get('priority_override') + + total_cost = 0.0 + prepared_jobs = [] + for job in jobs: + prepared_job = _prepare_job_for_database( + job=job, + user_id=user_id, + request_time=request_time, + remaining_credits=remaining_credits, + priority_override=priority_override, + running_cost=total_cost, + ) + prepared_jobs.append(prepared_job) + total_cost += prepared_job['credit_cost'] + + if remaining_credits is not None and total_cost > remaining_credits: + raise InsufficientCreditsError( + f'These jobs would cost {total_cost} credits, but you have only {remaining_credits} remaining.' + ) + + assert prepared_jobs[-1]['priority'] >= 0 + if not dry_run: + if remaining_credits is not None: + dynamo.user.decrement_credits(user_id, total_cost) + with table.batch_writer() as batch: + for prepared_job in prepared_jobs: + batch.put_item(Item=convert_floats_to_decimals(prepared_job)) + return prepared_jobs -def _get_job_priority(priority_override: Optional[int], has_quota: bool) -> Callable[[Optional[int], int], int]: - if priority_override is not None: - priority = lambda _, __: priority_override - elif has_quota: - priority = lambda previous_jobs, job_index: max(9999 - previous_jobs - job_index, 0) +def _prepare_job_for_database( + job: dict, + user_id: str, + request_time: str, + remaining_credits: Optional[float], + priority_override: Optional[int], + running_cost: float, +) -> dict: + if priority_override: + priority = priority_override + elif remaining_credits is None: + priority = 0 else: - priority = lambda _, __: 0 - return priority - - -def count_jobs(user, start=None, end=None): - table = DYNAMODB_RESOURCE.Table(environ['JOBS_TABLE_NAME']) - key_expression = Key('user_id').eq(user) - if start is not None or end is not None: - key_expression &= get_request_time_expression(start, end) - - params = { - 'IndexName': 'user_id', - 'KeyConditionExpression': key_expression, - 'Select': 'COUNT', + priority = min(int(remaining_credits - running_cost), 9999) + return { + 'job_id': str(uuid4()), + 'user_id': user_id, + 'status_code': 'PENDING', + 'execution_started': False, + 'request_time': request_time, + 'credit_cost': _get_credit_cost(job), + 'priority': priority, + **job, } - response = table.query(**params) - job_count = response['Count'] - while 'LastEvaluatedKey' in response: - params['ExclusiveStartKey'] = response['LastEvaluatedKey'] - response = table.query(**params) - job_count += response['Count'] - return job_count def query_jobs(user, start=None, end=None, status_code=None, name=None, job_type=None, start_key=None): diff --git a/lib/dynamo/dynamo/user.py b/lib/dynamo/dynamo/user.py index c2ceebc1d..ac5b3c444 100644 --- a/lib/dynamo/dynamo/user.py +++ b/lib/dynamo/dynamo/user.py @@ -1,30 +1,93 @@ +import os +from datetime import datetime, timezone +from decimal import Decimal from os import environ -from typing import Optional + +import botocore.exceptions from dynamo.util import DYNAMODB_RESOURCE -def get_user(user_id: str) -> Optional[dict]: - table = DYNAMODB_RESOURCE.Table(environ['USERS_TABLE_NAME']) - response = table.get_item(Key={'user_id': user_id}) - return response.get('Item') +class DatabaseConditionException(Exception): + """Raised when a DynamoDB condition expression check fails.""" -def get_priority(user_id: str) -> Optional[int]: - user = get_user(user_id) - if user: - priority = user.get('priority') - else: - priority = None - return priority +def get_or_create_user(user_id: str) -> dict: + current_month = _get_current_month() + default_credits = Decimal(os.environ['DEFAULT_CREDITS_PER_USER']) + users_table = DYNAMODB_RESOURCE.Table(environ['USERS_TABLE_NAME']) + user = users_table.get_item(Key={'user_id': user_id}).get('Item') -def get_max_jobs_per_month(user_id: str) -> Optional[int]: - user = get_user(user_id) - if user is not None and 'max_jobs_per_month' in user: - max_jobs_per_month = user['max_jobs_per_month'] - if max_jobs_per_month is not None: - max_jobs_per_month = int(max_jobs_per_month) + if user is not None: + user = _reset_credits_if_needed( + user=user, + default_credits=default_credits, + current_month=current_month, + users_table=users_table, + ) else: - max_jobs_per_month = int(environ['MONTHLY_JOB_QUOTA_PER_USER']) - return max_jobs_per_month + user = _create_user( + user_id=user_id, + default_credits=default_credits, + current_month=current_month, + users_table=users_table, + ) + return user + + +def _get_current_month() -> str: + return datetime.now(tz=timezone.utc).strftime('%Y-%m') + + +def _create_user(user_id: str, default_credits: Decimal, current_month: str, users_table) -> dict: + user = {'user_id': user_id, 'remaining_credits': default_credits, 'month_of_last_credits_reset': current_month} + try: + users_table.put_item(Item=user, ConditionExpression='attribute_not_exists(user_id)') + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'ConditionalCheckFailedException': + raise DatabaseConditionException(f'Failed to create user {user_id}') + raise + return user + + +def _reset_credits_if_needed(user: dict, default_credits: Decimal, current_month: str, users_table) -> dict: + if ( + os.environ['RESET_CREDITS_MONTHLY'] == 'true' + and user['month_of_last_credits_reset'] < current_month # noqa: W503 + and user['remaining_credits'] is not None # noqa: W503 + ): + user['month_of_last_credits_reset'] = current_month + user['remaining_credits'] = user.get('credits_per_month', default_credits) + try: + users_table.put_item( + Item=user, + ConditionExpression='month_of_last_credits_reset < :current_month' + ' AND attribute_type(remaining_credits, :number)', + ExpressionAttributeValues={ + ':current_month': current_month, + ':number': 'N', + }, + ) + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'ConditionalCheckFailedException': + raise DatabaseConditionException(f'Failed to perform monthly credits reset for user {user["user_id"]}') + raise + return user + + +def decrement_credits(user_id: str, cost: float) -> None: + if cost <= 0: + raise ValueError(f'Cost {cost} <= 0') + users_table = DYNAMODB_RESOURCE.Table(environ['USERS_TABLE_NAME']) + try: + users_table.update_item( + Key={'user_id': user_id}, + UpdateExpression='ADD remaining_credits :delta', + ConditionExpression='remaining_credits >= :cost', + ExpressionAttributeValues={':cost': Decimal(cost), ':delta': Decimal(-cost)}, + ) + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'ConditionalCheckFailedException': + raise DatabaseConditionException(f'Failed to decrement credits for user {user_id}') + raise diff --git a/requirements-all.txt b/requirements-all.txt index d64f760f0..cb2e9f189 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -5,7 +5,7 @@ -r requirements-apps-start-execution-worker.txt -r requirements-apps-disable-private-dns.txt -r requirements-apps-update-db.txt -boto3==1.34.29 +boto3==1.34.32 jinja2==3.1.3 moto[dynamodb]==5.0.0 pytest==8.0.0 diff --git a/requirements-apps-api-binary.txt b/requirements-apps-api-binary.txt index d0384ae0d..1041fe690 100644 --- a/requirements-apps-api-binary.txt +++ b/requirements-apps-api-binary.txt @@ -1 +1 @@ -cryptography==42.0.1 +cryptography==42.0.2 diff --git a/requirements-apps-disable-private-dns.txt b/requirements-apps-disable-private-dns.txt index b9c720e80..aa2ff1534 100644 --- a/requirements-apps-disable-private-dns.txt +++ b/requirements-apps-disable-private-dns.txt @@ -1 +1 @@ -boto3==1.34.29 +boto3==1.34.32 diff --git a/requirements-apps-start-execution-manager.txt b/requirements-apps-start-execution-manager.txt index 20ec2b546..a669566f2 100644 --- a/requirements-apps-start-execution-manager.txt +++ b/requirements-apps-start-execution-manager.txt @@ -1,3 +1,3 @@ -boto3==1.34.29 +boto3==1.34.32 ./lib/dynamo/ ./lib/lambda_logging/ diff --git a/requirements-apps-start-execution-worker.txt b/requirements-apps-start-execution-worker.txt index cfe814906..6301bd0a5 100644 --- a/requirements-apps-start-execution-worker.txt +++ b/requirements-apps-start-execution-worker.txt @@ -1,2 +1,2 @@ -boto3==1.34.29 +boto3==1.34.32 ./lib/lambda_logging/ diff --git a/tests/cfg.env b/tests/cfg.env index ee8c72400..42d4b557b 100644 --- a/tests/cfg.env +++ b/tests/cfg.env @@ -3,7 +3,8 @@ JOBS_TABLE_NAME=hyp3-db-table-job USERS_TABLE_NAME=hyp3-db-table-user AUTH_PUBLIC_KEY=123456789 AUTH_ALGORITHM=HS256 -MONTHLY_JOB_QUOTA_PER_USER=25 +DEFAULT_CREDITS_PER_USER=25 +RESET_CREDITS_MONTHLY=false SYSTEM_AVAILABLE=true AWS_DEFAULT_REGION=us-west-2 AWS_ACCESS_KEY_ID=testing diff --git a/tests/test_api/test_get_user.py b/tests/test_api/test_get_user.py index c564dbc68..af54b05f1 100644 --- a/tests/test_api/test_get_user.py +++ b/tests/test_api/test_get_user.py @@ -6,88 +6,56 @@ from dynamo.util import format_time -def test_get_user(client, tables, monkeypatch): - monkeypatch.setenv('MONTHLY_JOB_QUOTA_PER_USER', '25') - request_time = format_time(datetime.now(timezone.utc)) - user = 'user_with_jobs' - items = [ - make_db_record('job1', user_id=user, request_time=request_time, status_code='PENDING', name='job1'), - make_db_record('job2', user_id=user, request_time=request_time, status_code='RUNNING', name='job1'), - make_db_record('job3', user_id=user, request_time=request_time, status_code='FAILED', name='job2'), - make_db_record('job4', user_id=user, request_time=request_time, status_code='SUCCEEDED', name=None) - ] - for item in items: - tables.jobs_table.put_item(Item=item) +def test_get_new_user(client, tables, monkeypatch): + monkeypatch.setenv('DEFAULT_CREDITS_PER_USER', '25') - login(client, 'user_with_jobs') + login(client, 'user') response = client.get(USER_URI) assert response.status_code == HTTPStatus.OK assert response.json == { - 'user_id': 'user_with_jobs', - 'quota': { - 'max_jobs_per_month': 25, - 'remaining': 21, - }, - 'job_names': [ - 'job1', - 'job2', - ], + 'user_id': 'user', + 'remaining_credits': 25, + 'job_names': [], } -def test_user_at_quota(client, tables, monkeypatch): - monkeypatch.setenv('MONTHLY_JOB_QUOTA_PER_USER', '25') - request_time = format_time(datetime.now(timezone.utc)) - - items = [make_db_record(f'job{ii}', request_time=request_time) for ii in range(0, 24)] - for item in items: - tables.jobs_table.put_item(Item=item) - - login(client) - response = client.get(USER_URI) - assert response.status_code == HTTPStatus.OK - assert response.json['quota']['remaining'] == 1 - - tables.jobs_table.put_item(Item=make_db_record('anotherJob', request_time=request_time)) - response = client.get(USER_URI) - assert response.status_code == HTTPStatus.OK - assert response.json['quota']['remaining'] == 0 - - tables.jobs_table.put_item(Item=make_db_record('yetAnotherJob', request_time=request_time)) - response = client.get(USER_URI) - assert response.status_code == HTTPStatus.OK - assert response.json['quota']['remaining'] == 0 - - -def test_get_user_custom_quota(client, tables): - username = 'user_with_custom_quota' - login(client, username) - tables.users_table.put_item(Item={'user_id': username, 'max_jobs_per_month': 50}) +def test_get_existing_user(client, tables): + user = {'user_id': 'user', 'remaining_credits': None} + tables.users_table.put_item(Item=user) + login(client, 'user') response = client.get(USER_URI) assert response.status_code == HTTPStatus.OK assert response.json == { - 'user_id': username, - 'quota': { - 'max_jobs_per_month': 50, - 'remaining': 50, - }, + 'user_id': 'user', + 'remaining_credits': None, 'job_names': [], } -def test_get_user_no_quota(client, tables): - username = 'user_with_no_quota' - login(client, username) - tables.users_table.put_item(Item={'user_id': username, 'max_jobs_per_month': None}) +def test_get_user_with_jobs(client, tables): + user_id = 'user_with_jobs' + user = {'user_id': user_id, 'remaining_credits': 20, 'foo': 'bar'} + tables.users_table.put_item(Item=user) + + request_time = format_time(datetime.now(timezone.utc)) + items = [ + make_db_record('job1', user_id=user_id, request_time=request_time, status_code='PENDING', name='job1'), + make_db_record('job2', user_id=user_id, request_time=request_time, status_code='RUNNING', name='job1'), + make_db_record('job3', user_id=user_id, request_time=request_time, status_code='FAILED', name='job2'), + make_db_record('job4', user_id=user_id, request_time=request_time, status_code='SUCCEEDED', name=None) + ] + for item in items: + tables.jobs_table.put_item(Item=item) + login(client, 'user_with_jobs') response = client.get(USER_URI) assert response.status_code == HTTPStatus.OK assert response.json == { - 'user_id': username, - 'quota': { - 'max_jobs_per_month': None, - 'remaining': None, - }, - 'job_names': [], + 'user_id': 'user_with_jobs', + 'remaining_credits': 20, + 'job_names': [ + 'job1', + 'job2', + ], } diff --git a/tests/test_api/test_submit_job.py b/tests/test_api/test_submit_job.py index 9d7bf53df..1a0a80f05 100644 --- a/tests/test_api/test_submit_job.py +++ b/tests/test_api/test_submit_job.py @@ -1,7 +1,7 @@ -from datetime import datetime, timedelta, timezone +from datetime import datetime, timezone from http import HTTPStatus -from test_api.conftest import DEFAULT_USERNAME, login, make_db_record, make_job, setup_requests_mock, submit_batch +from test_api.conftest import DEFAULT_USERNAME, login, make_job, setup_requests_mock, submit_batch from dynamo.util import format_time @@ -112,24 +112,22 @@ def test_submit_many_jobs(client, tables): assert response.status_code == HTTPStatus.BAD_REQUEST -def test_submit_exceeds_quota(client, tables, monkeypatch): +def test_submit_exceeds_remaining_credits(client, tables, monkeypatch): login(client) - time_for_previous_month = format_time(datetime.now(timezone.utc) - timedelta(days=32)) - job_from_previous_month = make_db_record('0ddaeb98-7636-494d-9496-03ea4a7df266', - request_time=time_for_previous_month) - tables.jobs_table.put_item(Item=job_from_previous_month) + monkeypatch.setenv('DEFAULT_CREDITS_PER_USER', '25') - monkeypatch.setenv('MONTHLY_JOB_QUOTA_PER_USER', '25') - batch = [make_job() for ii in range(25)] - setup_requests_mock(batch) + batch1 = [make_job() for _ in range(20)] + setup_requests_mock(batch1) - response = submit_batch(client, batch) - assert response.status_code == HTTPStatus.OK + response1 = submit_batch(client, batch1) + assert response1.status_code == HTTPStatus.OK - response = submit_batch(client) - assert response.status_code == HTTPStatus.BAD_REQUEST - assert '25 jobs' in response.json['detail'] - assert '0 jobs' in response.json['detail'] + batch2 = [make_job() for _ in range(10)] + setup_requests_mock(batch2) + + response2 = submit_batch(client, batch2) + assert response2.status_code == HTTPStatus.BAD_REQUEST + assert response2.json['detail'] == 'These jobs would cost 10.0 credits, but you have only 5.0 remaining.' def test_submit_without_jobs(client): diff --git a/tests/test_dynamo/test_jobs.py b/tests/test_dynamo/test_jobs.py index 65a8939d9..ee9833856 100644 --- a/tests/test_dynamo/test_jobs.py +++ b/tests/test_dynamo/test_jobs.py @@ -1,6 +1,6 @@ +import unittest.mock from datetime import datetime, timezone from decimal import Decimal -from unittest.mock import MagicMock, call, patch import pytest from conftest import list_have_same_elements @@ -8,77 +8,6 @@ import dynamo -def test_count_jobs(tables): - table_items = [ - { - 'job_id': 'job1', - 'user_id': 'user1', - 'status_code': 'status1', - 'request_time': '2000-01-01T00:00:00+00:00', - }, - { - 'job_id': 'job2', - 'user_id': 'user1', - 'status_code': 'status1', - 'request_time': '2000-01-01T00:00:00+00:00', - }, - { - 'job_id': 'job3', - 'user_id': 'user2', - 'status_code': 'status1', - 'request_time': '2000-01-01T00:00:00+00:00', - }, - ] - for item in table_items: - tables.jobs_table.put_item(Item=item) - - assert dynamo.jobs.count_jobs('user1') == 2 - assert dynamo.jobs.count_jobs('user2') == 1 - - -def test_count_jobs_by_start(tables): - table_items = [ - { - 'job_id': 'job1', - 'user_id': 'user1', - 'status_code': 'status1', - 'request_time': '2000-01-01T00:00:00+00:00', - }, - { - 'job_id': 'job2', - 'user_id': 'user1', - 'status_code': 'status1', - 'request_time': '2000-01-02T00:00:00+00:00', - }, - { - 'job_id': 'job3', - 'user_id': 'user1', - 'status_code': 'status1', - 'request_time': '2000-01-03T00:00:00+00:00', - }, - ] - for item in table_items: - tables.jobs_table.put_item(Item=item) - - start = '2000-01-01T00:00:00+00:00' - end = '2000-01-03T00:00:00+00:00' - response = dynamo.jobs.count_jobs('user1', start, end) - assert response == 3 - - start = '2000-01-01T00:00:01+00:00' - end = '2000-01-02T00:59:59+00:00' - response = dynamo.jobs.count_jobs('user1', start, end) - assert response == 1 - - start = '2000-01-01T00:00:01+00:00' - response = dynamo.jobs.count_jobs('user1', start, None) - assert response == 2 - - end = '2000-01-02T00:59:59+00:00' - response = dynamo.jobs.count_jobs('user1', None, end) - assert response == 2 - - def test_query_jobs_by_user(tables): table_items = [ { @@ -253,41 +182,65 @@ def test_query_jobs_by_type(tables): assert list_have_same_elements(response, table_items[:2]) -def test_put_jobs(tables): - payload = [ - { - 'name': 'name1', - }, - { - 'name': 'name1', - }, - { - 'name': 'name2', - }, - ] - jobs = dynamo.jobs.put_jobs('user1', payload) +def test_put_jobs(tables, monkeypatch): + monkeypatch.setenv('DEFAULT_CREDITS_PER_USER', '10') + payload = [{'name': 'name1'}, {'name': 'name1'}, {'name': 'name2'}] + + with unittest.mock.patch('dynamo.user._get_current_month') as mock_get_current_month: + mock_get_current_month.return_value = '2024-02' + + jobs = dynamo.jobs.put_jobs('user1', payload) + + mock_get_current_month.assert_called_once_with() + assert len(jobs) == 3 for job in jobs: assert set(job.keys()) == { - 'name', 'job_id', 'user_id', 'status_code', 'execution_started', 'request_time', 'priority' + 'name', 'job_id', 'user_id', 'status_code', 'execution_started', 'request_time', 'priority', 'credit_cost' } assert job['request_time'] <= dynamo.util.format_time(datetime.now(timezone.utc)) assert job['user_id'] == 'user1' assert job['status_code'] == 'PENDING' assert job['execution_started'] is False + assert job['credit_cost'] == 1 - response = tables.jobs_table.scan() - assert response['Items'] == jobs + assert tables.jobs_table.scan()['Items'] == jobs + + assert tables.users_table.scan()['Items'] == [ + {'user_id': 'user1', 'remaining_credits': 7, 'month_of_last_credits_reset': '2024-02'} + ] + + +def test_put_jobs_user_exists(tables): + tables.users_table.put_item(Item={'user_id': 'user1', 'remaining_credits': 5}) + + jobs = dynamo.jobs.put_jobs('user1', [{}, {}]) + + assert len(jobs) == 2 + assert tables.jobs_table.scan()['Items'] == jobs + assert tables.users_table.scan()['Items'] == [{'user_id': 'user1', 'remaining_credits': 3}] -def test_put_jobs_no_quota(tables, monkeypatch): - monkeypatch.setenv('MONTHLY_JOB_QUOTA_PER_USER', '1') +def test_put_jobs_insufficient_credits(tables, monkeypatch): + monkeypatch.setenv('DEFAULT_CREDITS_PER_USER', '1') payload = [{'name': 'name1'}, {'name': 'name2'}] - with pytest.raises(dynamo.jobs.QuotaError): - jobs = dynamo.jobs.put_jobs('user1', payload) + with unittest.mock.patch('dynamo.user._get_current_month') as mock_get_current_month: + mock_get_current_month.return_value = '2024-02' + with pytest.raises(dynamo.jobs.InsufficientCreditsError): + dynamo.jobs.put_jobs('user1', payload) - tables.users_table.put_item(Item={'user_id': 'user1', 'max_jobs_per_month': None}) + assert tables.jobs_table.scan()['Items'] == [] + assert tables.users_table.scan()['Items'] == [ + {'user_id': 'user1', 'remaining_credits': 1, 'month_of_last_credits_reset': '2024-02'} + ] + + +def test_put_jobs_infinite_credits(tables, monkeypatch): + monkeypatch.setenv('DEFAULT_CREDITS_PER_USER', '1') + payload = [{'name': 'name1'}, {'name': 'name2'}] + + tables.users_table.put_item(Item={'user_id': 'user1', 'remaining_credits': None}) jobs = dynamo.jobs.put_jobs('user1', payload) @@ -298,7 +251,7 @@ def test_put_jobs_no_quota(tables, monkeypatch): def test_put_jobs_priority_override(tables): payload = [{'name': 'name1'}, {'name': 'name2'}] - tables.users_table.put_item(Item={'user_id': 'user1', 'priority': 100}) + tables.users_table.put_item(Item={'user_id': 'user1', 'priority_override': 100, 'remaining_credits': 3}) jobs = dynamo.jobs.put_jobs('user1', payload) @@ -306,7 +259,7 @@ def test_put_jobs_priority_override(tables): for job in jobs: assert job['priority'] == 100 - tables.users_table.put_item(Item={'user_id': 'user1', 'priority': 550}) + tables.users_table.put_item(Item={'user_id': 'user1', 'priority_override': 550, 'remaining_credits': None}) jobs = dynamo.jobs.put_jobs('user1', payload) @@ -316,23 +269,45 @@ def test_put_jobs_priority_override(tables): def test_put_jobs_priority(tables): - jobs = [] - jobs.extend(dynamo.jobs.put_jobs('user1', [{}])) - jobs.extend(dynamo.jobs.put_jobs('user1', [{}, {}])) - jobs.extend(dynamo.jobs.put_jobs('user2', [{}])) + tables.users_table.put_item(Item={'user_id': 'user1', 'remaining_credits': 7}) + + jobs = dynamo.jobs.put_jobs(user_id='user1', jobs=[{}, {}, {}]) + assert jobs[0]['priority'] == 7 + assert jobs[1]['priority'] == 6 + assert jobs[2]['priority'] == 5 + + jobs.extend(dynamo.jobs.put_jobs(user_id='user1', jobs=[{}, {}, {}, {}])) + assert jobs[3]['priority'] == 4 + assert jobs[4]['priority'] == 3 + assert jobs[5]['priority'] == 2 + assert jobs[6]['priority'] == 1 + + +def test_put_jobs_priority_extra_credits(tables): + tables.users_table.put_item(Item={'user_id': 'user1', 'remaining_credits': 10_003}) + + jobs = dynamo.jobs.put_jobs(user_id='user1', jobs=[{}]) assert jobs[0]['priority'] == 9999 - assert jobs[1]['priority'] == 9998 - assert jobs[2]['priority'] == 9997 + + jobs.extend(dynamo.jobs.put_jobs(user_id='user1', jobs=[{}])) + assert jobs[1]['priority'] == 9999 + + jobs.extend(dynamo.jobs.put_jobs(user_id='user1', jobs=[{}] * 6)) + assert jobs[2]['priority'] == 9999 assert jobs[3]['priority'] == 9999 + assert jobs[4]['priority'] == 9999 + assert jobs[5]['priority'] == 9998 + assert jobs[6]['priority'] == 9997 + assert jobs[7]['priority'] == 9996 + +def test_put_jobs_decrement_credits_failure(tables): + with unittest.mock.patch('dynamo.user.decrement_credits') as mock_decrement_credits: + mock_decrement_credits.side_effect = Exception('test error') + with pytest.raises(Exception, match=r'^test error$'): + dynamo.jobs.put_jobs('user1', [{'name': 'job1'}]) -def test_put_jobs_priority_overflow(tables, monkeypatch): - monkeypatch.setenv('MONTHLY_JOB_QUOTA_PER_USER', '10001') - many_jobs = [{} for ii in range(10001)] - jobs = dynamo.jobs.put_jobs('user3', many_jobs) - assert jobs[-1]['priority'] == 0 - assert jobs[-2]['priority'] == 0 - assert jobs[-3]['priority'] == 1 + assert tables.jobs_table.scan()['Items'] == [] def test_get_job(tables): @@ -493,27 +468,6 @@ def test_get_jobs_waiting_for_execution(tables): assert dynamo.jobs.get_jobs_waiting_for_execution(limit=6) == [items[0], items[1], items[4], items[6], items[9]] -def test_put_jobs_exceeds_quota(tables): - tables.users_table.put_item(Item={'user_id': 'user1', 'max_jobs_per_month': 3}) - - dynamo.jobs.put_jobs('user1', [{}, {}, {}]) - assert dynamo.jobs.count_jobs('user1') == 3 - - with pytest.raises(dynamo.jobs.QuotaError): - dynamo.jobs.put_jobs('user1', [{}]) - assert dynamo.jobs.count_jobs('user1') == 3 - - dynamo.jobs.put_jobs('user2', [{} for i in range(25)]) - assert dynamo.jobs.count_jobs('user2') == 25 - - with pytest.raises(dynamo.jobs.QuotaError): - dynamo.jobs.put_jobs('user3', [{} for i in range(26)]) - - results = dynamo.jobs.put_jobs('user4', [{} for i in range(26)], fail_when_over_quota=False) - assert dynamo.jobs.count_jobs('user4') == 25 - assert len(results) == 25 - - def test_decimal_conversion(tables): table_items = [ { @@ -551,56 +505,3 @@ def test_decimal_conversion(tables): assert response[0]['float_value'] == Decimal('30.04') assert response[1]['float_value'] == Decimal('0.0') assert response[2]['float_value'] == Decimal('0.1') - - -def test_get_job_priority(): - priority = dynamo.jobs._get_job_priority(priority_override=None, has_quota=True) - assert priority(0, 0) == 9999 - assert priority(1, 8) == 9990 - assert priority(0, 9998) == 1 - assert priority(0, 9999) == 0 - assert priority(0, 10000) == 0 - - with pytest.raises(TypeError, match=r".*NoneType.*"): - priority(None, 9) - - priority = dynamo.jobs._get_job_priority(priority_override=1, has_quota=True) - assert priority(0, 0) == 1 - assert priority(1, 8) == 1 - assert priority(0, 9998) == 1 - assert priority(0, 9999) == 1 - assert priority(0, 10000) == 1 - - priority = dynamo.jobs._get_job_priority(priority_override=None, has_quota=False) - assert priority(None, 0) == 0 - assert priority(1, 8) == 0 - assert priority(None, 9998) == 0 - assert priority(None, 9999) == 0 - assert priority(None, 10000) == 0 - - priority = dynamo.jobs._get_job_priority(priority_override=2, has_quota=False) - assert priority(None, 0) == 2 - assert priority(1, 8) == 2 - assert priority(None, 9998) == 2 - assert priority(None, 9999) == 2 - assert priority(None, 10000) == 2 - - -@patch('dynamo.jobs._get_job_count_for_month') -@patch('dynamo.user.get_max_jobs_per_month') -def test_get_quota_status(mock_get_max_jobs_per_month: MagicMock, mock_get_job_count_for_month: MagicMock): - mock_get_max_jobs_per_month.return_value = 5 - mock_get_job_count_for_month.return_value = 0 - assert dynamo.jobs.get_quota_status('user1') == (5, 0, 5) - - mock_get_job_count_for_month.return_value = 1 - assert dynamo.jobs.get_quota_status('user1') == (5, 1, 4) - - mock_get_job_count_for_month.return_value = 10 - assert dynamo.jobs.get_quota_status('user1') == (5, 10, 0) - - mock_get_max_jobs_per_month.return_value = None - assert dynamo.jobs.get_quota_status('user1') == (None, None, None) - - assert mock_get_max_jobs_per_month.mock_calls == [call('user1') for _ in range(4)] - assert mock_get_job_count_for_month.mock_calls == [call('user1') for _ in range(3)] diff --git a/tests/test_dynamo/test_user.py b/tests/test_dynamo/test_user.py index a6f28e12f..0dbab7679 100644 --- a/tests/test_dynamo/test_user.py +++ b/tests/test_dynamo/test_user.py @@ -1,49 +1,280 @@ -import dynamo - - -def test_get_user(tables): - table_items = [ - { - 'user_id': 'user1', - 'max_jobs_per_user': 5 - }, - { - 'user_id': 'user2', - 'max_jobs_per_user': 15 - }, +import unittest.mock +from decimal import Decimal + +import botocore.exceptions +import pytest + +import dynamo.user + + +def test_get_or_create_user_reset(tables, monkeypatch): + monkeypatch.setenv('DEFAULT_CREDITS_PER_USER', '25') + tables.users_table.put_item(Item={'user_id': 'foo'}) + + with unittest.mock.patch('dynamo.user._get_current_month') as mock_get_current_month, \ + unittest.mock.patch('dynamo.user._reset_credits_if_needed') as mock_reset_credits_if_needed: + mock_get_current_month.return_value = '2024-02' + mock_reset_credits_if_needed.return_value = 'reset_credits_return_value' + + user = dynamo.user.get_or_create_user('foo') + + mock_get_current_month.assert_called_once_with() + mock_reset_credits_if_needed.assert_called_once_with( + user={'user_id': 'foo'}, + default_credits=Decimal(25), + current_month='2024-02', + users_table=tables.users_table, + ) + + assert user == 'reset_credits_return_value' + + +def test_get_or_create_user_create(tables, monkeypatch): + monkeypatch.setenv('DEFAULT_CREDITS_PER_USER', '25') + + with unittest.mock.patch('dynamo.user._get_current_month') as mock_get_current_month, \ + unittest.mock.patch('dynamo.user._create_user') as mock_create_user: + mock_get_current_month.return_value = '2024-02' + mock_create_user.return_value = 'create_user_return_value' + + user = dynamo.user.get_or_create_user('foo') + + mock_get_current_month.assert_called_once_with() + mock_create_user.assert_called_once_with( + user_id='foo', + default_credits=Decimal(25), + current_month='2024-02', + users_table=tables.users_table, + ) + + assert user == 'create_user_return_value' + + +def test_create_user(tables): + user = dynamo.user._create_user( + user_id='foo', + default_credits=Decimal(25), + current_month='2024-02', + users_table=tables.users_table + ) + + assert user == {'user_id': 'foo', 'remaining_credits': Decimal(25), 'month_of_last_credits_reset': '2024-02'} + assert tables.users_table.scan()['Items'] == [user] + + +def test_create_user_already_exists(tables): + tables.users_table.put_item(Item={'user_id': 'foo'}) + + with pytest.raises(dynamo.user.DatabaseConditionException): + dynamo.user._create_user( + user_id='foo', + default_credits=Decimal(25), + current_month='2024-02', + users_table=tables.users_table + ) + + assert tables.users_table.scan()['Items'] == [{'user_id': 'foo'}] + + +def test_reset_credits(tables, monkeypatch): + monkeypatch.setenv('RESET_CREDITS_MONTHLY', 'true') + + original_user_record = { + 'user_id': 'foo', 'remaining_credits': Decimal(10), 'month_of_last_credits_reset': '2024-01' + } + tables.users_table.put_item(Item=original_user_record) + + user = dynamo.user._reset_credits_if_needed( + user=original_user_record, + default_credits=Decimal(25), + current_month='2024-02', + users_table=tables.users_table, + ) + + assert user == {'user_id': 'foo', 'remaining_credits': Decimal(25), 'month_of_last_credits_reset': '2024-02'} + assert tables.users_table.scan()['Items'] == [user] + + +def test_reset_credits_override(tables, monkeypatch): + monkeypatch.setenv('RESET_CREDITS_MONTHLY', 'true') + + original_user_record = { + 'user_id': 'foo', + 'remaining_credits': Decimal(10), + 'credits_per_month': Decimal(50), + 'month_of_last_credits_reset': '2024-01', + } + tables.users_table.put_item(Item=original_user_record) + + user = dynamo.user._reset_credits_if_needed( + user=original_user_record, + default_credits=Decimal(25), + current_month='2024-02', + users_table=tables.users_table, + ) + + assert user == { + 'user_id': 'foo', + 'remaining_credits': Decimal(50), + 'credits_per_month': Decimal(50), + 'month_of_last_credits_reset': '2024-02', + } + assert tables.users_table.scan()['Items'] == [user] + + +def test_reset_credits_no_reset(tables, monkeypatch): + monkeypatch.setenv('RESET_CREDITS_MONTHLY', 'false') + + original_user_record = { + 'user_id': 'foo', 'remaining_credits': Decimal(10), 'month_of_last_credits_reset': '2024-01' + } + tables.users_table.put_item(Item=original_user_record) + + user = dynamo.user._reset_credits_if_needed( + user=original_user_record, + default_credits=Decimal(25), + current_month='2024-02', + users_table=tables.users_table, + ) + + assert user == {'user_id': 'foo', 'remaining_credits': Decimal(10), 'month_of_last_credits_reset': '2024-01'} + assert tables.users_table.scan()['Items'] == [user] + + +def test_reset_credits_same_month(tables, monkeypatch): + monkeypatch.setenv('RESET_CREDITS_MONTHLY', 'true') + + original_user_record = { + 'user_id': 'foo', 'remaining_credits': Decimal(10), 'month_of_last_credits_reset': '2024-02' + } + tables.users_table.put_item(Item=original_user_record) + + user = dynamo.user._reset_credits_if_needed( + user=original_user_record, + default_credits=Decimal(25), + current_month='2024-02', + users_table=tables.users_table, + ) + + assert user == {'user_id': 'foo', 'remaining_credits': Decimal(10), 'month_of_last_credits_reset': '2024-02'} + assert tables.users_table.scan()['Items'] == [user] + + +def test_reset_credits_infinite_credits(tables, monkeypatch): + monkeypatch.setenv('RESET_CREDITS_MONTHLY', 'true') + + original_user_record = { + 'user_id': 'foo', 'remaining_credits': None, 'month_of_last_credits_reset': '2024-01' + } + tables.users_table.put_item(Item=original_user_record) + + user = dynamo.user._reset_credits_if_needed( + user=original_user_record, + default_credits=Decimal(25), + current_month='2024-02', + users_table=tables.users_table, + ) + + assert user == {'user_id': 'foo', 'remaining_credits': None, 'month_of_last_credits_reset': '2024-01'} + assert tables.users_table.scan()['Items'] == [user] + + +def test_reset_credits_failed_same_month(tables, monkeypatch): + monkeypatch.setenv('RESET_CREDITS_MONTHLY', 'true') + tables.users_table.put_item( + Item={'user_id': 'foo', 'remaining_credits': Decimal(10), 'month_of_last_credits_reset': '2024-02'} + ) + + with pytest.raises(dynamo.user.DatabaseConditionException): + dynamo.user._reset_credits_if_needed( + user={'user_id': 'foo', 'remaining_credits': Decimal(10), 'month_of_last_credits_reset': '2024-01'}, + default_credits=Decimal(25), + current_month='2024-02', + users_table=tables.users_table, + ) + + assert tables.users_table.scan()['Items'] == [ + {'user_id': 'foo', 'remaining_credits': Decimal(10), 'month_of_last_credits_reset': '2024-02'} ] - for item in table_items: - tables.users_table.put_item(Item=item) - assert dynamo.user.get_user('user1') == table_items[0] - assert dynamo.user.get_user('user2') == table_items[1] - assert dynamo.user.get_user('foo') is None +def test_reset_credits_failed_infinite_credits(tables, monkeypatch): + monkeypatch.setenv('RESET_CREDITS_MONTHLY', 'true') + tables.users_table.put_item( + Item={'user_id': 'foo', 'remaining_credits': None, 'month_of_last_credits_reset': '2024-01'} + ) + + with pytest.raises(dynamo.user.DatabaseConditionException): + dynamo.user._reset_credits_if_needed( + user={'user_id': 'foo', 'remaining_credits': Decimal(10), 'month_of_last_credits_reset': '2024-01'}, + default_credits=Decimal(25), + current_month='2024-02', + users_table=tables.users_table, + ) + + assert tables.users_table.scan()['Items'] == [ + {'user_id': 'foo', 'remaining_credits': None, 'month_of_last_credits_reset': '2024-01'} + ] + + +def test_decrement_credits(tables): + tables.users_table.put_item(Item={'user_id': 'foo', 'remaining_credits': Decimal(25)}) + + dynamo.user.decrement_credits('foo', 1) + assert tables.users_table.scan()['Items'] == [{'user_id': 'foo', 'remaining_credits': Decimal(24)}] + + dynamo.user.decrement_credits('foo', 4) + assert tables.users_table.scan()['Items'] == [{'user_id': 'foo', 'remaining_credits': Decimal(20)}] + + dynamo.user.decrement_credits('foo', 20) + assert tables.users_table.scan()['Items'] == [{'user_id': 'foo', 'remaining_credits': Decimal(0)}] + + +def test_decrement_credits_invalid_cost(tables): + with pytest.raises(ValueError, match=r'^Cost 0 <= 0$'): + dynamo.user.decrement_credits('foo', 0) + + assert tables.users_table.scan()['Items'] == [] + + with pytest.raises(ValueError, match=r'^Cost -1 <= 0$'): + dynamo.user.decrement_credits('foo', -1) + + assert tables.users_table.scan()['Items'] == [] + + +def test_decrement_credits_cost_too_high(tables): + tables.users_table.put_item(Item={'user_id': 'foo', 'remaining_credits': Decimal(1)}) + + with pytest.raises(dynamo.user.DatabaseConditionException): + dynamo.user.decrement_credits('foo', 2) + + assert tables.users_table.scan()['Items'] == [{'user_id': 'foo', 'remaining_credits': Decimal(1)}] + + dynamo.user.decrement_credits('foo', 1) + + assert tables.users_table.scan()['Items'] == [{'user_id': 'foo', 'remaining_credits': Decimal(0)}] + + with pytest.raises(dynamo.user.DatabaseConditionException): + dynamo.user.decrement_credits('foo', 1) -def test_get_max_jobs_per_month(tables, monkeypatch): - monkeypatch.setenv('MONTHLY_JOB_QUOTA_PER_USER', '5') - assert dynamo.user.get_max_jobs_per_month('user1') == 5 + assert tables.users_table.scan()['Items'] == [{'user_id': 'foo', 'remaining_credits': Decimal(0)}] - user = {'user_id': 'user1'} - tables.users_table.put_item(Item=user) - assert dynamo.user.get_max_jobs_per_month('user1') == 5 - user['max_jobs_per_month'] = 10 - tables.users_table.put_item(Item=user) - assert dynamo.user.get_max_jobs_per_month('user1') == 10 +def test_decrement_credits_infinite_credits(tables): + tables.users_table.put_item(Item={'user_id': 'foo', 'remaining_credits': None}) - user['max_jobs_per_month'] = None - tables.users_table.put_item(Item=user) - assert dynamo.user.get_max_jobs_per_month('user1') is None + with pytest.raises( + botocore.exceptions.ClientError, + match=r'^An error occurred \(ValidationException\) when calling the UpdateItem operation:' + r' An operand in the update expression has an incorrect data type$' + ): + dynamo.user.decrement_credits('foo', 1) + assert tables.users_table.scan()['Items'] == [{'user_id': 'foo', 'remaining_credits': None}] -def test_get_priority(tables): - assert dynamo.user.get_priority('user1') is None - user = {'user_id': 'user1'} - tables.users_table.put_item(Item=user) - assert dynamo.user.get_priority('user1') is None +def test_decrement_credits_user_does_not_exist(tables): + with pytest.raises(dynamo.user.DatabaseConditionException): + dynamo.user.decrement_credits('foo', 1) - user['priority'] = 10 - tables.users_table.put_item(Item=user) - assert dynamo.user.get_priority('user1') == 10 + assert tables.users_table.scan()['Items'] == []