CUMULUS 2670 - Update ingest module's tasks to allow configurable tim…

…eouts (#2439) * Update ingest module's tasks to allow configurable timeouts * Update CHANGELOG * Update CHANGELOG.md Co-authored-by: Mark Boyd <[email protected]> * Add user doc Co-authored-by: Mark Boyd <[email protected]>
nasa · Sep 7, 2021 · 1f1c173 · 1f1c173
1 parent 5ccc180
commit 1f1c173
Show file tree

Hide file tree

Showing 19 changed files with 83 additions and 13 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -16,6 +16,20 @@ of response and not the raw API endpoint response
 
 ### Added
 
+- **CUMULUS-2670**
+  - Updated core `cumulus` module to take lambda_timeouts string map variable that allows timeouts of ingest tasks to be configurable. Allowed properties for the mapping include:
+  - discover_granules_task_timeout
+  - discover_pdrs_task_timeout
+  - hyrax_metadata_update_tasks_timeout
+  - lzards_backup_task_timeout
+  - move_granules_task_timeout
+  - parse_pdr_task_timeout
+  - pdr_status_check_task_timeout
+  - post_to_cmr_task_timeout
+  - queue_granules_task_timeout
+  - queue_pdrs_task_timeout
+  - queue_workflow_task_timeout
+  - sync_granule_task_timeout
 - **CUMULUS-2575**
   - Adds `POST /granules` API endpoint to create a granule
   - Adds helper `createGranule` to `@cumulus/api-client`

diff --git a/docs/configuration/ingest-task-configuration.md b/docs/configuration/ingest-task-configuration.md
@@ -0,0 +1,37 @@
+---
+id: ingest-task-configuration
+title: Configuration of Ingest Tasks
+hide_title: false
+---
+
+The `cumulus` module exposes values for configuration of some of the provided ingest workflow tasks.   Currently the following are available as configurable variables:
+
+## lambda_timeouts
+
+A configurable map of timeouts (in seconds) for cumulus ingest module task lambdas in the form:
+
+```text
+<lambda_identifier>_timeout: <timeout>"
+  type = map(string)
+```
+
+Currently the following values are supported:
+
+- discover_granules_task_timeout
+- discover_pdrs_task_timeout
+- hyrax_metadata_update_tasks_timeout
+- lzards_backup_task_timeout
+- move_granules_task_timeout
+- parse_pdr_task_timeout
+- pdr_status_check_task_timeout
+- post_to_cmr_task_timeout
+- queue_granules_task_timeout
+- queue_pdrs_task_timeout
+- queue_workflow_task_timeout
+- sync_granule_task_timeout
+
+### Example
+
+```tf
+{ "discover_granules_task_timeout": 300 }
+```
diff --git a/example/cumulus-tf/variables.tf b/example/cumulus-tf/variables.tf
@@ -393,6 +393,11 @@ variable "ddl_dir" {
   description = "The location of the ddl dir that contains the sql to create the application database."
 }
 
+variable "lambda_timeouts" {
+  type = map(string)
+  default = {}
+}
+
 variable "lambda_timeout" {
   default = 300
   type = number

diff --git a/tf-modules/cumulus/ingest.tf b/tf-modules/cumulus/ingest.tf
@@ -31,6 +31,7 @@ module "ingest" {
   cmr_page_size      = var.cmr_page_size
   cmr_custom_host    = var.cmr_custom_host
 
+  lambda_timeouts    = var.lambda_timeouts
   # Launchpad config
   launchpad_api         = var.launchpad_api
   launchpad_passphrase  = var.launchpad_passphrase

diff --git a/tf-modules/cumulus/variables.tf b/tf-modules/cumulus/variables.tf
@@ -100,6 +100,12 @@ variable "elasticsearch_security_group_id" {
   default     = ""
 }
 
+variable "lambda_timeouts" {
+  description = "Configurable map of timeouts for ingest task lambdas in the form <lambda_identifier>_timeout: <timeout>"
+  type = map(string)
+  default = {}
+}
+
 variable "prefix" {
   description = "The unique prefix for your deployment resources"
   type        = string

diff --git a/tf-modules/ingest/discover-granules-task.tf b/tf-modules/ingest/discover-granules-task.tf
@@ -6,7 +6,7 @@ resource "aws_lambda_function" "discover_granules_task" {
   handler          = "index.handler"
   role             = var.lambda_processing_role_arn
   runtime          = "nodejs12.x"
-  timeout          = 300
+  timeout          = lookup (var.lambda_timeouts, "discover_granules_task_timeout", 300)
   memory_size      = 512
 
   layers = [var.cumulus_message_adapter_lambda_layer_version_arn]

diff --git a/tf-modules/ingest/discover-pdrs-task.tf b/tf-modules/ingest/discover-pdrs-task.tf
@@ -5,7 +5,7 @@ resource "aws_lambda_function" "discover_pdrs_task" {
   handler          = "index.handler"
   role             = var.lambda_processing_role_arn
   runtime          = "nodejs12.x"
-  timeout          = 300
+  timeout          = lookup (var.lambda_timeouts, "discover_pdrs_task_timeout", 300)
   memory_size      = 1024
 
   layers = [var.cumulus_message_adapter_lambda_layer_version_arn]

diff --git a/tf-modules/ingest/hyrax-metadata-updates-task.tf b/tf-modules/ingest/hyrax-metadata-updates-task.tf
@@ -5,7 +5,7 @@ resource "aws_lambda_function" "hyrax_metadata_updates_task" {
   handler          = "index.handler"
   role             = var.lambda_processing_role_arn
   runtime          = "nodejs12.x"
-  timeout          = 300
+  timeout          = lookup (var.lambda_timeouts, "hyrax_metadata_updates_task_timeout", 300)
   memory_size      = 256
 
   layers = [var.cumulus_message_adapter_lambda_layer_version_arn]

diff --git a/tf-modules/ingest/lzards-backup-task.tf b/tf-modules/ingest/lzards-backup-task.tf
@@ -6,7 +6,7 @@ resource "aws_lambda_function" "lzards_backup_task" {
   handler          = "index.handler"
   role             = var.lambda_processing_role_arn
   runtime          = "nodejs12.x"
-  timeout          = 900
+  timeout          = lookup (var.lambda_timeouts, "lzards_backup_task_timeout", 900)
   memory_size      = 512
 
   layers = [var.cumulus_message_adapter_lambda_layer_version_arn]

diff --git a/tf-modules/ingest/move-granules-task.tf b/tf-modules/ingest/move-granules-task.tf
@@ -5,7 +5,7 @@ resource "aws_lambda_function" "move_granules_task" {
   handler          = "index.handler"
   role             = var.lambda_processing_role_arn
   runtime          = "nodejs12.x"
-  timeout          = 300
+  timeout          = lookup (var.lambda_timeouts, "move_granules_task_timeout", 300)
   memory_size      = 1024
 
   layers = [var.cumulus_message_adapter_lambda_layer_version_arn]

diff --git a/tf-modules/ingest/parse-pdr-task.tf b/tf-modules/ingest/parse-pdr-task.tf
@@ -5,7 +5,7 @@ resource "aws_lambda_function" "parse_pdr_task" {
   handler          = "index.handler"
   role             = var.lambda_processing_role_arn
   runtime          = "nodejs12.x"
-  timeout          = 300
+  timeout          = lookup (var.lambda_timeouts, "parse_pdr_task_timeout", 300)
   memory_size      = 1024
 
   layers = [var.cumulus_message_adapter_lambda_layer_version_arn]

diff --git a/tf-modules/ingest/pdr-status-check-task.tf b/tf-modules/ingest/pdr-status-check-task.tf
@@ -5,7 +5,7 @@ resource "aws_lambda_function" "pdr_status_check_task" {
   handler          = "index.handler"
   role             = var.lambda_processing_role_arn
   runtime          = "nodejs12.x"
-  timeout          = 300
+  timeout          = lookup (var.lambda_timeouts, "pdr_status_check_task_timeout", 300)
   memory_size      = 1024
 
   layers = [var.cumulus_message_adapter_lambda_layer_version_arn]

diff --git a/tf-modules/ingest/post-to-cmr-task.tf b/tf-modules/ingest/post-to-cmr-task.tf
@@ -5,7 +5,7 @@ resource "aws_lambda_function" "post_to_cmr_task" {
   handler          = "index.handler"
   role             = var.lambda_processing_role_arn
   runtime          = "nodejs12.x"
-  timeout          = 300
+  timeout          = lookup (var.lambda_timeouts, "post_to_cmr_task_timeout", 300)
   memory_size      = 256
 
   layers = [var.cumulus_message_adapter_lambda_layer_version_arn]

diff --git a/tf-modules/ingest/queue-granules-task.tf b/tf-modules/ingest/queue-granules-task.tf
@@ -5,7 +5,7 @@ resource "aws_lambda_function" "queue_granules_task" {
   handler          = "index.handler"
   role             = var.lambda_processing_role_arn
   runtime          = "nodejs12.x"
-  timeout          = 300
+  timeout          = lookup (var.lambda_timeouts, "queue_granules_task_timeout", 300)
   memory_size      = 1024
 
   layers = [var.cumulus_message_adapter_lambda_layer_version_arn]

diff --git a/tf-modules/ingest/queue-pdrs-task.tf b/tf-modules/ingest/queue-pdrs-task.tf
@@ -5,7 +5,7 @@ resource "aws_lambda_function" "queue_pdrs_task" {
   handler          = "index.handler"
   role             = var.lambda_processing_role_arn
   runtime          = "nodejs12.x"
-  timeout          = 300
+  timeout          = lookup(var.lambda_timeouts, "queue_pdrs_task_timeout", 300)
   memory_size      = 1024
 
   layers = [var.cumulus_message_adapter_lambda_layer_version_arn]

diff --git a/tf-modules/ingest/queue-workflow-task.tf b/tf-modules/ingest/queue-workflow-task.tf
@@ -5,7 +5,7 @@ resource "aws_lambda_function" "queue_workflow_task" {
   handler          = "index.handler"
   role             = var.lambda_processing_role_arn
   runtime          = "nodejs12.x"
-  timeout          = 300
+  timeout          = lookup(var.lambda_timeouts, "queue_workflow_task_timeout", 300)
   memory_size      = 1024
 
   layers = [var.cumulus_message_adapter_lambda_layer_version_arn]

diff --git a/tf-modules/ingest/sync-granule-task.tf b/tf-modules/ingest/sync-granule-task.tf
@@ -5,7 +5,7 @@ resource "aws_lambda_function" "sync_granule_task" {
   handler          = "index.handler"
   role             = var.lambda_processing_role_arn
   runtime          = "nodejs12.x"
-  timeout          = 300
+  timeout          = lookup (var.lambda_timeouts, "sync_granule_task_timeout", 300)
   memory_size      = 1024
 
   layers = [var.cumulus_message_adapter_lambda_layer_version_arn]

diff --git a/tf-modules/ingest/variables.tf b/tf-modules/ingest/variables.tf
@@ -142,6 +142,12 @@ variable "tags" {
   default     = {}
 }
 
+variable "lambda_timeouts" {
+  description = "Configurable map of timeouts for ingest task lambdas in the form <lambda_identifier>_timeout: <timeout>"
+  type = map(string)
+  default = {}
+}
+
 variable "throttled_queues" {
   description = "Array of configuration for custom queues with execution limits"
   type    = list(object({

diff --git a/website/sidebars.json b/website/sidebars.json
@@ -37,7 +37,8 @@
       "configuration/server_access_logging",
       "configuration/cloudwatch-retention",
       "configuration/lifecycle-policies",
-      "configuration/collection-storage-best-practices"
+      "configuration/collection-storage-best-practices",
+      "configuration/ingest-task-configuration"
     ],
     "Development": [
       "workflows/developing-a-cumulus-workflow",