EFS Volume Lookup support for ecs-service (#969)

Co-authored-by: Dan Miller <[email protected]>

modules/ecs-service/README.md

@@ -191,6 +191,58 @@ We can then create a pointer to this service in the `acme.come` hosted zone.
 
 This will create a CNAME record in the `acme.com` hosted zone that points `echo.acme.com` to `echo-server.dev-acme.com`.
 
+### EFS
+
+EFS is supported by `ecs-service`. You can use either `efs_volumes` or `efs_component_volumes` in your task definition.
+
+This example shows how to use `efs_component_volumes` which remote looks up efs component and uses the `efs_id` to mount the volume.
+And how to use `efs_volumes`
+```yaml
+components:
+  terraform:
+    ecs-services/my-service:
+      metadata:
+        component: ecs-service
+        inherits:
+          - ecs-services/defaults
+      vars:
+        containers:
+          service:
+            name: app
+            image: my-image:latest
+            log_configuration:
+              logDriver: awslogs
+              options: {}
+            port_mappings:
+              - containerPort: 8080
+                hostPort: 8080
+                protocol: tcp
+            mount_points:
+              - containerPath: "/var/lib/"
+                sourceVolume: "my-volume-mount"
+
+        task:
+          efs_component_volumes:
+            - name: "my-volume-mount"
+              host_path: null
+              efs_volume_configuration:
+                - component: efs/my-volume-mount
+                  root_directory: "/var/lib/"
+                  transit_encryption: "ENABLED"
+                  transit_encryption_port: 2999
+                  authorization_config: []
+          efs_volumes:
+            - name: "my-volume-mount-2"
+              host_path: null
+              efs_volume_ configuration:
+                - file_system_id: "fs-1234"
+                  root_directory: "/var/lib/"
+                  transit_encryption: "ENABLED"
+                  transit_encryption_port: 2998
+                  authorization_config: []
+```
+
+
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Requirements
 
@@ -224,6 +276,7 @@ This will create a CNAME record in the `acme.com` hosted zone that points `echo.
 | <a name="module_ecs_cloudwatch_autoscaling"></a> [ecs\_cloudwatch\_autoscaling](#module\_ecs\_cloudwatch\_autoscaling) | cloudposse/ecs-cloudwatch-autoscaling/aws | 0.7.3 |
 | <a name="module_ecs_cloudwatch_sns_alarms"></a> [ecs\_cloudwatch\_sns\_alarms](#module\_ecs\_cloudwatch\_sns\_alarms) | cloudposse/ecs-cloudwatch-sns-alarms/aws | 0.12.3 |
 | <a name="module_ecs_cluster"></a> [ecs\_cluster](#module\_ecs\_cluster) | cloudposse/stack-config/yaml//modules/remote-state | 1.5.0 |
+| <a name="module_efs"></a> [efs](#module\_efs) | cloudposse/stack-config/yaml//modules/remote-state | 1.5.0 |
 | <a name="module_gha_assume_role"></a> [gha\_assume\_role](#module\_gha\_assume\_role) | ../account-map/modules/team-assume-role-policy | n/a |
 | <a name="module_gha_role_name"></a> [gha\_role\_name](#module\_gha\_role\_name) | cloudposse/label/null | 0.25.0 |
 | <a name="module_iam_role"></a> [iam\_role](#module\_iam\_role) | cloudposse/stack-config/yaml//modules/remote-state | 1.5.0 |
@@ -274,7 +327,7 @@ This will create a CNAME record in the `acme.com` hosted zone that points `echo.
 | <a name="input_autoscaling_enabled"></a> [autoscaling\_enabled](#input\_autoscaling\_enabled) | Should this service autoscale using SNS alarams | `bool` | `true` | no |
 | <a name="input_chamber_service"></a> [chamber\_service](#input\_chamber\_service) | SSM parameter service name for use with chamber. This is used in chamber\_format where /$chamber\_service/$name/$container\_name/$parameter would be the default. | `string` | `"ecs-service"` | no |
 | <a name="input_cluster_attributes"></a> [cluster\_attributes](#input\_cluster\_attributes) | The attributes of the cluster name e.g. if the full name is `namespace-tenant-environment-dev-ecs-b2b` then the `cluster_name` is `ecs` and this value should be `b2b`. | `list(string)` | `[]` | no |
-| <a name="input_containers"></a> [containers](#input\_containers) | Feed inputs into container definition module | <pre>map(object({<br>    name                     = string<br>    ecr_image                = optional(string)<br>    image                    = optional(string)<br>    memory                   = optional(number)<br>    memory_reservation       = optional(number)<br>    cpu                      = optional(number)<br>    essential                = optional(bool, true)<br>    readonly_root_filesystem = optional(bool, null)<br>    privileged               = optional(bool, null)<br>    container_depends_on = optional(list(object({<br>      containerName = string<br>      condition     = string # START, COMPLETE, SUCCESS, HEALTHY<br>    })), null)<br><br>    port_mappings = optional(list(object({<br>      containerPort = number<br>      hostPort      = number<br>      protocol      = string<br>    })), [])<br>    command    = optional(list(string), null)<br>    entrypoint = optional(list(string), null)<br>    healthcheck = optional(object({<br>      command     = list(string)<br>      interval    = number<br>      retries     = number<br>      startPeriod = number<br>      timeout     = number<br>    }), null)<br>    ulimits = optional(list(object({<br>      name      = string<br>      softLimit = number<br>      hardLimit = number<br>    })), null)<br>    log_configuration = optional(object({<br>      logDriver = string<br>      options   = optional(map(string), {})<br>    }))<br>    docker_labels   = optional(map(string), null)<br>    map_environment = optional(map(string), {})<br>    map_secrets     = optional(map(string), {})<br>    volumes_from = optional(list(object({<br>      sourceContainer = string<br>      readOnly        = bool<br>    })), null)<br>    mount_points = optional(list(object({<br>      sourceVolume  = string<br>      containerPath = string<br>      readOnly      = bool<br>    })), [])<br>  }))</pre> | `{}` | no |
+| <a name="input_containers"></a> [containers](#input\_containers) | Feed inputs into container definition module | <pre>map(object({<br>    name                     = string<br>    ecr_image                = optional(string)<br>    image                    = optional(string)<br>    memory                   = optional(number)<br>    memory_reservation       = optional(number)<br>    cpu                      = optional(number)<br>    essential                = optional(bool, true)<br>    readonly_root_filesystem = optional(bool, null)<br>    privileged               = optional(bool, null)<br>    container_depends_on = optional(list(object({<br>      containerName = string<br>      condition     = string # START, COMPLETE, SUCCESS, HEALTHY<br>    })), null)<br><br>    port_mappings = optional(list(object({<br>      containerPort = number<br>      hostPort      = number<br>      protocol      = string<br>    })), [])<br>    command    = optional(list(string), null)<br>    entrypoint = optional(list(string), null)<br>    healthcheck = optional(object({<br>      command     = list(string)<br>      interval    = number<br>      retries     = number<br>      startPeriod = number<br>      timeout     = number<br>    }), null)<br>    ulimits = optional(list(object({<br>      name      = string<br>      softLimit = number<br>      hardLimit = number<br>    })), null)<br>    log_configuration = optional(object({<br>      logDriver = string<br>      options   = optional(map(string), {})<br>    }))<br>    docker_labels   = optional(map(string), null)<br>    map_environment = optional(map(string), {})<br>    map_secrets     = optional(map(string), {})<br>    volumes_from = optional(list(object({<br>      sourceContainer = string<br>      readOnly        = bool<br>    })), null)<br>    mount_points = optional(list(object({<br>      sourceVolume  = optional(string)<br>      containerPath = optional(string)<br>      readOnly      = optional(bool)<br>    })), [])<br>  }))</pre> | `{}` | no |
 | <a name="input_context"></a> [context](#input\_context) | Single object for setting entire context at once.<br>See description of individual variables for details.<br>Leave string and numeric variables as `null` to use default value.<br>Individual variable settings (non-null) override settings in context object,<br>except for attributes, tags, and additional\_tag\_map, which are merged. | `any` | <pre>{<br>  "additional_tag_map": {},<br>  "attributes": [],<br>  "delimiter": null,<br>  "descriptor_formats": {},<br>  "enabled": true,<br>  "environment": null,<br>  "id_length_limit": null,<br>  "label_key_case": null,<br>  "label_order": [],<br>  "label_value_case": null,<br>  "labels_as_tags": [<br>    "unset"<br>  ],<br>  "name": null,<br>  "namespace": null,<br>  "regex_replace_chars": null,<br>  "stage": null,<br>  "tags": {},<br>  "tenant": null<br>}</pre> | no |
 | <a name="input_cpu_utilization_high_alarm_actions"></a> [cpu\_utilization\_high\_alarm\_actions](#input\_cpu\_utilization\_high\_alarm\_actions) | A list of ARNs (i.e. SNS Topic ARN) to notify on CPU Utilization High Alarm action | `list(string)` | `[]` | no |
 | <a name="input_cpu_utilization_high_evaluation_periods"></a> [cpu\_utilization\_high\_evaluation\_periods](#input\_cpu\_utilization\_high\_evaluation\_periods) | Number of periods to evaluate for the alarm | `number` | `1` | no |
@@ -352,7 +405,7 @@ This will create a CNAME record in the `acme.com` hosted zone that points `echo.
 | <a name="input_stickiness_type"></a> [stickiness\_type](#input\_stickiness\_type) | The type of sticky sessions. The only current possible value is `lb_cookie` | `string` | `"lb_cookie"` | no |
 | <a name="input_stream_mode"></a> [stream\_mode](#input\_stream\_mode) | Stream mode details for the Kinesis stream | `string` | `"PROVISIONED"` | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | Additional tags (e.g. `{'BusinessUnit': 'XYZ'}`).<br>Neither the tag keys nor the tag values will be modified by this module. | `map(string)` | `{}` | no |
-| <a name="input_task"></a> [task](#input\_task) | Feed inputs into ecs\_alb\_service\_task module | <pre>object({<br>    task_cpu                = optional(number)<br>    task_memory             = optional(number)<br>    task_role_arn           = optional(string, "")<br>    pid_mode                = optional(string, null)<br>    ipc_mode                = optional(string, null)<br>    network_mode            = optional(string)<br>    propagate_tags          = optional(string)<br>    assign_public_ip        = optional(bool, false)<br>    use_alb_security_groups = optional(bool, true)<br>    launch_type             = optional(string, "FARGATE")<br>    scheduling_strategy     = optional(string, "REPLICA")<br>    capacity_provider_strategies = optional(list(object({<br>      capacity_provider = string<br>      weight            = number<br>      base              = number<br>    })), [])<br><br>    deployment_minimum_healthy_percent = optional(number, null)<br>    deployment_maximum_percent         = optional(number, null)<br>    desired_count                      = optional(number, 0)<br>    min_capacity                       = optional(number, 1)<br>    max_capacity                       = optional(number, 2)<br>    wait_for_steady_state              = optional(bool, true)<br>    circuit_breaker_deployment_enabled = optional(bool, true)<br>    circuit_breaker_rollback_enabled   = optional(bool, true)<br><br>    ecs_service_enabled = optional(bool, true)<br>    bind_mount_volumes = optional(list(object({<br>      name      = string<br>      host_path = string<br>    })), [])<br>    efs_volumes = optional(list(object({<br>      host_path = string<br>      name      = string<br>      efs_volume_configuration = list(object({<br>        file_system_id          = string<br>        root_directory          = string<br>        transit_encryption      = string<br>        transit_encryption_port = string<br>        authorization_config = list(object({<br>          access_point_id = string<br>          iam             = string<br>        }))<br>      }))<br>    })), [])<br>    docker_volumes = optional(list(object({<br>      host_path = string<br>      name      = string<br>      docker_volume_configuration = list(object({<br>        autoprovision = bool<br>        driver        = string<br>        driver_opts   = map(string)<br>        labels        = map(string)<br>        scope         = string<br>      }))<br>    })), [])<br>    fsx_volumes = optional(list(object({<br>      host_path = string<br>      name      = string<br>      fsx_windows_file_server_volume_configuration = list(object({<br>        file_system_id = string<br>        root_directory = string<br>        authorization_config = list(object({<br>          credentials_parameter = string<br>          domain                = string<br>        }))<br>      }))<br>    })), [])<br>  })</pre> | `{}` | no |
+| <a name="input_task"></a> [task](#input\_task) | Feed inputs into ecs\_alb\_service\_task module | <pre>object({<br>    task_cpu                = optional(number)<br>    task_memory             = optional(number)<br>    task_role_arn           = optional(string, "")<br>    pid_mode                = optional(string, null)<br>    ipc_mode                = optional(string, null)<br>    network_mode            = optional(string)<br>    propagate_tags          = optional(string)<br>    assign_public_ip        = optional(bool, false)<br>    use_alb_security_groups = optional(bool, true)<br>    launch_type             = optional(string, "FARGATE")<br>    scheduling_strategy     = optional(string, "REPLICA")<br>    capacity_provider_strategies = optional(list(object({<br>      capacity_provider = string<br>      weight            = number<br>      base              = number<br>    })), [])<br><br>    deployment_minimum_healthy_percent = optional(number, null)<br>    deployment_maximum_percent         = optional(number, null)<br>    desired_count                      = optional(number, 0)<br>    min_capacity                       = optional(number, 1)<br>    max_capacity                       = optional(number, 2)<br>    wait_for_steady_state              = optional(bool, true)<br>    circuit_breaker_deployment_enabled = optional(bool, true)<br>    circuit_breaker_rollback_enabled   = optional(bool, true)<br><br>    ecs_service_enabled = optional(bool, true)<br>    bind_mount_volumes = optional(list(object({<br>      name      = string<br>      host_path = string<br>    })), [])<br>    efs_volumes = optional(list(object({<br>      host_path = string<br>      name      = string<br>      efs_volume_configuration = list(object({<br>        file_system_id          = string<br>        root_directory          = string<br>        transit_encryption      = string<br>        transit_encryption_port = string<br>        authorization_config = list(object({<br>          access_point_id = string<br>          iam             = string<br>        }))<br>      }))<br>    })), [])<br>    efs_component_volumes = optional(list(object({<br>      host_path = string<br>      name      = string<br>      efs_volume_configuration = list(object({<br>        component   = optional(string, "efs")<br>        tenant      = optional(string, null)<br>        environment = optional(string, null)<br>        stage       = optional(string, null)<br><br>        root_directory          = string<br>        transit_encryption      = string<br>        transit_encryption_port = string<br>        authorization_config = list(object({<br>          access_point_id = string<br>          iam             = string<br>        }))<br>      }))<br>    })), [])<br>    docker_volumes = optional(list(object({<br>      host_path = string<br>      name      = string<br>      docker_volume_configuration = list(object({<br>        autoprovision = bool<br>        driver        = string<br>        driver_opts   = map(string)<br>        labels        = map(string)<br>        scope         = string<br>      }))<br>    })), [])<br>    fsx_volumes = optional(list(object({<br>      host_path = string<br>      name      = string<br>      fsx_windows_file_server_volume_configuration = list(object({<br>        file_system_id = string<br>        root_directory = string<br>        authorization_config = list(object({<br>          credentials_parameter = string<br>          domain                = string<br>        }))<br>      }))<br>    })), [])<br>  })</pre> | `{}` | no |
 | <a name="input_task_enabled"></a> [task\_enabled](#input\_task\_enabled) | Whether or not to use the ECS task module | `bool` | `true` | no |
 | <a name="input_task_iam_role_component"></a> [task\_iam\_role\_component](#input\_task\_iam\_role\_component) | A component that outputs an iam\_role module as 'role' for adding to the service as a whole. | `string` | `null` | no |
 | <a name="input_task_policy_arns"></a> [task\_policy\_arns](#input\_task\_policy\_arns) | The IAM policy ARNs to attach to the ECS task IAM role | `list(string)` | <pre>[<br>  "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly",<br>  "arn:aws:iam::aws:policy/AmazonSSMReadOnlyAccess"<br>]</pre> | no |

modules/ecs-service/main.tf

@@ -40,6 +40,28 @@ locals {
   } : {}
 
   task = merge(var.task, local.task_s3)
+
+  efs_component_volumes = lookup(local.task, "efs_component_volumes", [])
+  efs_component_map = {
+    for efs in local.efs_component_volumes : efs["name"] => efs
+  }
+  efs_component_remote_state = {
+    for efs in local.efs_component_volumes : efs["name"] => module.efs[efs["name"]].outputs
+  }
+  efs_component_merged = [for efs_volume_name, efs_component_output in local.efs_component_remote_state : {
+    host_path = local.efs_component_map[efs_volume_name].host_path
+    name      = efs_volume_name
+    efs_volume_configuration = [ #again this is a hardcoded array because AWS does not support multiple configurations per volume
+      {
+        file_system_id          = efs_component_output.efs_id
+        root_directory          = local.efs_component_map[efs_volume_name].efs_volume_configuration[0].root_directory
+        transit_encryption      = local.efs_component_map[efs_volume_name].efs_volume_configuration[0].transit_encryption
+        transit_encryption_port = local.efs_component_map[efs_volume_name].efs_volume_configuration[0].transit_encryption_port
+        authorization_config    = local.efs_component_map[efs_volume_name].efs_volume_configuration[0].authorization_config
+      }
+    ]
+  }]
+  efs_volumes = concat(lookup(local.task, "efs_volumes", []), local.efs_component_merged)
 }
 
 data "aws_s3_objects" "mirror" {
@@ -268,7 +290,7 @@ module "ecs_alb_service_task" {
   task_role_arn                      = lookup(local.task, "task_role_arn", one(module.iam_role[*]["outputs"]["role"]["arn"]))
   capacity_provider_strategies       = lookup(local.task, "capacity_provider_strategies")
 
-  efs_volumes        = lookup(local.task, "efs_volumes", [])
+  efs_volumes        = local.efs_volumes
   docker_volumes     = lookup(local.task, "docker_volumes", [])
   fsx_volumes        = lookup(local.task, "fsx_volumes", [])
   bind_mount_volumes = lookup(local.task, "bind_mount_volumes", [])

modules/ecs-service/remote-state.tf

@@ -181,3 +181,20 @@ module "iam_role" {
 
   context = module.this.context
 }
+
+module "efs" {
+  for_each = local.efs_component_map
+
+  source  = "cloudposse/stack-config/yaml//modules/remote-state"
+  version = "1.5.0"
+
+  # Here we can use [0] because aws only allows one efs volume configuration per volume
+  component = each.value.efs_volume_configuration[0].component
+
+  context = module.this.context
+
+  tenant      = each.value.efs_volume_configuration[0].tenant
+  stage       = each.value.efs_volume_configuration[0].stage
+  environment = each.value.efs_volume_configuration[0].environment
+
+}