From abcacd6f84f5ddf6d3e1cadf77aa21fe362f3549 Mon Sep 17 00:00:00 2001 From: Maxwell Plotkin Date: Fri, 16 Aug 2024 14:54:40 -0700 Subject: [PATCH 1/2] Create MMGIS-IAC dir --- .gitignore | 3 + .../terraform/modules/ec2-docker/add-mmgis.sh | 55 +++++ MMGIS-IAC/terraform/modules/ec2-docker/bk.tf | 44 ++++ MMGIS-IAC/terraform/modules/ec2-docker/lb.tf | 70 ++++++ .../terraform/modules/ec2-docker/main.tf | 18 ++ .../terraform/modules/ec2-docker/output.tf | 84 +++++++ .../terraform/modules/ec2-docker/variables.tf | 108 +++++++++ MMGIS-IAC/terraform/terraform.tf | 128 +++++++++++ MMGIS-IAC/terraform/terraform.tfvars | 8 + MMGIS-IAC/terraform/variables.tf | 211 ++++++++++++++++++ 10 files changed, 729 insertions(+) create mode 100644 MMGIS-IAC/terraform/modules/ec2-docker/add-mmgis.sh create mode 100644 MMGIS-IAC/terraform/modules/ec2-docker/bk.tf create mode 100644 MMGIS-IAC/terraform/modules/ec2-docker/lb.tf create mode 100644 MMGIS-IAC/terraform/modules/ec2-docker/main.tf create mode 100644 MMGIS-IAC/terraform/modules/ec2-docker/output.tf create mode 100644 MMGIS-IAC/terraform/modules/ec2-docker/variables.tf create mode 100644 MMGIS-IAC/terraform/terraform.tf create mode 100644 MMGIS-IAC/terraform/terraform.tfvars create mode 100644 MMGIS-IAC/terraform/variables.tf diff --git a/.gitignore b/.gitignore index 1758360..8e9281e 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ aws_role_create/identity.txt aws_role_create/output.txt aws_role_create/policies.list aws_role_create/role.txt + +MMGIS-IAC/terraform/.terraform +MMGIS-IAC/terraform/.terraform* \ No newline at end of file diff --git a/MMGIS-IAC/terraform/modules/ec2-docker/add-mmgis.sh b/MMGIS-IAC/terraform/modules/ec2-docker/add-mmgis.sh new file mode 100644 index 0000000..0c051a3 --- /dev/null +++ b/MMGIS-IAC/terraform/modules/ec2-docker/add-mmgis.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# From https://gist.github.com/jamesmishra/18ee5d7d053db9958d0e4ccbb37f8e1d +set -Eeuxo pipefail +# Filesystem code is adapted from: +# https://github.com/GSA/devsecops-example/blob/03067f68ee2765f8477ae84235f7faa1d2f2cb70/terraform/files/attach-data-volume.sh +DEVICE=${local.block_device_path} +DEST=${var.persistent_volume_mount_path} +devpath=$(readlink -f $DEVICE) + +if [[ $(file -s $devpath) != *ext4* && -b $devpath ]]; then + # Filesystem has not been created. Create it! + mkfs -t ext4 $devpath +fi +# add to fstab if not present +if ! egrep "^$devpath" /etc/fstab; then + echo "$devpath $DEST ext4 defaults,nofail,noatime,nodiratime,barrier=0,data=writeback 0 2" | tee -a /etc/fstab > /dev/null +fi +mkdir -p $DEST +mount $DEST +chown ec2-user:ec2-user $DEST +chmod 0755 $DEST + +# Filesystem code is over +# Now we install docker and docker-compose. +# Adapted from: +# https://gist.github.com/npearce/6f3c7826c7499587f00957fee62f8ee9 +yum update -y +amazon-linux-extras install docker +systemctl start docker.service +usermod -a -G docker ec2-user +chkconfig docker on +yum install -y python3-pip +python3 -m pip install docker-compose + +# Put the docker-compose.yml file at the root of our persistent volume +cat > $DEST/docker-compose.yml <<-TEMPLATE +${var.docker_compose_str} +TEMPLATE + +# Write the systemd service that manages us bringing up the service +cat > /etc/systemd/system/mmgis.service <<-TEMPLATE +[Unit] +Description=${var.description} +After=${var.systemd_after_stage} +[Service] +Type=simple +User=${var.user} +ExecStart=/usr/local/bin/docker-compose -f $DEST/docker-compose.yml up +Restart=on-failure +[Install] +WantedBy=multi-user.target +TEMPLATE + +# Start the service. +systemctl start mmgis \ No newline at end of file diff --git a/MMGIS-IAC/terraform/modules/ec2-docker/bk.tf b/MMGIS-IAC/terraform/modules/ec2-docker/bk.tf new file mode 100644 index 0000000..d464a42 --- /dev/null +++ b/MMGIS-IAC/terraform/modules/ec2-docker/bk.tf @@ -0,0 +1,44 @@ +locals { + block_device_path = "/dev/sdh" +} + +resource "aws_iam_instance_profile" "unity_mmgis_instance_profile" { + name = "unity-mmgis-instance-profile-tf" + + role = var.role + + tags = { + Name = "unity_mmgis_instance_profile" + } +} + +resource "aws_ebs_volume" "persistent" { + availability_zone = aws_instance.unity_mmgis_instance.availability_zone + size = var.persistent_volume_size_gb +} + +resource "aws_volume_attachment" "persistent" { + device_name = local.block_device_path + volume_id = aws_ebs_volume.persistent.id + instance_id = aws_instance.unity_mmgis_instance.id +} + + +resource "aws_instance" "unity_mmgis_instance" { + ami = var.ami + instance_type = var.instance_type + + tags = { + Name = "unity-mmgis-instance-tf" + } + + key_name = var.key_name + + vpc_security_group_ids = [var.sg_id] + + subnet_id = var.subnet_id + + iam_instance_profile = aws_iam_instance_profile.unity_mmgis_instance_profile.name + + user_data = file("./modules/ec2-docker/add-mmgis.sh") +} \ No newline at end of file diff --git a/MMGIS-IAC/terraform/modules/ec2-docker/lb.tf b/MMGIS-IAC/terraform/modules/ec2-docker/lb.tf new file mode 100644 index 0000000..0a5ea2b --- /dev/null +++ b/MMGIS-IAC/terraform/modules/ec2-docker/lb.tf @@ -0,0 +1,70 @@ +# target group +resource "aws_lb_target_group" "unity_mmgis_tg_tf" { + name = "unity-mmgis-tg-tf" + port = 8080 + protocol = "TCP" + target_type = "instance" + #vpc_id = data.aws_vpc.default.id + vpc_id = var.vpc_id + + health_check { + enabled = true + protocol = "HTTP" + port = 8080 + path = "/unity/v0/collections/MUR25-JPL-L4-GLOB-v4.2_analysed_sst/processes" + interval = 30 + timeout = 10 + matcher = 200 + healthy_threshold = 5 + unhealthy_threshold = 2 + } + + tags = { + Name = "unity_mmgis_tg_tf" + } +} + +# attach instance +resource "aws_lb_target_group_attachment" "unity_mmgis_tg_attachment_tf" { + target_group_arn = aws_lb_target_group.unity_mmgis_tg_tf.arn + target_id = aws_instance.unity_mmgis_instance.id + port = 8080 +} + +# create alb +resource "aws_lb" "unity-mmgis-lb-tf" { + name = "unity-mmgis-lb-tf" + load_balancer_type = "network" + internal = true + #security_groups = [var.sg_id] + #security_groups = [] + #subnets = [for subnet in aws_subnet.public : subnet.id] + subnets = var.subnet_ids + + enable_deletion_protection = false + + #access_logs { + # bucket = "tbd" + # prefix = "mmgis/tbd/unity-mmgis-lb" + # enabled = true + #} + + tags = { + Name = "unity-mmgis-lb-tf" + } +} + +resource "aws_lb_listener" "unity_mmgis_lb_listener" { + load_balancer_arn = aws_lb.unity-mmgis-lb-tf.arn + port = 80 + protocol = "TCP" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.unity_mmgis_tg_tf.arn + } + + tags = { + Name = "unity_mmgis_lb_listener" + } +} \ No newline at end of file diff --git a/MMGIS-IAC/terraform/modules/ec2-docker/main.tf b/MMGIS-IAC/terraform/modules/ec2-docker/main.tf new file mode 100644 index 0000000..d727f24 --- /dev/null +++ b/MMGIS-IAC/terraform/modules/ec2-docker/main.tf @@ -0,0 +1,18 @@ +data "aws_ssm_parameter" "vpc_id" { + name = "/unity/account/network/vpc_id" +} + +data "aws_ssm_parameter" "subnet_list" { + name = "/unity/account/network/subnet_list" +} + +#data "aws_ssm_parameter" "u-cs-ecs" { +# name = "/unity/account/ecs/execution_role_arn" +#} + + +locals { + subnet_map = jsondecode(data.aws_ssm_parameter.subnet_list.value) + subnet_ids = nonsensitive(local.subnet_map["private"]) + public_subnet_ids = nonsensitive(local.subnet_map["public"]) +} \ No newline at end of file diff --git a/MMGIS-IAC/terraform/modules/ec2-docker/output.tf b/MMGIS-IAC/terraform/modules/ec2-docker/output.tf new file mode 100644 index 0000000..7136f94 --- /dev/null +++ b/MMGIS-IAC/terraform/modules/ec2-docker/output.tf @@ -0,0 +1,84 @@ +# We try to match the API contract that `aws_instance` has. +# Descriptions for these outputs are copied from: +# https://www.terraform.io/docs/providers/aws/r/instance.html +output "id" { + description = "The instance ID" + value = aws_instance.unity_mmgis_instance.id +} + +output "arn" { + description = "The ARN of the instance" + value = aws_instance.unity_mmgis_instance.arn +} + +output "availability_zone" { + description = "The availability zone of the instance" + value = aws_instance.unity_mmgis_instance.availability_zone +} + +output "placement_group" { + description = "The placement group of the instance" + value = aws_instance.unity_mmgis_instance.placement_group +} + +output "public_dns" { + description = "The public DNS name assigned to the instance. For EC2-VPC, this is only available if you've enabled DNS hostnames for your VPC" + value = aws_instance.unity_mmgis_instance.public_dns +} + +output "public_ip" { + description = "The public IP address assigned to the instance, if applicable. NOTE: If you are using an aws_eip with your instance, you should refer to the EIP's address directly and not use public_ip, as this field will change after the EIP is attached." + value = aws_instance.unity_mmgis_instance.public_ip +} + +output "ipv6_addresses" { + description = "A list of assigned IPv6 addresses, if any" + value = aws_instance.unity_mmgis_instance.ipv6_addresses +} + +output "primary_network_interface_id" { + description = "The ID of the instance's primary network interface" + value = aws_instance.unity_mmgis_instance.primary_network_interface_id +} + +output "private_dns" { + description = " The private DNS name assigned to the instance. Can only be used inside the Amazon EC2, and only available if you've enabled DNS hostnames for your VPC" + value = aws_instance.unity_mmgis_instance.private_dns +} + +output "private_ip" { + description = "The private IP address assigned to the instance" + value = aws_instance.unity_mmgis_instance.private_ip +} + +output "security_groups" { + description = " The associated security groups." + value = aws_instance.unity_mmgis_instance.security_groups +} + +output "vpc_security_group_ids" { + description = "The associated security groups in non-default VPC." + value = aws_instance.unity_mmgis_instance.vpc_security_group_ids +} + +output "subnet_id" { + description = "The VPC subnet ID." + value = aws_instance.unity_mmgis_instance.subnet_id +} + +output "credit_specification" { + description = " Credit specification of instance." + value = aws_instance.unity_mmgis_instance.credit_specification +} + +output "instance_state" { + description = "The state of the instance. One of: pending, running, shutting-down, terminated, stopping, stopped. See Instance Lifecycle for more information." + value = aws_instance.unity_mmgis_instance.instance_state +} + +# TODO: This is a list with the `aws_instance` resource and we are just +# returning a string. I know there is an obvious solution for this... +output "ebs_block_device_id" { + description = "The persistent block device that we are storing information on." + value = aws_ebs_volume.persistent.id +} \ No newline at end of file diff --git a/MMGIS-IAC/terraform/modules/ec2-docker/variables.tf b/MMGIS-IAC/terraform/modules/ec2-docker/variables.tf new file mode 100644 index 0000000..84cbbcc --- /dev/null +++ b/MMGIS-IAC/terraform/modules/ec2-docker/variables.tf @@ -0,0 +1,108 @@ +variable role { + type = string + default = "unset" +} + +variable vpc_id { + type = string + default = "unset" +} + +variable subnet_ids { + type = list + default = ["unset"] +} + +variable ami { + type = string + default = "unset" +} + +variable sg_id { + type = string + default = "unset" +} + +### + +variable "name" { + description = "Name to be used on all resources" + type = string + default = "mmgis" +} + +variable "description" { + description = "Description of the service for systemd" + type = string + default = "" +} + +variable "availability_zone" { + description = "The availability zone for both the AWS instance and the EBS volume." + type = string + default = "us-gov-west-1" +} + +variable "systemd_after_stage" { + description = "When to run our container. This usually does not need to change." + type = string + default = "network.target" +} + +variable "user" { + description = "What user to run as. You will need to run as root to use one of the lower ports." + type = string + default = "root" +} + +variable "key_name" { + description = "Name of the SSH key to log in with" + type = string + default = "mmgis-sds.ssh" +} + +variable "instance_type" { + description = "The default AWS instance size to run these containers on" + type = string + default = "t3.medium" +} + +variable "docker_compose_str" { + description = "The entire docker compose file to write." + type = string +} + +variable "subnet_id" { + description = "The VPC subnet to launch the instance in" + type = string +} + +variable "vpc_security_group_ids" { + description = "The security groups that the instance should have" + type = list(string) + default = [] +} + +variable "iam_instance_profile" { + description = "The name of the IAM instance profile to give to the EC2 instance" + type = string + default = "" +} + +variable "associate_public_ip_address" { + description = "Whether to associate a public IP address in the VPC" + type = bool + default = false +} + +variable "persistent_volume_size_gb" { + description = "The size of the volume mounted" + type = number + default = 20 +} + +variable "persistent_volume_mount_path" { + description = "Where on the filesystem to mount our persistent volume" + type = string + default = "/persistent" +} \ No newline at end of file diff --git a/MMGIS-IAC/terraform/terraform.tf b/MMGIS-IAC/terraform/terraform.tf new file mode 100644 index 0000000..282a574 --- /dev/null +++ b/MMGIS-IAC/terraform/terraform.tf @@ -0,0 +1,128 @@ +provider "aws" { + region = var.region + profile = var.profile + + default_tags { + tags = var.common_tags + } +} + +# ===== OUR MAGIC DOCKER-COMPOSE.YML FILE HERE ===== +# It is also possible to get Terraform to read an external `docker-compose.yml` +# file and load it into this variable. +# We'll be showing off a demo nginx page. +variable "docker-compose" { + type = string + default = < Date: Fri, 13 Sep 2024 12:15:42 -0700 Subject: [PATCH 2/2] MMGIS terraform deploys without error --- MMGIS-IAC/terraform/modules/ec2-docker/bk.tf | 34 ++++++++++++++++---- MMGIS-IAC/terraform/modules/ec2-docker/lb.tf | 4 +-- MMGIS-IAC/terraform/terraform.tf | 10 ++++++ MMGIS-IAC/terraform/terraform.tfvars | 2 +- 4 files changed, 40 insertions(+), 10 deletions(-) diff --git a/MMGIS-IAC/terraform/modules/ec2-docker/bk.tf b/MMGIS-IAC/terraform/modules/ec2-docker/bk.tf index d464a42..67c4fde 100644 --- a/MMGIS-IAC/terraform/modules/ec2-docker/bk.tf +++ b/MMGIS-IAC/terraform/modules/ec2-docker/bk.tf @@ -2,16 +2,38 @@ locals { block_device_path = "/dev/sdh" } +data "aws_iam_role" "existing_role" { + name = "Unity-CS_Service_Role" +} + +data "aws_ssm_parameter" "mmgis_ami_id" { + name = "/mcp/amis/ubuntu2004-cset" +} + +data "aws_ssm_parameter" "subnet_id" { + name = "/unity/account/network/publicsubnet1" +} + +resource "aws_security_group" "allow_tls" { + name = "allow_tls" + description = "Allow TLS inbound traffic and all outbound traffic" + vpc_id = data.aws_ssm_parameter.vpc_id.value + + tags = { + Name = "allow_tls" + } +} + resource "aws_iam_instance_profile" "unity_mmgis_instance_profile" { name = "unity-mmgis-instance-profile-tf" - role = var.role + role = data.aws_iam_role.existing_role.name tags = { Name = "unity_mmgis_instance_profile" } } - + resource "aws_ebs_volume" "persistent" { availability_zone = aws_instance.unity_mmgis_instance.availability_zone size = var.persistent_volume_size_gb @@ -25,18 +47,16 @@ resource "aws_volume_attachment" "persistent" { resource "aws_instance" "unity_mmgis_instance" { - ami = var.ami + ami = data.aws_ssm_parameter.mmgis_ami_id.value instance_type = var.instance_type tags = { Name = "unity-mmgis-instance-tf" } - key_name = var.key_name - - vpc_security_group_ids = [var.sg_id] + vpc_security_group_ids = [ aws_security_group.allow_tls.id ] - subnet_id = var.subnet_id + subnet_id = data.aws_ssm_parameter.subnet_id.value iam_instance_profile = aws_iam_instance_profile.unity_mmgis_instance_profile.name diff --git a/MMGIS-IAC/terraform/modules/ec2-docker/lb.tf b/MMGIS-IAC/terraform/modules/ec2-docker/lb.tf index 0a5ea2b..658ad8f 100644 --- a/MMGIS-IAC/terraform/modules/ec2-docker/lb.tf +++ b/MMGIS-IAC/terraform/modules/ec2-docker/lb.tf @@ -5,7 +5,7 @@ resource "aws_lb_target_group" "unity_mmgis_tg_tf" { protocol = "TCP" target_type = "instance" #vpc_id = data.aws_vpc.default.id - vpc_id = var.vpc_id + vpc_id = data.aws_ssm_parameter.vpc_id.value health_check { enabled = true @@ -39,7 +39,7 @@ resource "aws_lb" "unity-mmgis-lb-tf" { #security_groups = [var.sg_id] #security_groups = [] #subnets = [for subnet in aws_subnet.public : subnet.id] - subnets = var.subnet_ids + subnets = jsondecode(data.aws_ssm_parameter.subnet_list.value).public enable_deletion_protection = false diff --git a/MMGIS-IAC/terraform/terraform.tf b/MMGIS-IAC/terraform/terraform.tf index 282a574..3a11844 100644 --- a/MMGIS-IAC/terraform/terraform.tf +++ b/MMGIS-IAC/terraform/terraform.tf @@ -7,6 +7,14 @@ provider "aws" { } } +data "aws_ssm_parameter" "vpc_id" { + name = "/unity/account/network/vpc_id" +} + +data "aws_ssm_parameter" "subnet_list" { + name = "/unity/account/network/subnet_list" +} + # ===== OUR MAGIC DOCKER-COMPOSE.YML FILE HERE ===== # It is also possible to get Terraform to read an external `docker-compose.yml` # file and load it into this variable. @@ -97,6 +105,8 @@ EOF resource "aws_security_group" "allow_http" { name = "allow_http" description = "Show off how we run a docker-compose file." + vpc_id = data.aws_ssm_parameter.vpc_id.value + ingress { from_port = 80 diff --git a/MMGIS-IAC/terraform/terraform.tfvars b/MMGIS-IAC/terraform/terraform.tfvars index 35d7d28..b140baa 100644 --- a/MMGIS-IAC/terraform/terraform.tfvars +++ b/MMGIS-IAC/terraform/terraform.tfvars @@ -5,4 +5,4 @@ secret = "secret" db_user = "postgres" db_pass = "postgres" region = "us-west-2" -profile = "429178552491_mcp-tenantDeveloper" \ No newline at end of file +profile = "429178552491_mcp-tenantOperator" \ No newline at end of file