From 8878b26eeef580bcdd05d6d1dcdde276b0f42d93 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Thu, 11 Apr 2024 18:02:42 +0300 Subject: [PATCH] Support multiple rds instances (#9) * Allow multiple RDS instances * Update the test example * PR reviews updates * Note that the instance needs to be a writer * Set target group port to RDS instance port * Fix a typo * Change lambda status if update fails * Add source_code_hash * Terraform format --- README.md | 9 +- datasources.tf | 21 +++-- .../rds_privatelink_setup/.terraform.lock.hcl | 39 -------- examples/rds_privatelink_setup/README.md | 14 ++- examples/rds_privatelink_setup/outputs.tf | 4 +- lambda_function.py | 89 ++++++++++--------- main.tf | 44 +++++---- outputs.tf | 40 +++++---- terraform.tfvars.example | 7 +- variables.tf | 17 +++- 10 files changed, 146 insertions(+), 138 deletions(-) diff --git a/README.md b/README.md index 269e4d9..3f40bcd 100644 --- a/README.md +++ b/README.md @@ -17,12 +17,15 @@ The module creates the following resources: ## Important Remarks -> **Note** +> [!NOTE] > The RDS instance needs to be private. If your RDS instance is public, there is no need to use PrivateLink. +> [!NOTE] +> When using Aurora, the RDS instance needs to be a **writer** instance as the reader instances will not work. + - The RDS instance must be in the same VPC as the PrivateLink endpoint. - Review this module with your Cloud Security team to ensure that it meets your security requirements. -- Finally, after the Terraform module has been applied, you will need to make sure that the Target Groups heatlth checks are passing. As the NLB does not have security groups, you will need to make sure that the NLB is able to reach the RDS instance by allowing the subnet CIDR blocks in the security groups of the RDS instance. +- Finally, after the Terraform module has been applied, you will need to make sure that the Target Groups health checks are passing. As the NLB does not have security groups, you will need to make sure that the NLB is able to reach the RDS instance by allowing the subnet CIDR blocks in the security groups of the RDS instance. To override the default AWS provider variables, you can export the following environment variables: @@ -43,7 +46,7 @@ cp terraform.tfvars.example terraform.tfvars | Name | Description | Type | Example | Required | |------|-------------|:----:|:-----:|:-----:| -| mz_rds_instance_name | The name of the RDS instance | string | `'my-rds-instance'` | yes | +| mz_rds_instance_names | The name of the RDS instances | list | `{ name = "instance1", listener_port = 5001 }` | yes | | mz_rds_vpc_id | The VPC ID of the RDS instance | string | `'vpc-1234567890abcdef0'` | yes | | mz_acceptance_required | Whether or not to require manual acceptance of new connections | bool | `true` | no | | schedule_expression | [The scheduling expression](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule#schedule_expression). For example, `cron(0 20 * * ? *)` | string | `'rate(5 minutes)'` | no | diff --git a/datasources.tf b/datasources.tf index bc8e804..d6d022a 100644 --- a/datasources.tf +++ b/datasources.tf @@ -1,11 +1,13 @@ -# Get the state of the RDS instance using aws_db_instance +# Get the state of the RDS instances using aws_db_instance data "aws_db_instance" "mz_rds_instance" { - db_instance_identifier = var.mz_rds_instance_name + for_each = { for inst in var.mz_rds_instance_details : inst.name => inst } + + db_instance_identifier = each.key lifecycle { postcondition { - condition = self.publicly_accessible == false - error_message = "The RDS instance needs to be private, but it is public." + condition = self.publicly_accessible == false && self.replicate_source_db == "" + error_message = "The RDS instance must be private and a writer instance." } } } @@ -16,16 +18,19 @@ data "aws_vpc" "mz_rds_vpc" { } data "aws_db_subnet_group" "mz_rds_subnet_group" { - name = data.aws_db_instance.mz_rds_instance.db_subnet_group + for_each = { for inst in var.mz_rds_instance_details : inst.name => inst } + name = data.aws_db_instance.mz_rds_instance[each.key].db_subnet_group } data "aws_subnet" "mz_rds_subnet" { - for_each = toset(data.aws_db_subnet_group.mz_rds_subnet_group.subnet_ids) - id = each.value + for_each = toset(flatten([for inst in var.mz_rds_instance_details : data.aws_db_subnet_group.mz_rds_subnet_group[inst.name].subnet_ids])) + + id = each.value } data "dns_a_record_set" "rds_ip" { - host = data.aws_db_instance.mz_rds_instance.address + for_each = { for inst in var.mz_rds_instance_details : inst.name => inst } + host = data.aws_db_instance.mz_rds_instance[each.key].address } data "aws_iam_policy_document" "lambda_assume_role_policy" { diff --git a/examples/rds_privatelink_setup/.terraform.lock.hcl b/examples/rds_privatelink_setup/.terraform.lock.hcl index c7afbdd..d762ed2 100644 --- a/examples/rds_privatelink_setup/.terraform.lock.hcl +++ b/examples/rds_privatelink_setup/.terraform.lock.hcl @@ -1,25 +1,6 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. -provider "registry.terraform.io/hashicorp/archive" { - version = "2.4.2" - hashes = [ - "h1:1eOz9vM/55vnQjxk23RhnYga7PZq8n2rGxG+2Vx2s6w=", - "zh:08faed7c9f42d82bc3d406d0d9d4971e2d1c2d34eae268ad211b8aca57b7f758", - "zh:3564112ed2d097d7e0672378044a69b06642c326f6f1584d81c7cdd32ebf3a08", - "zh:53cd9afd223c15828c1916e68cb728d2be1cbccb9545568d6c2b122d0bac5102", - "zh:5ae4e41e3a1ce9d40b6458218a85bbde44f21723943982bca4a3b8bb7c103670", - "zh:5b65499218b315b96e95c5d3463ea6d7c66245b59461217c99eaa1611891cd2c", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7f45b35a8330bebd184c2545a41782ff58240ed6ba947274d9881dd5da44b02e", - "zh:87e67891033214e55cfead1391d68e6a3bf37993b7607753237e82aa3250bb71", - "zh:de3590d14037ad81fc5cedf7cfa44614a92452d7b39676289b704a962050bc5e", - "zh:e7e6f2ea567f2dbb3baa81c6203be69f9cd6aeeb01204fd93e3cf181e099b610", - "zh:fd24d03c89a7702628c2e5a3c732c0dede56fa75a08da4a1efe17b5f881c88e2", - "zh:febf4b7b5f3ff2adff0573ef6361f09b6638105111644bdebc0e4f575373935f", - ] -} - provider "registry.terraform.io/hashicorp/aws" { version = "5.37.0" hashes = [ @@ -42,26 +23,6 @@ provider "registry.terraform.io/hashicorp/aws" { ] } -provider "registry.terraform.io/hashicorp/dns" { - version = "3.3.2" - constraints = "3.3.2" - hashes = [ - "h1:c3E2vgk4f1yNNH68MpA6SBVW1iPwaaXzYBHsUDnvIGk=", - "zh:05d2d50e301318362a4a82e6b7a9734ace07bc01abaaa649c566baf98814755f", - "zh:1e9fd1c3bfdda777e83e42831dd45b7b9e794250a0f351e5fd39762e8a0fe15b", - "zh:40e715fc7a2ede21f919567249b613844692c2f8a64f93ee64e5b68bae7ac2a2", - "zh:454d7aa83000a6e2ba7a7bfde4bcf5d7ed36298b22d760995ca5738ab02ee468", - "zh:46124ded51b4153ad90f12b0305fdbe0c23261b9669aa58a94a31c9cca2f4b19", - "zh:55a4f13d20f73534515a6b05701abdbfc54f4e375ba25b2dffa12afdad20e49d", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7903b1ceb8211e2b8c79290e2e70906a4b88f4fba71c900eb3a425ce12f1716a", - "zh:b79fc4f444ef7a2fd7111a80428c070ad824f43a681699e99ab7f83074dfedbd", - "zh:ca9f45e0c4cb94e7d62536c226024afef3018b1de84f1ea4608b51bcd497a2a0", - "zh:ddc8bd894559d7d176e0ceb0bb1ae266519b01b315362ebfee8327bb7e7e5fa8", - "zh:e77334c0794ef8f9354b10e606040f6b0b67b373f5ff1db65bddcdd4569b428b", - ] -} - provider "registry.terraform.io/hashicorp/http" { version = "3.4.1" hashes = [ diff --git a/examples/rds_privatelink_setup/README.md b/examples/rds_privatelink_setup/README.md index 21b3e66..921b2e3 100644 --- a/examples/rds_privatelink_setup/README.md +++ b/examples/rds_privatelink_setup/README.md @@ -46,19 +46,25 @@ This example demonstrates how to create a new Amazon RDS Postgres instance and c 1. Once the resources have been created, you can test the module with: + > [!NOTE] + > The module requires that each RDS instance has a **unique** listener port. + ```hcl module "materialize_privatelink_rds" { source = "../.." - mz_rds_instance_name = var.mz_rds_instance_name - mz_rds_vpc_id = module.rds_postgres.vpc.vpc_id - aws_region = var.aws_region + mz_rds_instance_details = [ + { name = "instance1", listener_port = 5001 }, + { name = "instance2", listener_port = 5002 } + ] + mz_rds_vpc_id = module.rds_postgres.vpc.vpc_id + aws_region = var.aws_region } ``` 1. **Follow the Output Instructions** - After Terraform successfully applies the configuration, it will output instructions for configuring the PrivateLink endpoint and the Postgres connection in Materialize. Follow these instructions to complete the setup. + After Terraform successfully applies the configuration, it will output instructions for configuring the PrivateLink endpoint and the Postgres connections in Materialize. Follow these instructions to complete the setup. ## Cleanup diff --git a/examples/rds_privatelink_setup/outputs.tf b/examples/rds_privatelink_setup/outputs.tf index 4055b90..8f31c47 100644 --- a/examples/rds_privatelink_setup/outputs.tf +++ b/examples/rds_privatelink_setup/outputs.tf @@ -1,9 +1,9 @@ output "rds_instance_endpoint" { - value = module.rds_postgres.rds_instance.endpoint + value = module.rds_postgres.rds_instance.endpoint sensitive = true } output "mz_rds_details" { - value = module.rds_postgres.mz_rds_details + value = module.rds_postgres.mz_rds_details sensitive = true } diff --git a/lambda_function.py b/lambda_function.py index 77cf42e..a93aeea 100644 --- a/lambda_function.py +++ b/lambda_function.py @@ -1,57 +1,60 @@ import boto3 import socket import os +import json -# Define the clients at the top of your function +# Initialize clients elbv2_client = boto3.client('elbv2') rds_client = boto3.client('rds') -RDS_IDENTIFIER = os.environ['RDS_IDENTIFIER'] # RDS instance identifier -TARGET_GROUP_ARN = os.environ['TARGET_GROUP_ARN'] # Target Group ARN +# Load RDS details from environment variables +RDS_DETAILS = json.loads(os.environ['RDS_DETAILS']) +def update_target_registration(rds_identifier, details): + try: + # Retrieve the current IP address of the RDS instance + rds_instances = rds_client.describe_db_instances(DBInstanceIdentifier=rds_identifier) + rds_port = rds_instances['DBInstances'][0]['Endpoint']['Port'] + if not rds_instances['DBInstances']: + raise Exception(f"No instances found for {rds_identifier}") -def lambda_handler(event, context): - # Retrieve the current IP address of the RDS instance - rds_instances = rds_client.describe_db_instances( - DBInstanceIdentifier=RDS_IDENTIFIER) - rds_endpoint = rds_instances['DBInstances'][0]['Endpoint']['Address'] - ip_address = socket.gethostbyname(rds_endpoint) - rds_port = rds_instances['DBInstances'][0]['Endpoint']['Port'] - - # Retrieve the existing target of the target group - targets = elbv2_client.describe_target_health( - TargetGroupArn=TARGET_GROUP_ARN) - - # Get the current IP address in the target group - if targets['TargetHealthDescriptions']: - current_ip = targets['TargetHealthDescriptions'][0]['Target']['Id'] - else: - current_ip = None - - # If the IP addresses don't match, update the target group - if current_ip and current_ip != ip_address: - # Deregister the current target - elbv2_client.deregister_targets( - TargetGroupArn=TARGET_GROUP_ARN, - Targets=[ - { - 'Id': current_ip - }, - ] - ) + rds_endpoint = rds_instances['DBInstances'][0]['Endpoint']['Address'] + ip_address = socket.gethostbyname(rds_endpoint) + + # Retrieve the existing target of the target group + target_group_arn = details['target_group_arn'] + targets = elbv2_client.describe_target_health(TargetGroupArn=target_group_arn) + + # Check and update the target group + current_ip = targets['TargetHealthDescriptions'][0]['Target']['Id'] if targets['TargetHealthDescriptions'] else None + if current_ip != ip_address: + if current_ip: + # Deregister the current target + elbv2_client.deregister_targets(TargetGroupArn=target_group_arn, Targets=[{'Id': current_ip}]) # Register the new target - elbv2_client.register_targets( - TargetGroupArn=TARGET_GROUP_ARN, - Targets=[ - { - 'Id': ip_address, - 'Port': rds_port - }, - ] - ) + elbv2_client.register_targets(TargetGroupArn=target_group_arn, Targets=[{'Id': ip_address, 'Port': rds_port}]) + message = f"Target group {target_group_arn} updated. New target IP: {ip_address}" + else: + message = f"Target group {target_group_arn} already up to date. Current target IP: {ip_address} and Port: {rds_port}" + + return {'success': True, 'message': message} + except Exception as e: + return {'success': False, 'message': f"Failed to update targets for {rds_identifier} with error: {e}"} + +def lambda_handler(event, context): + update_messages = [] + all_success = True + + for rds_identifier, details in RDS_DETAILS.items(): + result = update_target_registration(rds_identifier, details) + update_messages.append(result['message']) + if not result['success']: + all_success = False + + status_code = 200 if all_success else 500 return { - 'statusCode': 200, - 'body': f'Target group updated. Current target IP: {ip_address}' + 'statusCode': status_code, + 'body': json.dumps(update_messages) } diff --git a/main.tf b/main.tf index b06b804..e25daf9 100644 --- a/main.tf +++ b/main.tf @@ -1,24 +1,30 @@ -# Create a target group for the RDS instance +# Create a target group for each RDS instance resource "aws_lb_target_group" "mz_rds_target_group" { - name = "mz-rds-${substr(var.mz_rds_instance_name, 0, 12)}-tg" - port = data.aws_db_instance.mz_rds_instance.port + for_each = { for inst in var.mz_rds_instance_details : inst.name => inst } + + name = "${substr(each.key, 0, 12)}-${each.value.listener_port}-tg" + port = data.aws_db_instance.mz_rds_instance[each.key].port protocol = "TCP" vpc_id = data.aws_vpc.mz_rds_vpc.id target_type = "ip" } -# Attach a target to the target group +# Attach a target to each target group resource "aws_lb_target_group_attachment" "mz_rds_target_group_attachment" { - target_group_arn = aws_lb_target_group.mz_rds_target_group.arn - target_id = data.dns_a_record_set.rds_ip.addrs[0] + for_each = { for inst in var.mz_rds_instance_details : inst.name => inst } + + target_group_arn = aws_lb_target_group.mz_rds_target_group[each.key].arn + target_id = data.dns_a_record_set.rds_ip[each.key].addrs[0] + lifecycle { ignore_changes = [target_id] } + depends_on = [aws_lb_target_group.mz_rds_target_group] } # Create a network Load Balancer resource "aws_lb" "mz_rds_lb" { - name = "mz-rds-${substr(var.mz_rds_instance_name, 0, 12)}-lb" + name = var.mz_nlb_name internal = true load_balancer_type = "network" subnets = values(data.aws_subnet.mz_rds_subnet)[*].id @@ -28,14 +34,16 @@ resource "aws_lb" "mz_rds_lb" { } } -# Create a tcp listener on the Load Balancer for the RDS instance +# Create listeners for each RDS instance, mapping each to its respective target group resource "aws_lb_listener" "mz_rds_listener" { + for_each = { for inst in var.mz_rds_instance_details : inst.name => inst } + load_balancer_arn = aws_lb.mz_rds_lb.arn - port = data.aws_db_instance.mz_rds_instance.port + port = each.value.listener_port protocol = "TCP" default_action { type = "forward" - target_group_arn = aws_lb_target_group.mz_rds_target_group.arn + target_group_arn = aws_lb_target_group.mz_rds_target_group[each.key].arn } } @@ -50,30 +58,32 @@ resource "aws_vpc_endpoint_service" "mz_rds_lb_endpoint_service" { # Create an IAM policy for the Lambda function resource "aws_iam_role" "lambda_execution_role" { - name = "lambda_execution_${substr(var.mz_rds_instance_name, 0, 12)}-role" + name = "lambda_execution_${substr(var.mz_nlb_name, 0, 12)}-role" assume_role_policy = data.aws_iam_policy_document.lambda_assume_role_policy.json } # Create a Lambda function to check the RDS instance IP address resource "aws_lambda_function" "check_rds_ip" { - function_name = "${substr(var.mz_rds_instance_name, 0, 12)}-check-rds-ip" + function_name = "${substr(var.mz_nlb_name, 0, 12)}-check-rds-ip" role = aws_iam_role.lambda_execution_role.arn handler = "lambda_function.lambda_handler" runtime = "python3.11" filename = data.archive_file.lambda_zip.output_path + source_code_hash = data.archive_file.lambda_zip.output_base64sha256 + environment { variables = { - RDS_IDENTIFIER = var.mz_rds_instance_name - TARGET_GROUP_ARN = aws_lb_target_group.mz_rds_target_group.arn + RDS_DETAILS = jsonencode({ for inst in var.mz_rds_instance_details : inst.name => { port = inst.listener_port, target_group_arn = aws_lb_target_group.mz_rds_target_group[inst.name].arn } }) } } } + # Create an IAM policy for the Lambda function resource "aws_iam_role_policy" "lambda_execution_role_policy" { - name = "${substr(var.mz_rds_instance_name, 0, 12)}-lambda-execution-role-policy" + name = "${substr(var.mz_nlb_name, 0, 12)}-lambda-execution-role-policy" role = aws_iam_role.lambda_execution_role.id policy = < < data.aws_db_instance.mz_rds_instance[inst.name] } } -# Get the data.dns_a_record_set for the RDS instance +# Get the data.dns_a_record_set for each RDS instance output "mz_rds_dns" { - value = data.dns_a_record_set.rds_ip + value = { for inst in var.mz_rds_instance_details : inst.name => data.dns_a_record_set.rds_ip[inst.name] } } diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 4d33a57..454ea29 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -1,7 +1,10 @@ # Provider environment variables aws_region = "us-east-1" -# The name of the existing MSK cluster -mz_rds_instance_name = +# The name of the existing RDS instances +mz_rds_instance_details = [ + { name = "instance1", listener_port = 5001 }, + { name = "instance2", listener_port = 5002 } +] # The VPC ID of the existing MSK cluster mz_rds_vpc_id = diff --git a/variables.tf b/variables.tf index 76c34e0..005ad82 100644 --- a/variables.tf +++ b/variables.tf @@ -7,9 +7,20 @@ variable "aws_region" { # List of variables that the user would need to change -# The name of the existing RDS instance -variable "mz_rds_instance_name" { - description = "The name of the existing RDS instance" +# The names of the existing RDS instances +variable "mz_rds_instance_details" { + description = "List of objects containing RDS instance names and their corresponding unique listener ports" + type = list(object({ + name = string + listener_port = number + })) +} + +# The name of the NLB to be created +variable "mz_nlb_name" { + description = "The name of the NLB to be created" + type = string + default = "mz-rds-lb" } # The VPC ID of the existing RDS instance