diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f7ab196 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*.pyc +__pycache__ +venv/ diff --git a/README.md b/README.md index f08315b..aa1cd1a 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,215 @@ # pulumi -Common Pulumi elements for use in Thunderbird infrastructure development + +Common Pulumi elements for use in Thunderbird infrastructure development. + +## Usage + +Typically, you want to implement the classes defined in this module to define infrastructure +resources to support your application. These represent common infrastructural patterns which you can +customize to some degree. + +See the [Documentation](#documentation) section below for details on how to include this in your +project and use each kind of resource. + + +## Pulumi setup + +Our Pulumi code is developed against Python 3.12 or later. If this is not your default version, you'll need to manage your own virtual environment. + +Check your default version: + +```sh +$ python -V +Python 3.12.4 +``` + +If you need a newer Python, [download and install it](https://www.python.org/downloads/). Then you'll have to set up the virtual environment yourself with something like this: + +```sh +virtualenv -p /path/to/python3.12 venv +./venv/bin/pip install -r requirements.txt +``` + +After this, `pulumi` commands should work. If 3.12 is your default version of Python, Pulumi should set up its own virtualenv, and you should not have to do this. + + +## Start a new Pulumi project + +### S3 bucket + +Create an S3 bucket in which to store state for the project. Generally, you should follow this +naming scheme: + +``` +tb-$PROJECT_NAME-pulumi +``` + +One bucket can hold states for all of that project's stacks, so you only need to create the one +bucket per project. + + +### Repo setup + +You probably already have a code repo with your application code in it. If not, create such a repo. + +Create a directory there called `pulumi` and create a new project and stack in it. You'll need the +name of the S3 bucket from the previous step here. If you are operating in an AWS region other than +what is set as your default for AWSCLI, be sure to `export AWS_REGION=us-east-1` or whatever else +you may need to do to override that. + +```sh +cd /path/to/pulumi/code +pulumi login s3://$S3_BUCKET_NAME +pulumi new aws-python +``` + +Follow the prompts to get everything named. + + +### Set up this module + +Ensure your pulumi code directory contains a `requirements.txt` file with at least this repo listed: + +``` +git+https://github.com/thunderbird/pulumi.git +``` + +You can pin your code to a specific version of this module by appending `@branch_name` to that. + +Pulumi will need these requirements installed. On your first run of a `pulumi preview` command (or +some others), Pulumi will attempt to set up its working environment. If this fails, or you need to +make adjustments later, you can activate Pulumi's virtual environment to perform pip changes. +Assuming Pulumi's virtual environment lives at `venv`, run: + +```sh +./venv/bin/pip install -U -r requirements.txt +``` + +You can now develop Python Pulumi code in that directory, referring to this module with imports such +as these: + +```python +import tb_pulumi + +# ...or... + +from tb_pulumi import (ec2, fargate, secrets) +``` + + +### Use this module + +When you issue `pulumi` commands (like "up" and "preview" and so on), it looks for a `__main__.py` +file in your current directory and executes the code in that file. To use this module, you'll import +it into that file and write up some code and configuration files. As a quickstart, you can copy +`__main__.py.example` and `config.stack.yaml.example` into your repo and begin to tweak them. + + +#### Create a config file + +It is assumed that a config file will exist at `config.$STACK.yaml` where `$STACK` is the currently +selected Pulumi stack. This file must contain a mapping of names of config settings to their desired +values. Currently, only one such setting is recognized. That is `resources`. + +This is a mostly arbitary mapping that you will have to interpret on your own (more on that later), +but some conventions are recommended. Namely: + + - `resources` should be a mapping where the keys are the Pulumi type-strings for the resources + they are configuring. For example, if you want to build a VPC with several subnets, you + might use the `tb_pulumi.network.MultiCidrVpc` class. Following this convention, that should + be accompanied by a `tb:network:MultiCidrVpc` key in this mapping. + - The values these keys map to should themselves be mappings. This provides a convention where + more than one of each pattern are configurable. The keys here should be arbitrary but unique + identifiers for the resources being configured. F/ex: `backend` or `api`. + - The values these keys map to should be a mapping where the keys are valid configuration + options for the resources being built. The full listing of these values can be found by + browsing the [documentation](#documentation). + + +#### Define a ThunderbirdPulumiProject + +In your `__main__.py` file, start with a simple skeleton (or use `__main__.py.example` to start): + +```python +import tb_pulumi + +project = tb_pulumi.ThunderbirdPulumiProject() +``` + +If you have followed the conventions outlined above, `project` is now an object with a key property, +`config`, which gives you access to the config file's data. You can use this in the next step to +feed parameters into resource declarations. + + +#### Declare ThunderbirdComponentResources + +A `pulumi.ComponentResource` is a collection of related resources. In an effort to keep consistent +tagging and such across all Thunderbird infrastructure projects, the resources available in this +module all extend a custom class called a `ThunderbirdComponentResource`. If you have +followed the conventions outlined so far, it should be easy to stamp out common patterns with them +by passing config options into the constructors for these classes. + + +#### A brief example + +You should be able to run through these steps to get a very simple working example: + + - Set up a pulumi project and a stack called "foobar". + - `cp __main__.py.example /my/project/__main__.py` + - `cp config.stack.yaml.example /my/project/config.foobar.yaml` + - Tweak the config as you see fit + +A `pulumi preview` should list out a few resources to be built. Depending on how you've configured +things, this could include: + + - A VPC + - A subnet for each of the two CIDRs defined + - Internet or NAT Gateways + - Routes + + +## Documentation + + +Documentation for this module is currently maintained through this readme and the commentary in the +code. If you like, you can browse that commentary using pydoc: + +```sh +virtualenv .venv +pip install -r requirements.txt +python -m pydoc -p 8080 . +``` + +Then click [this link](http://localhost:8080/tb_pulumi.html). + + +## Implementing ThunderbirdComponentResources + +So you want to develop a new pattern to stamp out? Here's what you'll need to do: + + - Determine the best place to put the code. Is there an existing module that fits the bill? + - Determine the Pulumi type string for it. This goes: `org:module:class`. The `org` will always + be "tb". The `module` will be the Python submodule you're placing the new class in. The + `class` is whatever you've called the class. + - Design the class following these guidelines: + - The constructor should always accept, before any other arguments, the following positional + options: + - **name:** The internal name of the resource as Pulumi tracks it. + - **project:** The ThunderbirdPulumiProject these resources belong to. + - The constructor should always accept the following keyword arguments: + - **opts:** A `pulumi.ResourceOptions` object which will get merged into the default set of + arguments managed by the project. + - The constructor should explicitly define only those arguments that you intend to have + default values which differ from the default values the provider will set, or which imply + larger patterns (such as "build_jumphost" implying other resources, like a security group + and its rules, not just an EC2 instance). + - The constructor may accept a final `**kwargs` argument with arbitrary meaning. Because the + nature of a component resource is to compile many other resources into one class, it is + not implicitly clear what "everything else" should apply to. If this is implemented, its + function should be clearly documented in the class. + - The class should extend `tb_pulumi.ThunderbirdComponentResource`. + - The class should call its superconstructor in the following way: + - `super().__init__(typestring, name, project, opts=opts)` + - Any resources you create should always be assigned a key in `self.resources`. + - Any resources you create must have the `parent=self` pulumi.ResourceOption set. + - At the end of the `__init__` function, you must call `self.finish()` diff --git a/__main__.py.example b/__main__.py.example new file mode 100644 index 0000000..0ec8153 --- /dev/null +++ b/__main__.py.example @@ -0,0 +1,23 @@ +#!/bin/env python3 + +import tb_pulumi +import tb_pulumi.network + + +# Create a project to aggregate resources. This will allow consistent tagging, resource protection, +# etc. The naming is derived from the currently selected Pulumi project/stack. A configuration file +# called `config.$stack.yaml` is loaded from the current directory. See config.stack.yaml.example. +project = tb_pulumi.ThunderbirdPulumiProject() + +# Pull the "resources" config mapping +resources = project.config.get('resources') + +# Let's say we want to build a VPC with some private IP space. We can do this with a `MultiCidrVpc`. +vpc_opts = resources['tb:network:MultiCidrVpc']['vpc'] +vpc = tb_pulumi.network.MultiCidrVpc( + # project.name_prefix combines the Pulumi project and stack name to create a unique prefix + f'{project.name_prefix}-vpc', + # Add this module's resources to the project + project, + # Map the rest of the config file directly into this function call, separating code from config + **vpc_opts) \ No newline at end of file diff --git a/config.stack.yaml.example b/config.stack.yaml.example new file mode 100644 index 0000000..1561347 --- /dev/null +++ b/config.stack.yaml.example @@ -0,0 +1,12 @@ +--- +resources: + tb:network:MultiCidrVpc: + vpc: + cidr_block: 10.0.0.0/16 + subnets: + us-east-1a: + - 10.0.101.0/24 + us-east-1b: + - 10.0.102.0/24 + us-east-1c: + - 10.0.103.0/24 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e571398 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,54 @@ +[project] +name = "tb_pulumi" +version = "0.0.1" +description = "Framework and patterns for using Pulumi at Thunderbird" +requires-python = ">3.12" +dynamic = ["dependencies"] + +[project.urls] +repository = "https://github.com/thunderbird/pulumi.git" +issues = "https://github.com/thunderbird/pulumi/issues" + +[tool.setuptools.dynamic] +dependencies = { file = ["requirements.txt"] } + +# Ruff +[tool.ruff] +line-length = 120 + +# Exclude a variety of commonly ignored directories. +exclude = [ + ".eggs", + ".git", + ".ruff_cache", + ".venv", + "__pycache__", + "__pypackages__", + "venv", +] + +# Always generate Python 3.12-compatible code. +target-version = "py312" + +[tool.ruff.format] +# Prefer single quotes over double quotes. +quote-style = "single" + +[tool.ruff.lint] +# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default. +select = ["E", "F"] +ignore = [] + +# Allow autofix for all enabled rules (when `--fix`) is provided. +fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"] +unfixable = [] + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +[tool.ruff.lint.flake8-quotes] +inline-quotes = "single" + +[tool.ruff.lint.mccabe] +# Unlike Flake8, default to a complexity level of 10. +max-complexity = 10 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9298f92 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +boto3>=1.34,<2.0 +cryptography>=43.0.0,<44.0 +pulumi>=3.130.0,<4.0.0 +pulumi-aws>=6.0.2,<7.0.0 +pulumi-random>=4.16,<5.0 +# pyyaml is also a requirement, but is installed for us by pulumi \ No newline at end of file diff --git a/tb_pulumi/__init__.py b/tb_pulumi/__init__.py new file mode 100644 index 0000000..4441955 --- /dev/null +++ b/tb_pulumi/__init__.py @@ -0,0 +1,158 @@ +"""Standardization library for the usage of Pulumi in Python at Thunderbird. For an overview of how +to use this library, read the README: https://github.com/thunderbird/pulumi/blob/main/README.md +""" + +import boto3 +import pulumi +import yaml + +from functools import cached_property +from os import environ, getlogin +from socket import gethostname +from tb_pulumi.constants import DEFAULT_PROTECTED_STACKS + + +class ThunderbirdPulumiProject: + """Manages Pulumi resources at Thunderbird. This class enforces some usage conventions that help + keep us organized and consistent. + """ + + def __init__(self, protected_stacks: list[str] = DEFAULT_PROTECTED_STACKS): + """Construct a ThunderbirdPulumiProject. + + - protected_stacks: List of stack names which should require explicit instruction to + modify. + """ + + # General runtime data + self.project = pulumi.get_project() + self.stack = pulumi.get_stack() + self.name_prefix = f'{self.project}-{self.stack}' + self.protected_stacks = protected_stacks + self.pulumi_config = pulumi.Config() + self.resources = {} + self.common_tags = { + 'project': self.project, + 'pulumi_last_run_by': f'{getlogin()}@{gethostname()}', + 'pulumi_project': self.project, + 'pulumi_stack': self.stack, + } + + # AWS client setup + self.__aws_clients = {} + self.__aws_session = boto3.session.Session() + sts = self.get_aws_client('sts') + self.aws_account_id = sts.get_caller_identity()['Account'] + self.aws_region = self.__aws_session.region_name + + def get_aws_client(self, service: str): + """Retrieves an AWS client for the requested service, preferably from the cache. + + - service: Name of the service as described in boto3 docs: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/index.html + """ + if service not in self.__aws_clients.keys(): + self.__aws_clients[service] = self.__aws_session.client(service) + + return self.__aws_clients[service] + + @cached_property + def config(self) -> dict: + """Provides read-only access to the project configuration""" + + config_file = f'config.{self.stack}.yaml' + with open(config_file, 'r') as fh: + return yaml.load(fh.read(), Loader=yaml.SafeLoader) + + +class ThunderbirdComponentResource(pulumi.ComponentResource): + """A special kind of pulumi.ComponentResource which handles common aspects of our resources such + as naming, tagging, and internal resource organization in code. + """ + + def __init__( + self, + pulumi_type: str, + name: str, + project: ThunderbirdPulumiProject, + opts: pulumi.ResourceOptions = None, + tags: dict = {}, + ): + """Construct a ThunderbirdComponentResource. + + - pulumi_type: The "type" string (commonly referred to in docs as just "t") of the component + as described by Pulumi's docs here: + https://www.pulumi.com/docs/concepts/resources/names/#types + - name: A string identifying this set of resources. + - project: The ThunderbirdPulumiProject this resource belongs to. + - opts: Additional pulumi.ResourceOptions to apply to this resource. + - tags: Key/value pairs to merge with the default tags which get applied to all resources in + this group. + """ + self.name = name + self.project = project + + if self.protect_resources: + pulumi.info( + f'Resource protection has been enabled on {name}. ' + 'To disable, export TBPULUMI_DISABLE_PROTECTION=True' + ) + + # Merge provided opts with defaults before calling superconstructor + default_opts = pulumi.ResourceOptions(protect=self.protect_resources) + final_opts = default_opts.merge(opts) + super().__init__(t=pulumi_type, name=name, opts=final_opts) + + self.tags = self.project.common_tags.copy() + self.tags.update(tags) + + self.resources = {} + + def finish(self): + """Registers outputs based on the contents of `self.resources` and adds those resources to + the project's internal tracking. All implementations of this class should call this function + at the end of their __init__ functions. + """ + + # Register outputs both with the ThunderbirdPulumiProject and Pulumi itself + self.project.resources[self.name] = self.resources + self.register_outputs({k: self.resources[k] for k in self.resources.keys()}) + + @property + def protect_resources(self) -> bool: + """Sets or unsets resource protection on the stack based on operating conditions.""" + + if self.project.stack not in self.project.protected_stacks: + protect = False + else: + protect = not env_var_is_true('TBPULUMI_DISABLE_PROTECTION') + + return protect + + +def env_var_matches(name: str, matches: list[str], default: bool = False) -> bool: + """Determines if the value of the given environment variable is in the given list. Returns True + if it does, otherwise the `default` value. This is a case-insensitive check. Returns None if the + variable is unset. + + - name: The environment variable to check + - matches: A list of strings to match against + - default: Default value if the variable doesn't match + """ + + # Convert to lowercase for case-insensitive matching + matches = [match.lower() for match in matches] + value = environ.get(name, None) + if value is None: + return None + if value.lower() in matches: + return True + return default + + +def env_var_is_true(name: str) -> bool: + """Determines if the value of the given environment variable represents "True" in some way. + + - name: The environment variable to check + """ + + return env_var_matches(name, ['t', 'true', 'yes'], False) diff --git a/tb_pulumi/constants.py b/tb_pulumi/constants.py new file mode 100644 index 0000000..4227d0d --- /dev/null +++ b/tb_pulumi/constants.py @@ -0,0 +1,21 @@ +"""Some global values that should not change often and do not rely on runtime data.""" + +# Shell for ARPs +ASSUME_ROLE_POLICY = { + 'Version': '2012-10-17', + 'Statement': [{'Sid': '', 'Effect': 'Allow', 'Principal': {'Service': None}, 'Action': 'sts:AssumeRole'}], +} + +# Global default values to fall back on +DEFAULT_AWS_SSL_POLICY = 'ELBSecurityPolicy-2016-08' +DEFAULT_PROTECTED_STACKS = ['prod'] # Which Pulumi stacks should get resource protection + +# Policy document shell +IAM_POLICY_DOCUMENT = {'Version': '2012-10-17', 'Statement': [{'Sid': 'DefaultSid', 'Effect': 'Allow'}]} + +# Map of common services to their typical ports +SERVICE_PORTS = { + 'mariadb': 3306, + 'mysql': 3306, + 'postgres': 5432, +} diff --git a/tb_pulumi/ec2.py b/tb_pulumi/ec2.py new file mode 100644 index 0000000..9ef15c7 --- /dev/null +++ b/tb_pulumi/ec2.py @@ -0,0 +1,356 @@ +import pulumi +import pulumi_aws as aws +import tb_pulumi +import tb_pulumi.network +import tb_pulumi.secrets + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa + + +AMAZON_LINUX_AMI = 'ami-0427090fd1714168b' + + +class NetworkLoadBalancer(tb_pulumi.ThunderbirdComponentResource): + """Create a Network Load Balancer.""" + + def __init__( + self, + name: str, + project: tb_pulumi.ThunderbirdPulumiProject, + listener_port: int, + subnets: list[str], + target_port: int, + ingress_cidrs: list[str] = None, + internal: bool = True, + ips: list[str] = [], + security_group_description: str = None, + opts: pulumi.ResourceOptions = None, + **kwargs, + ): + """Construct a NetworkLoadBalancer to route TCP traffic to a collection of backends. This + targets backend services by IP address, connecting a frontend listening port to a + backend port on the round-robin load balanced targets. + + Positional arguments: + - name: A string identifying this set of resources. + - project: The ThunderbirdPulumiProject to add these resources to. + - listener_port: The port that the load balancer should accept traffic on. + - subnets: List of subnet resource outputs. The NLB will be built in these network + spaces, and in the VPC of the first subnet listed (they must all be in the same + VPC). + - target_port: The port to route to on the backends. + + Keyword arguments: + - ingress_cidrs: List of CIDR blocks to allow ingress to the NLB from. If not provided, + traffic to the listener_port will be allowed from anywhere. + - internal: When True (default), ingress is restricted to traffic sourced within the + VPC. When False, the NLB gets a public IP to listen on. + - ips: List of IP addresses to balance load between. + - security_group_description: Text to use for the security group's description field. + - opts: Additional pulumi.ResourceOptions to apply to these resources. + - kwargs: Any other keyword arguments which will be passed as inputs to the LoadBalancer + resource. A full listing of options is found here: + https://www.pulumi.com/registry/packages/aws/api-docs/alb/loadbalancer/#inputs + """ + + super().__init__('tb:ec2:NetworkLoadBalancer', name, project, opts=opts) + + # Build a security group that allows ingress on our listener port + self.resources['security_group_with_rules'] = tb_pulumi.network.SecurityGroupWithRules( + f'{name}-sg', + project, + vpc_id=subnets[0].vpc_id, + rules={ + 'ingress': [ + { + 'cidr_blocks': ingress_cidrs if ingress_cidrs else ['0.0.0.0/0'], + 'description': 'Allow ingress', + 'protocol': 'tcp', + 'from_port': listener_port, + 'to_port': listener_port, + } + ], + 'egress': [ + { + 'cidr_blocks': ['0.0.0.0/0'], + 'description': 'Allow egress', + 'protocol': 'tcp', + 'from_port': target_port, + 'to_port': target_port, + } + ], + }, + tags=self.tags, + opts=pulumi.ResourceOptions(parent=self), + ).resources + + # Build the load balancer first, as other resources must be attached to it later + self.resources['nlb'] = aws.alb.LoadBalancer( + f'{name}-nlb', + enable_cross_zone_load_balancing=True, + internal=internal, + load_balancer_type='network', + name=name, + security_groups=[self.resources['security_group_with_rules']['sg']], + subnets=[subnet.id for subnet in subnets], + tags=self.tags, + opts=pulumi.ResourceOptions(parent=self, depends_on=[self.resources['security_group_with_rules']['sg']]), + **kwargs, + ) + + # Build and attach a target group + self.resources['target_group'] = aws.lb.TargetGroup( + f'{name}-targetgroup', + health_check={ + 'enabled': True, + 'healthy_threshold': 3, + 'interval': 20, + 'port': target_port, + 'protocol': 'TCP', + 'timeout': 10, + 'unhealthy_threshold': 3, + }, + load_balancing_cross_zone_enabled=True, + name=name, + port=target_port, + protocol='TCP', + target_type='ip', + vpc_id=subnets[0].vpc_id, + tags=self.tags, + opts=pulumi.ResourceOptions(parent=self, depends_on=[self.resources['nlb']]), + ) + + # Add targets to the target group + self.resources['target_group_attachments'] = [] + for idx, ip in enumerate(ips): + self.resources['target_group_attachments'].append( + aws.lb.TargetGroupAttachment( + f'{name}-tga-{idx}', + target_group_arn=self.resources['target_group'].arn, + target_id=ip, + port=target_port, + opts=pulumi.ResourceOptions(parent=self, depends_on=[self.resources['target_group']]), + ) + ) + + # Build the listener, sending traffic to the target group + self.resources['listener'] = aws.lb.Listener( + f'{name}-listener', + default_actions=[{'type': 'forward', 'target_group_arn': self.resources['target_group'].arn}], + load_balancer_arn=self.resources['nlb'].arn, + port=listener_port, + protocol='TCP', + tags=self.tags, + opts=pulumi.ResourceOptions( + parent=self, depends_on=[self.resources['nlb'], self.resources['target_group']] + ), + ) + + self.finish() + + +class SshableInstance(tb_pulumi.ThunderbirdComponentResource): + """Builds an EC2 instance which can be accessed with SSH from somewhere on the Internet.""" + + def __init__( + self, + name: str, + project: tb_pulumi.ThunderbirdPulumiProject, + subnet_id: str, + ami: str = AMAZON_LINUX_AMI, + kms_key_id: str = None, + public_key: str = None, + source_cidrs: list[str] = ['0.0.0.0/0'], + user_data: str = None, + vpc_id: str = None, + vpc_security_group_ids: list[str] = None, + opts: pulumi.ResourceOptions = None, + **kwargs, + ): + """Construct an SshableInstance. + + Positional arguments: + - name: A string identifying this set of resources. + - project: The ThunderbirdPulumiProject to add these resources to. + - subnet_id: The ID of the subnet to build the instance in. + + Keyword arguments: + - ami: ID of the AMI to build the instance with. Defaults to Amazon Linux 2023. + - kms_key_id: ID of the KMS key for encrypting all database storage. + - public_key: The RSA public key used for SSH authentication. + - source_cidrs: List of CIDRs which should be allowed to open SSH connections to the + instance. + - user_data: Custom user data to launch the instance with. + - vpc_id: The VPC to build this instance in. + - vpc_security_group_ids: If provided, sets the security groups for the instance. + Otherwise, a security group allowing only port 22 from the `source_cidrs` will be + created and used. + - opts: Additional pulumi.ResourceOptions to apply to these resources. + - kwargs: Any other keyword arguments which will be passed as inputs to the + ThunderbirdComponentResource superconstructor. + """ + + super().__init__('tb:ec2:SshableInstance', name, project, opts=opts, **kwargs) + + self.resources['keypair'] = SshKeyPair(f'{name}-keypair', project, public_key=public_key).resources + + if not vpc_security_group_ids: + self.resources['security_group_with_rules'] = tb_pulumi.network.SecurityGroupWithRules( + f'{name}-sg', + project, + vpc_id=vpc_id, + rules={ + 'ingress': [ + { + 'cidr_blocks': source_cidrs, + 'description': 'SSH access', + 'protocol': 'tcp', + 'from_port': 22, + 'to_port': 22, + } + ], + 'egress': [ + { + 'cidr_blocks': ['0.0.0.0/0'], + 'description': 'Allow all egress', + 'protocol': 'tcp', + 'from_port': 0, + 'to_port': 65535, + } + ], + }, + opts=pulumi.ResourceOptions(parent=self), + ).resources + sg_ids = [self.resources['security_group_with_rules']['sg'].id] + else: + sg_ids = vpc_security_group_ids + + instance_tags = {'Name': name} + instance_tags.update(self.project.common_tags) + self.resources['instance'] = aws.ec2.Instance( + f'{name}-instance', + ami=ami, + associate_public_ip_address=True, + disable_api_stop=False, # Jump hosts should never contain live services or + disable_api_termination=False, # be the source of data; they don't need protection. + instance_type='t3.micro', + key_name=self.resources['keypair']['keypair'].key_name, + root_block_device={'encrypted': True, 'kms_key_id': kms_key_id, 'volume_size': 10, 'volume_type': 'gp3'}, + subnet_id=subnet_id, + user_data=user_data, + volume_tags=self.tags, + vpc_security_group_ids=sg_ids, + tags=instance_tags, + opts=pulumi.ResourceOptions(parent=self), + ) + + self.finish() + + +class SshKeyPair(tb_pulumi.ThunderbirdComponentResource): + """Builds an SSH keypair and stores its values in Secrets Manager. + + NOTE: This should typically be used by specifying the public_key. If you do not, Pulumi will + generate a new key for you. However, at the moment, it appears there's no way to have Pulumi + generate a private key ONE TIME and ONLY ONE TIME. Each `pulumi up/preview` command generates a + new keypair, which generates new secret versions (and if this is attached to an instance + downstream, it triggers the recreation of that instance). This is otherwise good code that will + correctly build these resources. + """ + + def __init__( + self, + name: str, + project: tb_pulumi.ThunderbirdPulumiProject, + key_size: int = 4096, + public_key: str = None, + secret_name: str = 'keypair', + opts: pulumi.ResourceOptions = None, + **kwargs, + ): + """Construct an SshKeyPair. + + Positional arguments: + - name: A string identifying this set of resources. + - project: The ThunderbirdPulumiProject to add these resources to. + + Keyword arguments: + - key_size: Byte length of the private key to generate. Only used if public_key is not + supplied. + - public_key: RSA public key to stash in the KeyPair. It is highly recommended that you + always provide this. That is, you should usually generate a keypair on your local + machine (ssh-keygen -t rsa -b 4096) and provide that public key to this resource. + - secret_name: A slash ("/") delimited name to give the Secrets Manager secret. If not + supplied, one will be generated based on `name`. Only used if public_key is not + provided. + - opts: Additional pulumi.ResourceOptions to apply to these resources. + - kwargs: Any other keyword arguments which will be passed as inputs to the + ThunderbirdComponentResource superconstructor. + """ + + super().__init__('tb:ec2:SshKeyPair', name, project, opts=opts, **kwargs) + + if not public_key: + self.resources['private_key'], self.resources['public_key'] = generate_ssh_keypair(key_size=key_size) + self.resources['keypair'] = aws.ec2.KeyPair( + f'{name}-keypair', + key_name=name, + public_key=self.resources['public_key'], + tags=self.tags, + opts=pulumi.ResourceOptions(parent=self, depends_on=[self.resources['private_key']]), + ) + + if secret_name is not None: + suffix = 'keypair' + else: + suffix = secret_name + prefix = f'{tb_pulumi.PROJECT}/{tb_pulumi.STACK}/{suffix}' + priv_secret = f'{prefix}/private_key' + pub_secret = f'{prefix}/public_key' + + self.resources['private_key_secret'] = tb_pulumi.secrets.SecretsManagerSecret( + f'{name}/privatekey', + project, + secret_name=priv_secret, + secret_value=self.resources['private_key'], + opts=pulumi.ResourceOptions(parent=self, depends_on=[self.resources['private_key']]), + ) + self.resources['public_key_secret'] = tb_pulumi.secrets.SecretsManagerSecret( + f'{name}/publickey', + project, + secret_name=pub_secret, + secret_value=self.resources['public_key'], + opts=pulumi.ResourceOptions(parent=self, depends_on=[self.resources['public_key']]), + ) + else: + self.resources['keypair'] = aws.ec2.KeyPair( + f'{name}-keypair', + key_name=name, + public_key=public_key, + tags=self.tags, + opts=pulumi.ResourceOptions(parent=self), + ) + + self.finish() + + +def generate_ssh_keypair(key_size=4096) -> (str, str): + """Returns plaintext representations of a private and public RSA key for use in SSH + authentication. + + - key_size: Byte length of the private key. + """ + + # Ref: https://cryptography.io/en/latest/hazmat/primitives/asymmetric/rsa/#module-cryptography.hazmat.primitives.asymmetric.rsa + key = rsa.generate_private_key(public_exponent=65537, key_size=key_size) + private_key = key.private_bytes( + serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption() + ).decode('utf-8') + public_key = ( + key.public_key() + .public_bytes(serialization.Encoding.OpenSSH, serialization.PublicFormat.OpenSSH) + .decode('utf-8') + ) + + return private_key, public_key diff --git a/tb_pulumi/fargate.py b/tb_pulumi/fargate.py new file mode 100644 index 0000000..92d4d52 --- /dev/null +++ b/tb_pulumi/fargate.py @@ -0,0 +1,435 @@ +import json +import pulumi +import pulumi_aws as aws +import tb_pulumi + +from tb_pulumi.constants import ASSUME_ROLE_POLICY, DEFAULT_AWS_SSL_POLICY + + +class FargateClusterWithLogging(tb_pulumi.ThunderbirdComponentResource): + """Builds a Fargate cluster running a variable number of tasks. Logs from these tasks will be + sent to CloudWatch. + """ + + def __init__( + self, + name: str, + project: tb_pulumi.ThunderbirdPulumiProject, + subnets: list[str], + assign_public_ip: bool = False, + desired_count: int = 1, + ecr_resources: list = ['*'], + enable_container_insights: bool = False, + health_check_grace_period_seconds: int = None, + internal: bool = True, + key_deletion_window_in_days: int = 7, + security_groups: list[str] = [], + services: dict = {}, + task_definition: dict = {}, + opts: pulumi.ResourceOptions = None, + **kwargs, + ): + """Construct a FargateClusterWithLogging. + + Positional arguments: + - name: A string identifying this set of resources. + - project: The ThunderbirdPulumiProject to add these resources to. + - subnets: A list of subnet IDs to build Fargate containers on. There must be at least + one subnet to use. + + Keyword arguments: + - assign_public_ip: When True, containers will receive Internet-facing network + interfaces. Must be enabled for Fargate-backed containers to talk out to the net. + - desired_count: The number of containers the service should target to run. + - ecr_resources: The containers will be granted permissions to pull images from ECR. If + you would like to restrict these permissions, supply this argument as a list of ARNs + as they would appear in an IAM Policy. + - enable_container_insights: When True, enables advanced CloudWatch metrics collection. + - health_check_grace_period_seconds: Time to wait for a container to come online before + attempting health checks. This can be used to prevent accidental health check + failures. + - internal: Whether traffic should be accepted from the Internet (False) or not (True) + - key_deletion_window_in_days: Number of days after the KMS key is deleted that it will + be recoverable. If you need to forcibly delete a key, set this to 0. + - security_groups: A list of security group IDs to attach to the load balancer. + - services: A dict defining the ports to use when routing requests to each service. The keys + should be the name of the service as described in a container definition. The values + should be dicts supporting the options shown below. If no listenter_port is specified, + the container_port will be used. The container_name is the name of a container as + specified in a container definition which can receive this traffic. + + {'web_portal': { + 'container_port': 8080, + 'container_name': 'web_backend', + 'listener_cert_arn': 'arn:aws:acm:region:account:certificate/id', + 'listener_port': 80, + 'listener_proto': 'HTTPS', + 'name': 'Arbitrary name for the ALB; must be unique and no longer than 32 characters.', + 'health_check': { + # Keys match parameters listed here: + # https://www.pulumi.com/registry/packages/aws/api-docs/alb/targetgroup/#targetgrouphealthcheck + } + }} + + - task_definition: A dict representing an ECS task definition. + - opts: Additional pulumi.ResourceOptions to apply to these resources. + - kwargs: Any other keyword arguments which will be passed as inputs to the + ThunderbirdComponentResource superconstructor. + """ + + if len(subnets) < 1: + raise IndexError('You must provide at least one subnet.') + + super().__init__('tb:fargate:FargateClusterWithLogging', name, project, opts=opts, **kwargs) + family = name + + # Key to encrypt logs + log_key_tags = {'Name': f'{name}-fargate-logs'} + log_key_tags.update(self.tags) + self.resources['log_key'] = aws.kms.Key( + f'{name}-logging', + description=f'Key to encrypt logs for {name}', + deletion_window_in_days=key_deletion_window_in_days, + tags=log_key_tags, + opts=pulumi.ResourceOptions(parent=self), + ) + + # Log group + self.resources['log_group'] = aws.cloudwatch.LogGroup( + f'{name}-fargate-logs', + name=f'{name}-fargate-logs', + tags=self.tags, + opts=pulumi.ResourceOptions(parent=self), + ) + + # Set up an assume role policy + arp = ASSUME_ROLE_POLICY.copy() + arp['Statement'][0]['Principal']['Service'] = 'ecs-tasks.amazonaws.com' + arp = json.dumps(arp) + + # IAM policy for shipping logs + doc = self.resources['log_group'].arn.apply( + lambda arn: json.dumps( + { + 'Version': '2012-10-17', + 'Statement': [ + { + 'Sid': 'AllowECSLogSending', + 'Effect': 'Allow', + 'Action': 'logs:CreateLogGroup', + 'Resource': arn, + } + ], + } + ) + ) + self.resources['policy_log_sending'] = aws.iam.Policy( + f'{name}-policy-logs', + name=f'{name}-logging', + description='Allows Fargate tasks to log to their log group', + policy=doc, + opts=pulumi.ResourceOptions(parent=self, depends_on=[self.resources['log_group']]), + ) + + # IAM policy for accessing container dependencies + doc = json.dumps( + { + 'Version': '2012-10-17', + 'Statement': [ + { + 'Sid': 'AllowSecretsAccess', + 'Effect': 'Allow', + 'Action': 'secretsmanager:GetSecretValue', + 'Resource': f'arn:aws:secretsmanager:{self.project.aws_region}:' + f'{self.project.aws_account_id}:' + f'secret:{self.project.project}/{self.project.stack}/*', + }, + { + 'Sid': 'AllowECRAccess', + 'Effect': 'Allow', + 'Action': [ + 'ecr:BatchCheckLayerAvailability', + 'ecr:BatchGetImage', + 'ecr:DescribeImages', + 'ecr:GetDownloadUrlForLayer', + 'ecr:ListImages', + 'ecr:ListTagsForResource', + ], + 'Resource': ecr_resources, + }, + { + 'Sid': 'AllowParametersAccess', + 'Effect': 'Allow', + 'Action': 'ssm:GetParameters', + 'Resource': f'arn:aws:ssm:{self.project.aws_region}:{self.project.aws_account_id}:' + f'parameter/{self.project.project}/{self.project.stack}/*', + }, + ], + } + ) + self.resources['policy_exec'] = aws.iam.Policy( + f'{name}-policy-exec', + name=f'{name}-exec', + description=f'Allows {self.project.project} tasks access to resources they need to run', + policy=doc, + opts=pulumi.ResourceOptions(parent=self), + ) + + # Create an IAM role for tasks to run as + self.resources['task_role'] = aws.iam.Role( + f'{name}-taskrole', + name=name, + description=f'Task execution role for {self.project.name_prefix}', + assume_role_policy=arp, + managed_policy_arns=[ + 'arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy', + self.resources['policy_log_sending'], + self.resources['policy_exec'], + ], + tags=self.tags, + opts=pulumi.ResourceOptions(parent=self), + ) + + # Fargate Cluster + self.resources['cluster'] = aws.ecs.Cluster( + f'{name}-cluster', + opts=pulumi.ResourceOptions( + parent=self, depends_on=[self.resources['log_key'], self.resources['log_group']] + ), + name=name, + configuration={ + 'executeCommandConfiguration': { + 'kmsKeyId': self.resources['log_key'].arn, + 'logging': 'OVERRIDE', + 'logConfiguration': { + 'cloudWatchEncryptionEnabled': True, + 'cloudWatchLogGroupName': self.resources['log_group'].name, + }, + } + }, + settings=[{'name': 'containerInsights', 'value': 'enabled' if enable_container_insights else 'disabled'}], + tags=self.tags, + ) + + # Prep the task definition + self.resources['task_definition'] = pulumi.Output.all( + self.resources['log_group'].name, self.project.aws_region, self.resources['task_role'].arn + ).apply(lambda outputs: self.task_definition(task_definition, family, outputs[0], outputs[1])) + + # Build ALBs and related resources to route traffic to our services + fsalb_name = f'{name}-fargateservicealb' + self.resources['fargate_service_alb'] = FargateServiceAlb( + fsalb_name, + project, + subnets=subnets, + internal=internal, + security_groups=security_groups, + services=services, + opts=pulumi.ResourceOptions(parent=self), + ).resources + + # We only need one Fargate Service config, but that might have multiple load balancer + # configs. Build those now. + lb_configs = [ + { + 'targetGroupArn': self.resources['fargate_service_alb']['target_groups'][svc_name].arn, + 'containerName': svc['container_name'], + 'containerPort': svc['container_port'], + } + for svc_name, svc in services.items() + ] + + # Fargate Service + self.resources['service'] = aws.ecs.Service( + f'{name}-service', + name=name, + cluster=self.resources['cluster'].id, + desired_count=desired_count, + health_check_grace_period_seconds=health_check_grace_period_seconds, + launch_type='FARGATE', + load_balancers=lb_configs, + network_configuration={ + 'subnets': subnets, + 'assign_public_ip': assign_public_ip, + 'security_groups': security_groups, + }, + task_definition=self.resources['task_definition'], + tags=self.tags, + opts=pulumi.ResourceOptions( + parent=self, depends_on=[self.resources['cluster'], self.resources['task_definition']] + ), + ) + + self.finish() + + def task_definition( + self, + task_def: dict, + family: str, + log_group_name: str, + aws_region: str, + ) -> aws.ecs.TaskDefinition: + """Returns an ECS task definition resource. + + - task_def: A dict defining the task definition template which needs modification. + - family: A unique name for the task definition. + - log_group_name: Name of the log group to ship logs to. + - aws_region: AWS region to build in. + """ + + for cont_name, cont_def in task_def['container_definitions'].items(): + # If not overridden, inject a default log configuration + if 'logConfiguration' not in cont_def: + cont_def['logConfiguration'] = { + 'logDriver': 'awslogs', + 'options': { + 'awslogs-group': log_group_name, + 'awslogs-create-group': 'true', + 'awslogs-region': aws_region, + 'awslogs-stream-prefix': 'ecs', + }, + } + cont_def['name'] = cont_name + + # Convert container defs into list + cont_defs = [v for k, v in task_def['container_definitions'].items()] + + task_def.update( + { + 'execution_role_arn': self.resources['task_role'].arn, + 'family': family, + 'container_definitions': json.dumps(cont_defs), + } + ) + + return aws.ecs.TaskDefinition( + f'{family}-taskdef', + opts=pulumi.ResourceOptions(parent=self, depends_on=[self.resources['log_group']]), + **task_def, + ) + + +class FargateServiceAlb(tb_pulumi.ThunderbirdComponentResource): + """Builds an ALB with all of its constituent components to serve traffic for a set of ECS + services. ECS does not allow reuse of a single ALB with multiple listeners, so if there are + multiple services, multiple ALBs will be constructed. + """ + + def __init__( + self, + name: str, + project: tb_pulumi.ThunderbirdPulumiProject, + subnets: list[pulumi.Output], + internal: bool = True, + security_groups: list[str] = [], + services: dict = {}, + opts: pulumi.ResourceOptions = None, + **kwargs, + ): + """Construct an ApplicationLoadBalancer. + + Positional arguments: + - name: A string identifying this set of resources. + - project: The ThunderbirdPulumiProject to add these resources to. + - subnets: A list of subnet resources (pulumi outputs) to attach the ALB to. + + Keyword arguments: + - internal: Whether traffic should be accepted from the Internet (False) or not (True). + - security_groups: A list of security group IDs to attach to the load balancer. + - services: A dict defining the ports to use when routing requests to each service. The + keys should be the name of the service as described in a container definition. The + values should be dicts supporting the options shown below. If no listenter_port is + specified, the container_port will be used. The name field is mandatory because we + have to get around a 32-character limit for naming things, and the generated names + are far too long and result in namespace collisions when automatically shortened. + + {'web_portal': { + 'container_port': 8080, + 'container_name': 'web_backend', + 'listener_cert_arn': 'arn:aws:acm:region:account:certificate/id', + 'listener_port': 80, + 'listener_proto': 'HTTPS', + 'name': 'Arbitrary name for the ALB; must be unique and no longer than 32 characters.', + 'health_check': { + # Keys match parameters listed here: + # https://www.pulumi.com/registry/packages/aws/api-docs/alb/targetgroup/#targetgrouphealthcheck + } + }} + + - opts: Additional pulumi.ResourceOptions to apply to these resources. + - kwargs: Any other keyword arguments which will be passed as inputs to the + ThunderbirdComponentResource superconstructor. + """ + + super().__init__('tb:fargate:FargateServiceAlb', name, project, opts=opts, **kwargs) + + # We'll track these per-service + self.resources['albs'] = {} + self.resources['listeners'] = {} + self.resources['target_groups'] = {} + + # For each service... + for svc_name, svc in services.items(): + # Determine SSL settings based on other values + listener_proto = svc['listener_proto'] if 'listener_proto' in svc else 'HTTP' + ssl_policy = None + if 'ssl_policy' in svc: + ssl_policy = svc['ssl_policy'] + else: + if listener_proto == 'HTTPS': + ssl_policy = DEFAULT_AWS_SSL_POLICY + + # Add special tagging to these resources to identify the service they're built for + svc_tags = {'service': svc_name} + svc_tags.update(self.tags) + + # Build the load balancer first; we'll need it for everything else + # TODO: Support access logging; AWS only supports S3 buckets, not apparently CloudWatch + self.resources['albs'][svc_name] = aws.lb.LoadBalancer( + f'{name}-alb-{svc_name}', + # AWS imposes a 32-character limit on service names. Simply cropping the name length + # down is insufficient because it creates name conflicts. So these are explicitly + # named in our configs. + name=svc['name'], + internal=internal, + load_balancer_type='application', + security_groups=security_groups, + subnets=[subnet.id for subnet in subnets], + tags=self.tags, + opts=pulumi.ResourceOptions(parent=self), + ) + + # Build a target group + tg_name = f'{name}-targetgroup-{svc_name}' + self.resources['target_groups'][svc_name] = aws.alb.TargetGroup( + tg_name, + # AWS imposes a 32-character limit on service names. Simply cropping the name length + # down is insufficient because it creates name conflicts. So these are explicitly + # named in our configs. + health_check=svc['health_check'] if 'health_check' in svc else None, + name=svc['name'], + port=svc['container_port'], + protocol='HTTP', + vpc_id=subnets[0].vpc_id, + # Next two options are required for ECS services; ref: + # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/alb.html + target_type='ip', + ip_address_type='ipv4', + tags=svc_tags, + opts=pulumi.ResourceOptions(parent=self), + ) + + # Build a listener for the target group + self.resources['listeners'][svc_name] = aws.lb.Listener( + f'{name}-listener-{svc_name}', + certificate_arn=svc['listener_cert_arn'] if 'listener_cert_arn' in svc else None, + default_actions=[{'type': 'forward', 'targetGroupArn': self.resources['target_groups'][svc_name].arn}], + load_balancer_arn=self.resources['albs'][svc_name].arn, + port=svc['listener_port'] if 'listener_port' in svc else svc['container_port'], + protocol=listener_proto, + ssl_policy=ssl_policy, + tags=svc_tags, + opts=pulumi.ResourceOptions(parent=self), + ) + + self.finish() diff --git a/tb_pulumi/network.py b/tb_pulumi/network.py new file mode 100644 index 0000000..b8dfba1 --- /dev/null +++ b/tb_pulumi/network.py @@ -0,0 +1,301 @@ +import pulumi +import pulumi_aws as aws +import tb_pulumi + + +class MultiCidrVpc(tb_pulumi.ThunderbirdComponentResource): + """Builds a VPC with configurable network space.""" + + def __init__( + self, + name: str, + project: tb_pulumi.ThunderbirdPulumiProject, + cidr_block: str = '10.0.0.0/16', + egress_via_internet_gateway: bool = False, + egress_via_nat_gateway: bool = False, + enable_dns_hostnames: bool = None, + enable_internet_gateway: bool = False, + enable_nat_gateway: bool = False, + endpoint_gateways: list[str] = [], + endpoint_interfaces: list[str] = [], + subnets: dict = {}, + opts: pulumi.ResourceOptions = None, + **kwargs, + ): + """Construct a MultiCidrVpc resource. + + Positional arguments: + - name: A string identifying this set of resources. + - project: The ThunderbirdPulumiProject to add these resources to. + + Keyword arguments: + - cidr_block: A CIDR describing the IP space of this VPC. + - egress_via_internet_gateway: When True, establish an outbound route to the Internet + via the Internet Gateway. Requires `enable_internet_gateway=True`. Conflicts with + `egress_via_nat_gateway=True`. + - egress_via_nat_gateway: When True, establish an outbound route to the Internet via + the NAT Gateway. Requires `enable_nat_gateway=True`. Conflicts with + `egress_via_internet_gateway=True`. + - enable_dns_hostnames: When True, internal DNS mappings get built for IPs assigned + within the VPC. This is required for the use of certain other services like + load-balanced Fargate clusters. + - enable_internet_gateway: Build an IGW will to allow traffic outbond to the Internet. + - enable_nat_gateway: Build a NAT Gateway to route inbound traffic. + - endpoint_gateways: List of public-facing AWS services (such as S3) to create VPC + gateways to. + - endpoint_interfaces: List of AWS services to create VPC Interface endpoints for. These + must match service names listed here: + https://docs.aws.amazon.com/vpc/latest/privatelink/aws-services-privatelink-support.html) + **Do not** list the full qualifying name, only the service name portion. f/ex, do + not use "com.amazonaws.us-east-1.secretsmanager", only use "secretsmanager". + - subnets: A dict where the keys are the names of AWS Availability Zones in which to + build subnets and the values are lists of CIDRs describing valid subsets of IPs in + the VPC `cidr_block` to build in that AZ. f/ex: + + { + 'us-east-1': ['10.0.100.0/24'], + 'us-east-2': ['10.0.101.0/24', '10.0.102.0/24'] + } + + - opts: Additional pulumi.ResourceOptions to apply to these resources. + - kwargs: Any other keyword arguments which will be passed as inputs to the + ThunderbirdComponentResource superconstructor. + """ + + super().__init__('tb:network:MultiCidrVpc', name, project, opts=opts, **kwargs) + + # Build a VPC + vpc_tags = {'Name': name} + vpc_tags.update(self.tags) + self.resources['vpc'] = aws.ec2.Vpc( + name, + opts=pulumi.ResourceOptions(parent=self), + cidr_block=cidr_block, + enable_dns_hostnames=enable_dns_hostnames, + tags=vpc_tags, + ) + + # Build subnets in that VPC + self.resources['subnets'] = [] + for idx, subnet in enumerate(subnets.items()): + az, cidrs = subnet + for cidr in cidrs: + subnet_resname = f'{name}-subnet-{idx}' + subnet_tags = {'Name': subnet_resname} + subnet_tags.update(self.tags) + self.resources['subnets'].append( + aws.ec2.Subnet( + subnet_resname, + availability_zone=az, + cidr_block=cidr, + tags=subnet_tags, + vpc_id=self.resources['vpc'].id, + opts=pulumi.ResourceOptions(parent=self, depends_on=[self.resources['vpc']]), + ) + ) + + # Associate the VPC's default route table to all of the subnets + self.resources['route_table_subnet_associations'] = [] + for idx, subnet in enumerate(self.resources['subnets']): + self.resources['route_table_subnet_associations'].append( + aws.ec2.RouteTableAssociation( + f'{name}-subnetassoc-{idx}', + route_table_id=self.resources['vpc'].default_route_table_id, + subnet_id=subnet.id, + ) + ) + + # Allow traffic in from the internet + if enable_internet_gateway: + ig_tags = {'Name': name} + ig_tags.update(self.tags) + self.resources['internet_gateway'] = aws.ec2.InternetGateway( + f'{name}-ig', + vpc_id=self.resources['vpc'].id, + tags=ig_tags, + opts=pulumi.ResourceOptions(parent=self, depends_on=self.resources['vpc']), + ) + if egress_via_internet_gateway: + self.resources['subnet_ig_route'] = aws.ec2.Route( + f'{name}-igroute', + route_table_id=self.resources['vpc'].default_route_table_id, + destination_cidr_block='0.0.0.0/0', + gateway_id=self.resources['internet_gateway'].id, + opts=pulumi.ResourceOptions( + parent=self, depends_on=[self.resources['vpc'], self.resources['internet_gateway']] + ), + ) + + if enable_nat_gateway: + self.resources['nat_eip'] = aws.ec2.Eip( + f'{name}-eip', + domain='vpc', + public_ipv4_pool='amazon', + network_border_group=self.project.aws_region, + opts=pulumi.ResourceOptions(parent=self, depends_on=self.resources['vpc']), + ) + ng_tags = {'Name': name} + ng_tags.update(self.tags) + self.resources['nat_gateway'] = aws.ec2.NatGateway( + f'{name}-nat', + allocation_id=self.resources['nat_eip'].allocation_id, + subnet_id=self.resources['subnets'][0].id, + tags=ng_tags, + opts=pulumi.ResourceOptions(parent=self, depends_on=self.resources['nat_eip']), + ) + if egress_via_nat_gateway: + self.resources['subnet_ng_route'] = aws.ec2.Route( + f'{name}-ngroute', + route_table_id=self.resources['vpc'].default_route_table_id, + destination_cidr_block='0.0.0.0/0', + gateway_id=self.resources['nat_gateway'].id, + opts=pulumi.ResourceOptions( + parent=self, depends_on=[self.resources['vpc'], self.resources['nat_gateway']] + ), + ) + + + # If we have to build endpoints, we have to have a security group to let local traffic in + if len(endpoint_interfaces + endpoint_gateways) > 0: + self.resources['endpoint_sg'] = tb_pulumi.network.SecurityGroupWithRules( + f'{name}-endpoint-sg', + project, + vpc_id=self.resources['vpc'].id, + rules={ + 'ingress': [ + { + 'cidr_blocks': [cidr_block], + 'description': 'Allow VPC access to endpoint-fronted AWS services', + 'protocol': 'TCP', + 'from_port': 443, + 'to_port': 443, + } + ], + 'egress': [ + { + 'cidr_blocks': ['0.0.0.0/0'], + 'description': 'Allow all TCP egress', + 'protocol': 'TCP', + 'from_port': 0, + 'to_port': 65535, + } + ], + }, + opts=pulumi.ResourceOptions(parent=self), + tags=self.tags, + ).resources + + self.resources['interfaces'] = [] + for svc in endpoint_interfaces: + self.resources['interfaces'].append( + aws.ec2.VpcEndpoint( + f'{name}-interface-{svc}', + private_dns_enabled=True, + service_name=f'com.amazonaws.{self.project.aws_region}.{svc}', + security_group_ids=[self.resources['endpoint_sg']['sg'].id], + subnet_ids=[subnet.id for subnet in self.resources['subnets']], + vpc_endpoint_type='Interface', + vpc_id=self.resources['vpc'].id, + tags=self.tags, + opts=pulumi.ResourceOptions( + parent=self, depends_on=[*self.resources['subnets'], self.resources['endpoint_sg']['sg']] + ), + ) + ) + + self.resources['gateways'] = [] + for svc in endpoint_gateways: + self.resources['gateways'].append( + aws.ec2.VpcEndpoint( + f'{name}-gateway-{svc}', + route_table_ids=[self.resources['vpc'].default_route_table_id], + service_name=f'com.amazonaws.{self.project.aws_region}.{svc}', + vpc_endpoint_type='Gateway', + vpc_id=self.resources['vpc'].id, + tags=self.tags, + opts=pulumi.ResourceOptions( + parent=self, depends_on=[*self.resources['subnets'], self.resources['endpoint_sg']['sg']] + ), + ) + ) + + self.finish() + + +class SecurityGroupWithRules(tb_pulumi.ThunderbirdComponentResource): + """Builds a security group and sets rules for it.""" + + def __init__( + self, + name: str, + project: tb_pulumi.ThunderbirdPulumiProject, + rules: dict = {}, + vpc_id: str = None, + opts: pulumi.ResourceOptions = None, + **kwargs, + ): + """Construct a SecurityGroupWithRules resource. + + Positional arguments: + - name: A string identifying this set of resources. + - project: The ThunderbirdPulumiProject to add these resources to. + + Keyword arguments: + - rules: A dict describing in/egress rules of the following construction: + + { + 'ingress': [{ + # Valid inputs to the SecurityGroupRule resource go here. Ref: + # https://www.pulumi.com/registry/packages/aws/api-docs/ec2/securitygrouprule/#inputs + }], + 'egress': [{ + # The same inputs are valid here + }] + } + + - vpc_id: ID of the VPC this security group should belong to. When not set, defaults to + the region's default VPC. + - opts: Additional pulumi.ResourceOptions to apply to these resources. + - kwargs: Any other keyword arguments which will be passed as inputs to the + ThunderbirdComponentResource superconstructor. + """ + + super().__init__('tb:network:SecurityGroupWithRules', name, project, opts=opts, **kwargs) + + # Build a security group in the provided VPC + self.resources['sg'] = aws.ec2.SecurityGroup( + f'{name}-sg', + opts=pulumi.ResourceOptions(parent=self), + name=name, + description=f'Send Suite backend security group ({self.project.stack})', + vpc_id=vpc_id, + tags=self.tags, + ) + + # Set up security group rules for that SG + self.resources['ingress_rules'] = [] + self.resources['egress_rules'] = [] + + ingress_ruledefs = rules['ingress'] + for rule in ingress_ruledefs: + rule.update({'type': 'ingress', 'security_group_id': self.resources['sg'].id}) + self.resources['ingress_rules'].append( + aws.ec2.SecurityGroupRule( + f'{name}-ingress-{rule['to_port']}', + opts=pulumi.ResourceOptions(parent=self, depends_on=[self.resources['sg']]), + **rule, + ) + ) + + egress_ruledefs = rules['egress'] + for rule in egress_ruledefs: + rule.update({'type': 'egress', 'security_group_id': self.resources['sg'].id}) + self.resources['egress_rules'].append( + aws.ec2.SecurityGroupRule( + f'{name}-egress-{rule['to_port']}', + opts=pulumi.ResourceOptions(parent=self, depends_on=[self.resources['sg']]), + **rule, + ) + ) + + self.finish() diff --git a/tb_pulumi/rds.py b/tb_pulumi/rds.py new file mode 100644 index 0000000..17ba5db --- /dev/null +++ b/tb_pulumi/rds.py @@ -0,0 +1,369 @@ +import pulumi +import pulumi_aws as aws +import pulumi_random +import socket +import tb_pulumi +import tb_pulumi.ec2 +import tb_pulumi.network + +from tb_pulumi.constants import SERVICE_PORTS + + +class RdsDatabaseGroup(tb_pulumi.ThunderbirdComponentResource): + """Builds a group of RDS databases. Note that this does not build a "proper" cluster, but a + series of manually operated RDS instances with replication. + """ + + def __init__( + self, + name: str, + project: tb_pulumi.ThunderbirdPulumiProject, + db_name: str, + subnets: list[pulumi.Output], + vpc_cidr: str, + vpc_id: str, + allocated_storage: int = 20, + auto_minor_version_upgrade: bool = True, + apply_immediately: bool = False, + backup_retention_period: int = 7, + blue_green_update: bool = False, + build_jumphost: bool = False, + db_username: str = 'root', + enabled_cluster_cloudwatch_logs_exports: list[str] = [], + enabled_instance_cloudwatch_logs_exports: list[str] = [], + engine: str = 'postgres', + engine_version: str = '15.7', + instance_class: str = 'db.t3.micro', + internal: bool = True, + jumphost_public_key: str = None, + jumphost_source_cidrs: list[str] = ['0.0.0.0/0'], + jumphost_user_data: str = None, + max_allocated_storage: int = 0, + num_instances: int = 1, + override_special='!#$%&*()-_=+[]{}<>:?', + parameters: list[dict] = None, + parameter_group_family: str = 'postgres15', + performance_insights_enabled: bool = False, + port: int = None, + sg_cidrs: list[str] = None, + skip_final_snapshot: bool = False, + storage_type: str = 'gp3', + opts: pulumi.ResourceOptions = None, + **kwargs, + ): + """Construct an RdsDatabaseGroup, which builds a primary database and zero or more read + replicas. An NLB is created to spread load across the read replicas. + + Positional arguments: + - name: A string identifying this set of resources. + - project: The ThunderbirdPulumiProject to add these resources to. + - db_name: What to call the name of the database at the schema level. + - subnets: List of subnet Output objects defining the network space to build in. + - vpc_cidr: An IP range to allow incoming traffic from, which is a subset of the IP + range allowed by the VPC in which this cluster is built. If you do not specify + `sg_cidrs`, but `internal` is True, then ingress traffic will be limited to being + sourced in this CIDR. + - vpc_id: The ID of the VPC to build in. + + Keyword arguments: + - allocated_storage: GB of storage to allot to each instance. AWS may impose different + minimum values for this option depending on other storage options. Details are here: + https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Storage.html + - auto_minor_version_upgrade: Allow RDS to upgrade the engine as long as it's only a + minor version change, and therefore backward compatible. + - apply_immediately: When True, changes to the DB config will be applied right away + instead of during the next maintenance window. Depending on the change, this could + cause downtime. + - backup_retention_period: Number of days to keep old backups. + - blue_green_update: When RDS applies updates, it will deploy a new cluster and fail + over to it. Ref: + https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/blue-green-deployments.html + - build_jumphost: When True, an EC2 instance in the same network space but with a public + IP address will be built, allowing access to a database that's only internally + accessible. + - db_username: The username to use for the root-level administrative user in the + database. Defaults to 'root'. + - enabled_cluster_cloudwatch_logs_exports: Any combination of valid log types for a DB + instance to export. These include: audit, error, general, slowquery, postgresql + - enabled_instance_cloudwatch_logs_exports: Any combination of valid log types for a DB + cluster to export. For details, see the "EnableCloudwatchLogsExports" section of + these docs: + https://docs.aws.amazon.com/AmazonRDS/latest/APIReference/API_CreateDBInstance.html + - engine: The core database engine to use, such as "postgres" or "mysql". + - engine_version: The version of the engine to use. This is a specific string that AWS + recognizes. You can see a list of those strings by running this command: + `aws rds describe-db-engine-versions` + - instance_class: One of the database sizes listed here: + https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.DBInstanceClass.html + - internal: When True, if no sg_cidrs are set, allows ingress only from what `vpc_cidr` + is set to. If False and no sg_cidrs are set, allows ingress from anywhere. + - jumphost_public_key: The public key you want to use when authenticating against the + jumphost's SSH service. + - jumphost_source_cidrs: List of CIDRs to allow SSH ingress to the jump host from. + - jumphost_user_data: Plaintext value (not base64-encoded) of the user data to pass the + jumphost. Use this to launch the server with your database client of choice pre- + installed, for example. + - max_allocated_storage: Gigabytes of storage which storage autoscaling will refuse to + increase beyond. To disable autoscaling, set this to zero. + - num_instances: Number of database servers to build. This must be at least 1. This + module interprets this number to mean that we should build a primary instance and + (num_instances - 1) read replicas. All servers will be built from the same set of + options described here. + - override_special: The root password is generated using "special characters". Set this + value to a string containing only those special characters that you want included in + your otherwise random password. + - parameters: A list of dicts describing parameters to override from the defaults set by + the parameter_group_family. These dicts should describe one of these: + https://www.pulumi.com/registry/packages/aws/api-docs/rds/parametergroup/#parametergroupparameter + - parameter_group_family: A special string known to AWS describing the base set of DB + parameters to use. These parameters can be overridden with the `parameters` option. + You can get a list of options by running: + `aws rds describe-db-engine-versions \ + --query "DBEngineVersions[].DBParameterGroupFamily" + - performance_insights_enabled: Record more detailed monitoring metrics to CloudWatch. + Incurs additional costs. + - port: Specify a non-default listening port. + - sg_cidrs: A list of CIDRs from which ingress should be allowed. Also see `internal` + `vpc_cidr`. + - skip_final_snapshot: Allow deletion of an RDS instance without performing a final + backup. + - storage_type: Type of storage to provision. Defaults to `gp3` but could be set to + other values such as `io2`. For details, see: + https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Storage.html + - opts: Additional pulumi.ResourceOptions to apply to these resources. + - kwargs: Key/value pairs describing additional arguments to be passed into *all* RDS + Instance declarations. Detail can be found here: + https://www.pulumi.com/registry/packages/aws/api-docs/rds/instance/#inputs + """ + + super().__init__('tb:rds:RdsDatabaseGroup', name, project, opts=opts) + + # Generate a random password + self.resources['password'] = pulumi_random.RandomPassword( + f'{name}-password', + length=29, + override_special=override_special, + special=True, + min_lower=1, + min_numeric=1, + min_special=1, + min_upper=1, + opts=pulumi.ResourceOptions(parent=self), + ) + + # Store the password in Secrets Manager + secret_fullname = f'{self.project.project}/{self.project.stack}/{name}/root_password' + self.resources['secret'] = aws.secretsmanager.Secret( + f'{name}-secret', opts=pulumi.ResourceOptions(parent=self), name=secret_fullname + ) + self.resources['secret_version'] = aws.secretsmanager.SecretVersion( + f'{name}-secretversion', + secret_id=self.resources['secret'].id, + secret_string=self.resources['password'].result, + opts=pulumi.ResourceOptions(parent=self, depends_on=self.resources['password']), + ) + + # If no ingress CIDRs have been defined, find a reasonable default + if sg_cidrs is None: + cidrs = [vpc_cidr] if internal else ['0.0.0.0/0'] + else: + cidrs = sg_cidrs + + # If no port has been specified, try to look it up by the engine name + if port is None: + port = SERVICE_PORTS.get(engine, None) + if port is None: + raise ValueError('Cannot determine the correct port to open') + + # Build a security group allowing the specified access + self.resources['security_group_with_rules'] = tb_pulumi.network.SecurityGroupWithRules( + f'{name}-sg', + project, + vpc_id=vpc_id, + rules={ + 'ingress': [ + { + 'cidr_blocks': cidrs, + 'description': 'Database access', + 'protocol': 'tcp', + 'from_port': port, + 'to_port': port, + } + ], + 'egress': {}, + }, + opts=pulumi.ResourceOptions(parent=self), + ).resources + + # Build a subnet group to launch instances in + self.resources['subnet_group'] = aws.rds.SubnetGroup( + f'{name}-subnetgroup', + name=name, + subnet_ids=[subnet.id for subnet in subnets], + tags=self.tags, + opts=pulumi.ResourceOptions(parent=self), + ) + + # Build a parameter group + self.resources['parameter_group'] = aws.rds.ParameterGroup( + f'{name}-parametergroup', + name=name, + opts=pulumi.ResourceOptions(parent=self), + family=parameter_group_family, + parameters=parameters, + ) + + # Build a KMS Key + self.resources['key'] = aws.kms.Key( + f'{name}-storage', + opts=pulumi.ResourceOptions(parent=self), + description=f'Key to encrypt database storage for {name}', + deletion_window_in_days=7, + tags=self.tags, + ) + + # Build the primary instance + instance_id = f'{self.project.name_prefix}-000' + instance_tags = {'instanceId': instance_id} + instance_tags.update(self.tags) + primary = aws.rds.Instance( + f'{name}-instance-{000}', + allocated_storage=allocated_storage, + allow_major_version_upgrade=False, + auto_minor_version_upgrade=auto_minor_version_upgrade, + backup_retention_period=backup_retention_period, + blue_green_update={'enabled': blue_green_update}, + copy_tags_to_snapshot=True, + db_name=db_name, + db_subnet_group_name=self.resources['subnet_group'].name, + enabled_cloudwatch_logs_exports=enabled_instance_cloudwatch_logs_exports, + engine=engine, + engine_version=engine_version, + identifier=instance_id, + instance_class=instance_class, + kms_key_id=self.resources['key'].arn, + max_allocated_storage=max_allocated_storage, + password=self.resources['password'].result, + parameter_group_name=self.resources['parameter_group'].name, + performance_insights_enabled=performance_insights_enabled, + performance_insights_kms_key_id=self.resources['key'].arn if performance_insights_enabled else None, + port=port, + publicly_accessible=False, + skip_final_snapshot=skip_final_snapshot, + storage_encrypted=True, + storage_type=storage_type, + username=db_username, + vpc_security_group_ids=[self.resources['security_group_with_rules']['sg'].id], + tags=instance_tags, + opts=pulumi.ResourceOptions( + parent=self, + depends_on=[ + self.resources['key'], + self.resources['parameter_group'], + self.resources['password'], + self.resources['subnet_group'], + ], + ), + **kwargs, + ) + + # Build replica instances in the cluster + self.resources['instances'] = [primary] + for idx in range(1, num_instances): # Start at 1, taking the primary into account + # Pad the index with zeroes to produce a 3-char string ID; set tags + idx_str = str(idx).rjust(3, '0') + instance_id = f'{tb_pulumi.PROJECT}-{tb_pulumi.STACK}-{idx_str}' + instance_tags = {'instanceId': instance_id} + instance_tags.update(self.tags) + + self.resources['instances'].append( + aws.rds.Instance( + f'{name}-instance-{idx_str}', + allow_major_version_upgrade=False, + auto_minor_version_upgrade=auto_minor_version_upgrade, + backup_retention_period=backup_retention_period, + blue_green_update={'enabled': blue_green_update}, + copy_tags_to_snapshot=True, + enabled_cloudwatch_logs_exports=enabled_instance_cloudwatch_logs_exports, + engine=engine, + engine_version=engine_version, + identifier=instance_id, + instance_class=instance_class, + kms_key_id=self.resources['key'].arn, + max_allocated_storage=max_allocated_storage, + parameter_group_name=self.resources['parameter_group'].name, + performance_insights_enabled=performance_insights_enabled, + performance_insights_kms_key_id=self.resources['key'].id if performance_insights_enabled else None, + port=port, + publicly_accessible=False, + replicate_source_db=primary.identifier, + skip_final_snapshot=skip_final_snapshot, + storage_encrypted=True, + storage_type=storage_type, + vpc_security_group_ids=[self.resources['security_group_with_rules']['sg'].id], + tags=instance_tags, + opts=pulumi.ResourceOptions(parent=self, depends_on=[primary]), + ), + **kwargs, + ) + + # Store some data as SSM params for later retrieval + self.resources['ssm_param_port'] = ( + self.ssm_param(f'{name}-ssm-port', f'/{self.project.project}/{self.project.stack}/db-port', port), + ) + self.resources['ssm_param_db_name'] = ( + self.ssm_param(f'{name}-ssm-dbname', f'/{self.project.project}/{self.project.stack}/db-name', db_name), + ) + self.resources['ssm_param_db_write_host'] = ( + self.ssm_param( + f'{name}-ssm-dbwritehost', + f'/{self.project.project}/{self.project.stack}/db-write-host', + primary.address, + ), + ) + + # Figure out the IPs once the instances are ready and build a load balancer targeting them + port = SERVICE_PORTS.get(engine, 5432) + inst_addrs = [instance.address for instance in self.resources['instances']] + pulumi.Output.all(*inst_addrs).apply( + lambda addresses: self.load_balancer(name, project, port, subnets, vpc_cidr, *addresses) + ) + + if build_jumphost: + self.resources['jumphost'] = tb_pulumi.ec2.SshableInstance( + f'{name}-jumphost', + project, + subnets[0].id, + kms_key_id=self.resources['key'].arn, + public_key=jumphost_public_key, + source_cidrs=jumphost_source_cidrs, + user_data=jumphost_user_data, + vpc_id=vpc_id, + opts=pulumi.ResourceOptions(parent=self, depends_on=[self.resources['key']]), + ).resources + + self.finish() + + def load_balancer(self, name, project, port, subnets, vpc_cidr, *addresses): + # Build a load balancer + self.resources['nlb'] = tb_pulumi.ec2.NetworkLoadBalancer( + f'{name}-nlb', + project, + port, + subnets, + port, + ingress_cidrs=[vpc_cidr], + internal=True, + ips=[socket.gethostbyname(addr) for addr in addresses], + security_group_description=f'Allow database traffic for {name}', + opts=pulumi.ResourceOptions(parent=self, depends_on=[*self.resources['instances']]), + ).resources + self.resources['ssm_param_read_host'] = self.ssm_param( + f'{name}-ssm-dbreadhost', + f'/{self.project.project}/{self.project.stack}/db-read-host', + self.resources['nlb']['nlb'].dns_name.apply(lambda dns_name: dns_name), + ) + + def ssm_param(self, name, param_name, value): + """Build an SSM Parameter.""" + return aws.ssm.Parameter(name, name=param_name, type=aws.ssm.ParameterType.STRING, value=value) diff --git a/tb_pulumi/secrets.py b/tb_pulumi/secrets.py new file mode 100644 index 0000000..51b344d --- /dev/null +++ b/tb_pulumi/secrets.py @@ -0,0 +1,130 @@ +import json +import pulumi +import pulumi_aws as aws +import tb_pulumi +import typing + + +class SecretsManagerSecret(tb_pulumi.ThunderbirdComponentResource): + """Stores a value as a Secrets Manager secret.""" + + def __init__( + self, + name: str, + project: tb_pulumi.ThunderbirdPulumiProject, + secret_name: str, + secret_value: typing.Any, + opts: pulumi.ResourceOptions = None, + **kwargs, + ): + """Construct a SecretsManagerSecret. + + Positional arguments: + - name: A string identifying this set of resources. + - project: The ThunderbirdPulumiProject to add these resources to. + + Keyword arguments: + - secret_name: A slash ("/") delimited name for the secret in AWS. The last segment of + this will be used as the "short name" for abbreviated references. + - secret_value: The secret data to store. This should be a string or some other type + that can be serialized with `str()`. + - opts: Additional pulumi.ResourceOptions to apply to these resources. + - kwargs: Any other keyword arguments which will be passed as inputs to the + ThunderbirdComponentResource superconstructor. + """ + + super().__init__('tb:secrets:SecretsManagerSecret', name, project, opts=opts, **kwargs) + + short_name = secret_name.split('/')[-1] + self.resources['secret'] = aws.secretsmanager.Secret( + f'{name}-secret-{short_name}', opts=pulumi.ResourceOptions(parent=self), name=secret_name + ) + + self.resources['version'] = aws.secretsmanager.SecretVersion( + f'{name}-secretversion-{short_name}', + secret_id=self.resources['secret'].id, + secret_string=secret_value, + opts=pulumi.ResourceOptions(parent=self, depends_on=[self.resources['secret']]), + ) + + self.finish() + + +class PulumiSecretsManager(tb_pulumi.ThunderbirdComponentResource): + """Builds a set of AWS SecretsManager Secrets based on specific secrets in Pulumi's config.""" + + def __init__( + self, + name: str, + project: tb_pulumi.ThunderbirdPulumiProject, + secret_names: list[str] = [], + opts: pulumi.ResourceOptions = None, + **kwargs, + ): + """Construct a PulumiSecretsManager resource. + + Positional arguments: + - name: A string identifying this set of resources. + - project: The ThunderbirdPulumiProject to add these resources to. + + Keyword arguments + - secret_names: A list of secrets as they are known to Pulumi. To get a list of valid + values, run `pulumi config | grep 'secret' | cut -d ' ' -f1`. For more info on + Pulumi secrets, see: https://www.pulumi.com/learn/building-with-pulumi/secrets/ + - opts: Additional pulumi.ResourceOptions to apply to these resources. + - kwargs: Any other keyword arguments which will be passed as inputs to the + ThunderbirdComponentResource superconstructor. + """ + + super().__init__('tb:secrets:PulumiSecretsManager', name, project, opts=opts, **kwargs) + self.resources['secrets'] = [] + self.resources['versions'] = [] + + # First build the secrets + for secret_name in secret_names: + # Pull the secret's value from Pulumi's encrypted state + secret_string = self.project.pulumi_config.require_secret(secret_name) + + # Declare a Secrets Manager Secret + secret_fullname = f'{self.project.project}/{self.project.stack}/{secret_name}' + secret = aws.secretsmanager.Secret( + f'{name}-secret-{secret_name}', opts=pulumi.ResourceOptions(parent=self), name=secret_fullname + ) + self.resources['secrets'].append(secret) + + # Populate its value + self.resources['versions'].append( + aws.secretsmanager.SecretVersion( + f'{name}-secretversion-{secret_name}', + opts=pulumi.ResourceOptions(parent=self), + secret_id=secret.id, + secret_string=secret_string, + ) + ) + + # Then create an IAM policy allowing access to them + secret_arns = [secret.arn for secret in self.resources['secrets']] + policy = pulumi.Output.all(*secret_arns).apply( + lambda secret_arns: json.dumps( + { + 'Version': '2012-10-17', + 'Statement': [ + { + 'Sid': 'AllowSecretsAccess', + 'Effect': 'Allow', + 'Action': 'secretsmanager:GetSecretValue', + 'Resource': [arn for arn in secret_arns], + } + ], + } + ) + ) + self.resources['policy'] = aws.iam.Policy( + f'{name}-policy', + opts=pulumi.ResourceOptions(parent=self), + name=name, + description=f'Allows access to secrets related to {name}', + policy=policy, + ) + + self.finish()