[feature] Add GitHub Webhooks archiver and S3 private bucket modules

README.md

@@ -92,6 +92,13 @@ This creates a security-audit role, based off the AWS-managed policy, but with a
 
 [Read More](aws-iam-role-security-audit/README.md)
 
+### GitHub Webhooks to S3
+
+Accept GitHub webhooks and store them in S3
+
+[Read More](github-webhooks-to-s3/README.md)
+
+
 ## Contributing
 
 ### Writing tests

github-webhooks-to-s3/Makefile

@@ -0,0 +1,7 @@
+# Auto-generated by fogg. Do not edit
+# Make improvements in fogg, so that everyone can benefit.
+
+export TERRAFORM_VERSION := 0.12.6
+export TF_PLUGIN_CACHE_DIR := ../../..//.terraform.d/plugin-cache
+
+include ../../..//scripts/module.mk

github-webhooks-to-s3/README.md

@@ -0,0 +1,23 @@
+<!-- START -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| certificate\_arn | A certificate in us-east-1 for var.fqdn | string | n/a | yes |
+| env | Env for tagging and naming. | string | n/a | yes |
+| fqdn | The fqdn to expose the api gateway as | string | n/a | yes |
+| iam\_path |  | string | `"/"` | no |
+| lambda\_source\_s3\_bucket | The s3 bucket where to find the lambda executable | string | `"shared-infra-prod-assets"` | no |
+| lambda\_source\_s3\_key | The s3 key where to find the lambda executable | string | `"go-misc/lambdas/2019/06/03/github_to_firehose.zip"` | no |
+| owner | Owner for tagging and naming. | string | n/a | yes |
+| project | Project for tagging and naming. | string | n/a | yes |
+| route53\_zone\_id | The route53 zone id for fqdn's domain | string | n/a | yes |
+| service | Service for tagging and naming. | string | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| fivetran\_role\_arn |  |
+
+<!-- END -->

github-webhooks-to-s3/api_gateway.tf

@@ -0,0 +1,64 @@
+// https://learn.hashicorp.com/terraform/aws/lambda-api-gateway
+resource "aws_api_gateway_rest_api" "github" {
+  name        = "${local.name}"
+  description = "Github webhook ingestion"
+}
+
+resource "aws_api_gateway_resource" "github" {
+  rest_api_id = "${aws_api_gateway_rest_api.github.id}"
+  parent_id   = "${aws_api_gateway_rest_api.github.root_resource_id}"
+  path_part   = "{proxy+}"
+}
+
+resource "aws_api_gateway_method" "github" {
+  rest_api_id   = "${aws_api_gateway_rest_api.github.id}"
+  resource_id   = "${aws_api_gateway_resource.github.id}"
+  http_method   = "POST"
+  authorization = "NONE"
+}
+
+resource "aws_api_gateway_integration" "lambda" {
+  rest_api_id = "${aws_api_gateway_rest_api.github.id}"
+  resource_id = "${aws_api_gateway_method.github.resource_id}"
+  http_method = "${aws_api_gateway_method.github.http_method}"
+
+  integration_http_method = "POST"
+  type                    = "AWS_PROXY"
+  uri                     = "${aws_lambda_function.lambda.invoke_arn}"
+}
+
+resource "aws_api_gateway_method" "github_root" {
+  rest_api_id   = "${aws_api_gateway_rest_api.github.id}"
+  resource_id   = "${aws_api_gateway_rest_api.github.root_resource_id}"
+  http_method   = "ANY"
+  authorization = "NONE"
+}
+
+resource "aws_api_gateway_integration" "lambda_root" {
+  rest_api_id = "${aws_api_gateway_rest_api.github.id}"
+  resource_id = "${aws_api_gateway_method.github_root.resource_id}"
+  http_method = "${aws_api_gateway_method.github_root.http_method}"
+
+  integration_http_method = "POST"
+  type                    = "AWS_PROXY"
+  uri                     = "${aws_lambda_function.lambda.invoke_arn}"
+}
+
+resource "aws_api_gateway_deployment" "github" {
+  depends_on = [
+    "aws_api_gateway_integration.lambda",
+    "aws_api_gateway_integration.lambda_root",
+  ]
+
+  rest_api_id = "${aws_api_gateway_rest_api.github.id}"
+  stage_name  = "${var.env}"
+}
+
+resource "aws_lambda_permission" "apigw" {
+  statement_id  = "AllowAPIGatewayInvoke"
+  action        = "lambda:InvokeFunction"
+  function_name = "${aws_lambda_function.lambda.arn}"
+  principal     = "apigateway.amazonaws.com"
+
+  source_arn = "${aws_api_gateway_deployment.github.execution_arn}/*/*"
+}

github-webhooks-to-s3/certs.tf

@@ -0,0 +1,34 @@
+resource "aws_api_gateway_domain_name" "github" {
+  certificate_arn = "${var.certificate_arn}"
+  domain_name     = "${var.fqdn}"
+}
+
+resource "aws_api_gateway_base_path_mapping" "github" {
+  api_id      = "${aws_api_gateway_rest_api.github.id}"
+  stage_name  = "${aws_api_gateway_deployment.github.stage_name}"
+  domain_name = "${aws_api_gateway_domain_name.github.domain_name}"
+}
+
+resource "aws_route53_record" "github" {
+  name    = "${aws_api_gateway_domain_name.github.domain_name}"
+  type    = "A"
+  zone_id = "${var.route53_zone_id}"
+
+  alias {
+    evaluate_target_health = true
+    name                   = "${aws_api_gateway_domain_name.github.cloudfront_domain_name}"
+    zone_id                = "${aws_api_gateway_domain_name.github.cloudfront_zone_id}"
+  }
+}
+
+resource "aws_route53_record" "github-ipv6" {
+  name    = "${aws_api_gateway_domain_name.github.domain_name}"
+  type    = "AAAA"
+  zone_id = "${var.route53_zone_id}"
+
+  alias {
+    evaluate_target_health = true
+    name                   = "${aws_api_gateway_domain_name.github.cloudfront_domain_name}"
+    zone_id                = "${aws_api_gateway_domain_name.github.cloudfront_zone_id}"
+  }
+}

github-webhooks-to-s3/firehose.tf

@@ -0,0 +1,102 @@
+module "bucket" {
+  source = "../aws-s3-private-bucket"
+
+  bucket_name   = "${local.name}"
+  bucket_policy = ""
+
+  project = "${var.project}"
+  env     = "${var.env}"
+  service = "${var.service}"
+  owner   = "${var.owner}"
+}
+
+data "aws_iam_policy_document" "firehose" {
+  statement {
+    sid     = "EnableFirehoseAssumeRole"
+    effect  = "Allow"
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["firehose.amazonaws.com"]
+    }
+  }
+}
+
+resource "aws_iam_role" "firehose" {
+  name = "${local.name}-firehose"
+
+  assume_role_policy = "${data.aws_iam_policy_document.firehose.json}"
+  tags               = "${local.tags}"
+}
+
+data "aws_iam_policy_document" "firehose-to-s3" {
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "s3:AbortMultipartUpload",
+      "s3:GetBucketLocation",
+      "s3:GetObject",
+      "s3:ListBucket",
+      "s3:ListBucketMultipartUploads",
+      "s3:PutObject",
+    ]
+
+    resources = [
+      "${module.bucket.arn}",
+      "${module.bucket.arn}/*",
+    ]
+  }
+
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "logs:CreateLogStream",
+      "logs:PutLogEvents",
+      "logs:GetLogEvents",
+    ]
+
+    resources = [
+      "${aws_cloudwatch_log_group.firehose.arn}",
+      "${aws_cloudwatch_log_group.firehose.arn}/*",
+    ]
+  }
+}
+
+resource "aws_iam_role_policy" "firehose-s3" {
+  name   = "firehose-s3"
+  role   = "${aws_iam_role.firehose.id}"
+  policy = "${data.aws_iam_policy_document.firehose-to-s3.json}"
+}
+
+resource "aws_kinesis_firehose_delivery_stream" "firehose" {
+  name        = "${local.name}"
+  destination = "s3"
+
+  s3_configuration {
+    role_arn           = "${aws_iam_role.firehose.arn}"
+    bucket_arn         = "${module.bucket.arn}"
+    prefix             = ""
+    compression_format = "GZIP"
+
+    cloudwatch_logging_options {
+      enabled         = true
+      log_group_name  = "${aws_cloudwatch_log_group.firehose.name}"
+      log_stream_name = "${aws_cloudwatch_log_stream.firehose.name}"
+    }
+  }
+
+  tags = "${local.tags}"
+}
+
+resource "aws_cloudwatch_log_group" "firehose" {
+  name = "${local.name}-firehose"
+  tags = "${local.tags}"
+}
+
+resource "aws_cloudwatch_log_stream" "firehose" {
+  name           = "status"
+  log_group_name = "${aws_cloudwatch_log_group.firehose.name}"
+}

github-webhooks-to-s3/fivetran.tf

@@ -0,0 +1,12 @@
+module "fivetran-role" {
+  source = "../fivetran_s3_role"
+
+  project = "${var.project}"
+  env     = "${var.env}"
+  service = "${var.service}"
+  owner   = "${var.owner}"
+
+  bucket_name = "${module.bucket.id}"
+
+  bucket_prefix = "/*"
+}

github-webhooks-to-s3/fogg.tf

@@ -0,0 +1,6 @@
+# Auto-generated by fogg. Do not edit
+# Make improvements in fogg, so that everyone can benefit.
+
+terraform {
+  required_version = "~>0.12.6"
+}