diff --git a/terraform/aws/iglu_server/default/README.md b/terraform/aws/iglu_server/default/README.md index e87a797..c9faaff 100644 --- a/terraform/aws/iglu_server/default/README.md +++ b/terraform/aws/iglu_server/default/README.md @@ -18,7 +18,7 @@ |------|--------|---------| | [iglu\_lb](#module\_iglu\_lb) | snowplow-devops/alb/aws | 0.2.0 | | [iglu\_rds](#module\_iglu\_rds) | snowplow-devops/rds/aws | 0.4.0 | -| [iglu\_server](#module\_iglu\_server) | snowplow-devops/iglu-server-ec2/aws | 0.5.0 | +| [iglu\_server](#module\_iglu\_server) | snowplow-devops/iglu-server-ec2/aws | 0.5.1 | ## Resources @@ -43,6 +43,7 @@ | [cloudwatch\_logs\_enabled](#input\_cloudwatch\_logs\_enabled) | Whether application logs should be reported to CloudWatch | `bool` | `true` | no | | [cloudwatch\_logs\_retention\_days](#input\_cloudwatch\_logs\_retention\_days) | The length of time in days to retain logs for | `number` | `7` | no | | [iam\_permissions\_boundary](#input\_iam\_permissions\_boundary) | The permissions boundary ARN to set on IAM roles created | `string` | `""` | no | +| [private\_ecr\_registry](#input\_private\_ecr\_registry) | The URL of an ECR registry that the sub-account has access to (e.g. '000000000000.dkr.ecr.cn-north-1.amazonaws.com.cn/') | `string` | `""` | no | | [ssl\_information](#input\_ssl\_information) | The ARN of an Amazon Certificate Manager certificate to bind to the load balancer |
object({|
enabled = bool
certificate_arn = string
})
{| no | | [tags](#input\_tags) | The tags to append to the resources in this module | `map(string)` | `{}` | no | | [telemetry\_enabled](#input\_telemetry\_enabled) | Whether or not to send telemetry information back to Snowplow Analytics Ltd | `bool` | `true` | no | diff --git a/terraform/aws/iglu_server/default/main.tf b/terraform/aws/iglu_server/default/main.tf index 57ae67c..73a5f36 100644 --- a/terraform/aws/iglu_server/default/main.tf +++ b/terraform/aws/iglu_server/default/main.tf @@ -38,7 +38,7 @@ module "iglu_lb" { module "iglu_server" { source = "snowplow-devops/iglu-server-ec2/aws" - version = "0.5.0" + version = "0.5.1" accept_limited_use_license = var.accept_limited_use_license @@ -70,4 +70,6 @@ module "iglu_server" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } diff --git a/terraform/aws/iglu_server/default/terraform.tfvars b/terraform/aws/iglu_server/default/terraform.tfvars index 36dc856..c5ede04 100644 --- a/terraform/aws/iglu_server/default/terraform.tfvars +++ b/terraform/aws/iglu_server/default/terraform.tfvars @@ -52,3 +52,6 @@ tags = {} # --- CloudWatch logging to ensure logs are saved outside of the server cloudwatch_logs_enabled = true cloudwatch_logs_retention_days = 7 + +# --- Image Repositories +private_ecr_registry = "" diff --git a/terraform/aws/iglu_server/default/variables.tf b/terraform/aws/iglu_server/default/variables.tf index 7c1fd81..4a5bda2 100644 --- a/terraform/aws/iglu_server/default/variables.tf +++ b/terraform/aws/iglu_server/default/variables.tf @@ -103,3 +103,9 @@ variable "cloudwatch_logs_retention_days" { default = 7 type = number } + +variable "private_ecr_registry" { + description = "The URL of an ECR registry that the sub-account has access to (e.g. '000000000000.dkr.ecr.cn-north-1.amazonaws.com.cn/')" + type = string + default = "" +} diff --git a/terraform/aws/iglu_server/secure/README.md b/terraform/aws/iglu_server/secure/README.md index c8bef9a..5539ec1 100644 --- a/terraform/aws/iglu_server/secure/README.md +++ b/terraform/aws/iglu_server/secure/README.md @@ -18,7 +18,7 @@ |------|--------|---------| | [iglu\_lb](#module\_iglu\_lb) | snowplow-devops/alb/aws | 0.2.0 | | [iglu\_rds](#module\_iglu\_rds) | snowplow-devops/rds/aws | 0.4.0 | -| [iglu\_server](#module\_iglu\_server) | snowplow-devops/iglu-server-ec2/aws | 0.5.0 | +| [iglu\_server](#module\_iglu\_server) | snowplow-devops/iglu-server-ec2/aws | 0.5.1 | ## Resources @@ -44,6 +44,7 @@ | [cloudwatch\_logs\_enabled](#input\_cloudwatch\_logs\_enabled) | Whether application logs should be reported to CloudWatch | `bool` | `true` | no | | [cloudwatch\_logs\_retention\_days](#input\_cloudwatch\_logs\_retention\_days) | The length of time in days to retain logs for | `number` | `7` | no | | [iam\_permissions\_boundary](#input\_iam\_permissions\_boundary) | The permissions boundary ARN to set on IAM roles created | `string` | `""` | no | +| [private\_ecr\_registry](#input\_private\_ecr\_registry) | The URL of an ECR registry that the sub-account has access to (e.g. '000000000000.dkr.ecr.cn-north-1.amazonaws.com.cn/') | `string` | `""` | no | | [ssl\_information](#input\_ssl\_information) | The ARN of an Amazon Certificate Manager certificate to bind to the load balancer |
"certificate_arn": "",
"enabled": false
}
object({|
enabled = bool
certificate_arn = string
})
{| no | | [tags](#input\_tags) | The tags to append to the resources in this module | `map(string)` | `{}` | no | | [telemetry\_enabled](#input\_telemetry\_enabled) | Whether or not to send telemetry information back to Snowplow Analytics Ltd | `bool` | `true` | no | diff --git a/terraform/aws/iglu_server/secure/main.tf b/terraform/aws/iglu_server/secure/main.tf index b16c52d..8ee1380 100644 --- a/terraform/aws/iglu_server/secure/main.tf +++ b/terraform/aws/iglu_server/secure/main.tf @@ -38,7 +38,7 @@ module "iglu_lb" { module "iglu_server" { source = "snowplow-devops/iglu-server-ec2/aws" - version = "0.5.0" + version = "0.5.1" accept_limited_use_license = var.accept_limited_use_license @@ -72,4 +72,6 @@ module "iglu_server" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } diff --git a/terraform/aws/iglu_server/secure/terraform.tfvars b/terraform/aws/iglu_server/secure/terraform.tfvars index b0273e2..d512889 100644 --- a/terraform/aws/iglu_server/secure/terraform.tfvars +++ b/terraform/aws/iglu_server/secure/terraform.tfvars @@ -55,3 +55,6 @@ tags = {} # --- CloudWatch logging to ensure logs are saved outside of the server cloudwatch_logs_enabled = true cloudwatch_logs_retention_days = 7 + +# --- Image Repositories +private_ecr_registry = "" diff --git a/terraform/aws/iglu_server/secure/variables.tf b/terraform/aws/iglu_server/secure/variables.tf index fa39b76..0affce8 100644 --- a/terraform/aws/iglu_server/secure/variables.tf +++ b/terraform/aws/iglu_server/secure/variables.tf @@ -108,3 +108,9 @@ variable "cloudwatch_logs_retention_days" { default = 7 type = number } + +variable "private_ecr_registry" { + description = "The URL of an ECR registry that the sub-account has access to (e.g. '000000000000.dkr.ecr.cn-north-1.amazonaws.com.cn/')" + type = string + default = "" +} diff --git a/terraform/aws/pipeline/default/README.md b/terraform/aws/pipeline/default/README.md index 95d2148..1b63f60 100644 --- a/terraform/aws/pipeline/default/README.md +++ b/terraform/aws/pipeline/default/README.md @@ -18,25 +18,25 @@ |------|--------|---------| | [bad\_1\_stream](#module\_bad\_1\_stream) | snowplow-devops/kinesis-stream/aws | 0.3.0 | | [bad\_2\_stream](#module\_bad\_2\_stream) | snowplow-devops/kinesis-stream/aws | 0.3.0 | -| [collector\_kinesis](#module\_collector\_kinesis) | snowplow-devops/collector-kinesis-ec2/aws | 0.9.0 | +| [collector\_kinesis](#module\_collector\_kinesis) | snowplow-devops/collector-kinesis-ec2/aws | 0.9.1 | | [collector\_lb](#module\_collector\_lb) | snowplow-devops/alb/aws | 0.2.0 | -| [db\_loader](#module\_db\_loader) | snowplow-devops/databricks-loader-ec2/aws | 0.2.0 | -| [db\_transformer\_wrp](#module\_db\_transformer\_wrp) | snowplow-devops/transformer-kinesis-ec2/aws | 0.4.0 | -| [enrich\_kinesis](#module\_enrich\_kinesis) | snowplow-devops/enrich-kinesis-ec2/aws | 0.6.0 | +| [db\_loader](#module\_db\_loader) | snowplow-devops/databricks-loader-ec2/aws | 0.2.1 | +| [db\_transformer\_wrp](#module\_db\_transformer\_wrp) | snowplow-devops/transformer-kinesis-ec2/aws | 0.4.1 | +| [enrich\_kinesis](#module\_enrich\_kinesis) | snowplow-devops/enrich-kinesis-ec2/aws | 0.6.1 | | [enriched\_stream](#module\_enriched\_stream) | snowplow-devops/kinesis-stream/aws | 0.3.0 | -| [postgres\_loader\_bad](#module\_postgres\_loader\_bad) | snowplow-devops/postgres-loader-kinesis-ec2/aws | 0.5.0 | -| [postgres\_loader\_enriched](#module\_postgres\_loader\_enriched) | snowplow-devops/postgres-loader-kinesis-ec2/aws | 0.5.0 | +| [postgres\_loader\_bad](#module\_postgres\_loader\_bad) | snowplow-devops/postgres-loader-kinesis-ec2/aws | 0.5.1 | +| [postgres\_loader\_enriched](#module\_postgres\_loader\_enriched) | snowplow-devops/postgres-loader-kinesis-ec2/aws | 0.5.1 | | [postgres\_loader\_rds](#module\_postgres\_loader\_rds) | snowplow-devops/rds/aws | 0.4.0 | | [raw\_stream](#module\_raw\_stream) | snowplow-devops/kinesis-stream/aws | 0.3.0 | -| [rs\_loader](#module\_rs\_loader) | snowplow-devops/redshift-loader-ec2/aws | 0.2.0 | -| [rs\_transformer\_stsv](#module\_rs\_transformer\_stsv) | snowplow-devops/transformer-kinesis-ec2/aws | 0.4.0 | -| [s3\_loader\_bad](#module\_s3\_loader\_bad) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.5.0 | -| [s3\_loader\_enriched](#module\_s3\_loader\_enriched) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.5.0 | -| [s3\_loader\_raw](#module\_s3\_loader\_raw) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.5.0 | +| [rs\_loader](#module\_rs\_loader) | snowplow-devops/redshift-loader-ec2/aws | 0.2.1 | +| [rs\_transformer\_stsv](#module\_rs\_transformer\_stsv) | snowplow-devops/transformer-kinesis-ec2/aws | 0.4.1 | +| [s3\_loader\_bad](#module\_s3\_loader\_bad) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.5.1 | +| [s3\_loader\_enriched](#module\_s3\_loader\_enriched) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.5.1 | +| [s3\_loader\_raw](#module\_s3\_loader\_raw) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.5.1 | | [s3\_pipeline\_bucket](#module\_s3\_pipeline\_bucket) | snowplow-devops/s3-bucket/aws | 0.2.0 | -| [sf\_loader](#module\_sf\_loader) | snowplow-devops/snowflake-loader-ec2/aws | 0.3.0 | -| [sf\_transformer\_wrj](#module\_sf\_transformer\_wrj) | snowplow-devops/transformer-kinesis-ec2/aws | 0.4.0 | -| [snowflake\_streaming\_loader\_enriched](#module\_snowflake\_streaming\_loader\_enriched) | snowplow-devops/snowflake-streaming-loader-ec2/aws | 0.1.0 | +| [sf\_loader](#module\_sf\_loader) | snowplow-devops/snowflake-loader-ec2/aws | 0.3.1 | +| [sf\_transformer\_wrj](#module\_sf\_transformer\_wrj) | snowplow-devops/transformer-kinesis-ec2/aws | 0.4.1 | +| [snowflake\_streaming\_loader\_enriched](#module\_snowflake\_streaming\_loader\_enriched) | snowplow-devops/snowflake-streaming-loader-ec2/aws | 0.1.2 | ## Resources @@ -79,6 +79,7 @@ | [postgres\_db\_password](#input\_postgres\_db\_password) | The password to use to connect to the database | `string` | `""` | no | | [postgres\_db\_publicly\_accessible](#input\_postgres\_db\_publicly\_accessible) | Whether to make the Postgres RDS instance accessible over the internet | `bool` | `false` | no | | [postgres\_db\_username](#input\_postgres\_db\_username) | The username to use to connect to the database | `string` | `""` | no | +| [private\_ecr\_registry](#input\_private\_ecr\_registry) | The URL of an ECR registry that the sub-account has access to (e.g. '000000000000.dkr.ecr.cn-north-1.amazonaws.com.cn/') | `string` | `""` | no | | [redshift\_database](#input\_redshift\_database) | Redshift database name | `string` | `""` | no | | [redshift\_enabled](#input\_redshift\_enabled) | Whether to enable loading into a Redshift Database | `bool` | `false` | no | | [redshift\_host](#input\_redshift\_host) | Redshift cluster hostname | `string` | `""` | no | diff --git a/terraform/aws/pipeline/default/main.tf b/terraform/aws/pipeline/default/main.tf index 6929c3f..5b87ccd 100644 --- a/terraform/aws/pipeline/default/main.tf +++ b/terraform/aws/pipeline/default/main.tf @@ -86,7 +86,7 @@ module "collector_lb" { module "collector_kinesis" { source = "snowplow-devops/collector-kinesis-ec2/aws" - version = "0.9.0" + version = "0.9.1" accept_limited_use_license = var.accept_limited_use_license @@ -113,12 +113,14 @@ module "collector_kinesis" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } # 3. Deploy Enrichment module "enrich_kinesis" { source = "snowplow-devops/enrich-kinesis-ec2/aws" - version = "0.6.0" + version = "0.6.1" accept_limited_use_license = var.accept_limited_use_license @@ -148,4 +150,6 @@ module "enrich_kinesis" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } diff --git a/terraform/aws/pipeline/default/target_amazon_s3.tf b/terraform/aws/pipeline/default/target_amazon_s3.tf index e6dae32..9a8894e 100644 --- a/terraform/aws/pipeline/default/target_amazon_s3.tf +++ b/terraform/aws/pipeline/default/target_amazon_s3.tf @@ -1,6 +1,6 @@ module "s3_loader_raw" { source = "snowplow-devops/s3-loader-kinesis-ec2/aws" - version = "0.5.0" + version = "0.5.1" accept_limited_use_license = var.accept_limited_use_license @@ -28,11 +28,13 @@ module "s3_loader_raw" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } module "s3_loader_bad" { source = "snowplow-devops/s3-loader-kinesis-ec2/aws" - version = "0.5.0" + version = "0.5.1" accept_limited_use_license = var.accept_limited_use_license @@ -63,11 +65,13 @@ module "s3_loader_bad" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } module "s3_loader_enriched" { source = "snowplow-devops/s3-loader-kinesis-ec2/aws" - version = "0.5.0" + version = "0.5.1" accept_limited_use_license = var.accept_limited_use_license @@ -97,4 +101,6 @@ module "s3_loader_enriched" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } diff --git a/terraform/aws/pipeline/default/target_databricks.tf b/terraform/aws/pipeline/default/target_databricks.tf index a92b8c8..539aa41 100644 --- a/terraform/aws/pipeline/default/target_databricks.tf +++ b/terraform/aws/pipeline/default/target_databricks.tf @@ -9,7 +9,7 @@ resource "aws_sqs_queue" "db_message_queue" { module "db_transformer_wrp" { source = "snowplow-devops/transformer-kinesis-ec2/aws" - version = "0.4.0" + version = "0.4.1" accept_limited_use_license = var.accept_limited_use_license @@ -44,11 +44,13 @@ module "db_transformer_wrp" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } module "db_loader" { source = "snowplow-devops/databricks-loader-ec2/aws" - version = "0.2.0" + version = "0.2.1" accept_limited_use_license = var.accept_limited_use_license @@ -82,4 +84,6 @@ module "db_loader" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } diff --git a/terraform/aws/pipeline/default/target_postgres.tf b/terraform/aws/pipeline/default/target_postgres.tf index 7643831..7238544 100644 --- a/terraform/aws/pipeline/default/target_postgres.tf +++ b/terraform/aws/pipeline/default/target_postgres.tf @@ -21,7 +21,7 @@ module "postgres_loader_rds" { module "postgres_loader_enriched" { source = "snowplow-devops/postgres-loader-kinesis-ec2/aws" - version = "0.5.0" + version = "0.5.1" accept_limited_use_license = var.accept_limited_use_license @@ -59,11 +59,13 @@ module "postgres_loader_enriched" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } module "postgres_loader_bad" { source = "snowplow-devops/postgres-loader-kinesis-ec2/aws" - version = "0.5.0" + version = "0.5.1" accept_limited_use_license = var.accept_limited_use_license @@ -101,4 +103,6 @@ module "postgres_loader_bad" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } diff --git a/terraform/aws/pipeline/default/target_redshift.tf b/terraform/aws/pipeline/default/target_redshift.tf index f9944bc..2bc1b61 100644 --- a/terraform/aws/pipeline/default/target_redshift.tf +++ b/terraform/aws/pipeline/default/target_redshift.tf @@ -9,7 +9,7 @@ resource "aws_sqs_queue" "rs_message_queue" { module "rs_transformer_stsv" { source = "snowplow-devops/transformer-kinesis-ec2/aws" - version = "0.4.0" + version = "0.4.1" accept_limited_use_license = var.accept_limited_use_license @@ -44,11 +44,13 @@ module "rs_transformer_stsv" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } module "rs_loader" { source = "snowplow-devops/redshift-loader-ec2/aws" - version = "0.2.0" + version = "0.2.1" accept_limited_use_license = var.accept_limited_use_license @@ -82,4 +84,6 @@ module "rs_loader" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } diff --git a/terraform/aws/pipeline/default/target_snowflake.tf b/terraform/aws/pipeline/default/target_snowflake.tf index e7acce4..2e61ca3 100644 --- a/terraform/aws/pipeline/default/target_snowflake.tf +++ b/terraform/aws/pipeline/default/target_snowflake.tf @@ -9,7 +9,7 @@ resource "aws_sqs_queue" "sf_message_queue" { module "sf_transformer_wrj" { source = "snowplow-devops/transformer-kinesis-ec2/aws" - version = "0.4.0" + version = "0.4.1" accept_limited_use_license = var.accept_limited_use_license @@ -44,11 +44,13 @@ module "sf_transformer_wrj" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } module "sf_loader" { source = "snowplow-devops/snowflake-loader-ec2/aws" - version = "0.3.0" + version = "0.3.1" accept_limited_use_license = var.accept_limited_use_license @@ -83,4 +85,6 @@ module "sf_loader" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } diff --git a/terraform/aws/pipeline/default/target_snowflake_streaming.tf b/terraform/aws/pipeline/default/target_snowflake_streaming.tf index 26f677d..55f04dd 100644 --- a/terraform/aws/pipeline/default/target_snowflake_streaming.tf +++ b/terraform/aws/pipeline/default/target_snowflake_streaming.tf @@ -1,6 +1,6 @@ module "snowflake_streaming_loader_enriched" { source = "snowplow-devops/snowflake-streaming-loader-ec2/aws" - version = "0.1.0" + version = "0.1.2" accept_limited_use_license = var.accept_limited_use_license @@ -33,4 +33,6 @@ module "snowflake_streaming_loader_enriched" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } diff --git a/terraform/aws/pipeline/default/terraform.tfvars b/terraform/aws/pipeline/default/terraform.tfvars index 7f5506b..ff8448b 100644 --- a/terraform/aws/pipeline/default/terraform.tfvars +++ b/terraform/aws/pipeline/default/terraform.tfvars @@ -129,3 +129,6 @@ cloudwatch_logs_retention_days = 7 # Extra Tags to append to created resources (optional) tags = {} + +# Image Repositories +private_ecr_registry = "" diff --git a/terraform/aws/pipeline/default/variables.tf b/terraform/aws/pipeline/default/variables.tf index 1f5e675..b6055f9 100644 --- a/terraform/aws/pipeline/default/variables.tf +++ b/terraform/aws/pipeline/default/variables.tf @@ -121,6 +121,12 @@ variable "cloudwatch_logs_retention_days" { type = number } +variable "private_ecr_registry" { + description = "The URL of an ECR registry that the sub-account has access to (e.g. '000000000000.dkr.ecr.cn-north-1.amazonaws.com.cn/')" + type = string + default = "" +} + # --- Target: Amazon S3 variable "s3_raw_enabled" { diff --git a/terraform/aws/pipeline/secure/README.md b/terraform/aws/pipeline/secure/README.md index deaa851..07d219e 100644 --- a/terraform/aws/pipeline/secure/README.md +++ b/terraform/aws/pipeline/secure/README.md @@ -18,25 +18,25 @@ |------|--------|---------| | [bad\_1\_stream](#module\_bad\_1\_stream) | snowplow-devops/kinesis-stream/aws | 0.3.0 | | [bad\_2\_stream](#module\_bad\_2\_stream) | snowplow-devops/kinesis-stream/aws | 0.3.0 | -| [collector\_kinesis](#module\_collector\_kinesis) | snowplow-devops/collector-kinesis-ec2/aws | 0.9.0 | +| [collector\_kinesis](#module\_collector\_kinesis) | snowplow-devops/collector-kinesis-ec2/aws | 0.9.1 | | [collector\_lb](#module\_collector\_lb) | snowplow-devops/alb/aws | 0.2.0 | -| [db\_loader](#module\_db\_loader) | snowplow-devops/databricks-loader-ec2/aws | 0.2.0 | -| [db\_transformer\_wrp](#module\_db\_transformer\_wrp) | snowplow-devops/transformer-kinesis-ec2/aws | 0.4.0 | -| [enrich\_kinesis](#module\_enrich\_kinesis) | snowplow-devops/enrich-kinesis-ec2/aws | 0.6.0 | +| [db\_loader](#module\_db\_loader) | snowplow-devops/databricks-loader-ec2/aws | 0.2.1 | +| [db\_transformer\_wrp](#module\_db\_transformer\_wrp) | snowplow-devops/transformer-kinesis-ec2/aws | 0.4.1 | +| [enrich\_kinesis](#module\_enrich\_kinesis) | snowplow-devops/enrich-kinesis-ec2/aws | 0.6.1 | | [enriched\_stream](#module\_enriched\_stream) | snowplow-devops/kinesis-stream/aws | 0.3.0 | -| [postgres\_loader\_bad](#module\_postgres\_loader\_bad) | snowplow-devops/postgres-loader-kinesis-ec2/aws | 0.5.0 | -| [postgres\_loader\_enriched](#module\_postgres\_loader\_enriched) | snowplow-devops/postgres-loader-kinesis-ec2/aws | 0.5.0 | +| [postgres\_loader\_bad](#module\_postgres\_loader\_bad) | snowplow-devops/postgres-loader-kinesis-ec2/aws | 0.5.1 | +| [postgres\_loader\_enriched](#module\_postgres\_loader\_enriched) | snowplow-devops/postgres-loader-kinesis-ec2/aws | 0.5.1 | | [postgres\_loader\_rds](#module\_postgres\_loader\_rds) | snowplow-devops/rds/aws | 0.4.0 | | [raw\_stream](#module\_raw\_stream) | snowplow-devops/kinesis-stream/aws | 0.3.0 | -| [rs\_loader](#module\_rs\_loader) | snowplow-devops/redshift-loader-ec2/aws | 0.2.0 | -| [rs\_transformer\_stsv](#module\_rs\_transformer\_stsv) | snowplow-devops/transformer-kinesis-ec2/aws | 0.4.0 | -| [s3\_loader\_bad](#module\_s3\_loader\_bad) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.5.0 | -| [s3\_loader\_enriched](#module\_s3\_loader\_enriched) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.5.0 | -| [s3\_loader\_raw](#module\_s3\_loader\_raw) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.5.0 | +| [rs\_loader](#module\_rs\_loader) | snowplow-devops/redshift-loader-ec2/aws | 0.2.1 | +| [rs\_transformer\_stsv](#module\_rs\_transformer\_stsv) | snowplow-devops/transformer-kinesis-ec2/aws | 0.4.1 | +| [s3\_loader\_bad](#module\_s3\_loader\_bad) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.5.1 | +| [s3\_loader\_enriched](#module\_s3\_loader\_enriched) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.5.1 | +| [s3\_loader\_raw](#module\_s3\_loader\_raw) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.5.1 | | [s3\_pipeline\_bucket](#module\_s3\_pipeline\_bucket) | snowplow-devops/s3-bucket/aws | 0.2.0 | -| [sf\_loader](#module\_sf\_loader) | snowplow-devops/snowflake-loader-ec2/aws | 0.3.0 | -| [sf\_transformer\_wrj](#module\_sf\_transformer\_wrj) | snowplow-devops/transformer-kinesis-ec2/aws | 0.4.0 | -| [snowflake\_streaming\_loader\_enriched](#module\_snowflake\_streaming\_loader\_enriched) | snowplow-devops/snowflake-streaming-loader-ec2/aws | 0.1.0 | +| [sf\_loader](#module\_sf\_loader) | snowplow-devops/snowflake-loader-ec2/aws | 0.3.1 | +| [sf\_transformer\_wrj](#module\_sf\_transformer\_wrj) | snowplow-devops/transformer-kinesis-ec2/aws | 0.4.1 | +| [snowflake\_streaming\_loader\_enriched](#module\_snowflake\_streaming\_loader\_enriched) | snowplow-devops/snowflake-streaming-loader-ec2/aws | 0.1.2 | ## Resources @@ -79,6 +79,7 @@ | [postgres\_db\_name](#input\_postgres\_db\_name) | The name of the database to connect to | `string` | `""` | no | | [postgres\_db\_password](#input\_postgres\_db\_password) | The password to use to connect to the database | `string` | `""` | no | | [postgres\_db\_username](#input\_postgres\_db\_username) | The username to use to connect to the database | `string` | `""` | no | +| [private\_ecr\_registry](#input\_private\_ecr\_registry) | The URL of an ECR registry that the sub-account has access to (e.g. '000000000000.dkr.ecr.cn-north-1.amazonaws.com.cn/') | `string` | `""` | no | | [redshift\_database](#input\_redshift\_database) | Redshift database name | `string` | `""` | no | | [redshift\_enabled](#input\_redshift\_enabled) | Whether to enable loading into a Redshift Database | `bool` | `false` | no | | [redshift\_host](#input\_redshift\_host) | Redshift cluster hostname | `string` | `""` | no | diff --git a/terraform/aws/pipeline/secure/main.tf b/terraform/aws/pipeline/secure/main.tf index 3057d06..170dd1d 100644 --- a/terraform/aws/pipeline/secure/main.tf +++ b/terraform/aws/pipeline/secure/main.tf @@ -86,7 +86,7 @@ module "collector_lb" { module "collector_kinesis" { source = "snowplow-devops/collector-kinesis-ec2/aws" - version = "0.9.0" + version = "0.9.1" accept_limited_use_license = var.accept_limited_use_license @@ -115,12 +115,14 @@ module "collector_kinesis" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } # 3. Deploy Enrichment module "enrich_kinesis" { source = "snowplow-devops/enrich-kinesis-ec2/aws" - version = "0.6.0" + version = "0.6.1" accept_limited_use_license = var.accept_limited_use_license @@ -152,4 +154,6 @@ module "enrich_kinesis" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } diff --git a/terraform/aws/pipeline/secure/target_amazon_s3.tf b/terraform/aws/pipeline/secure/target_amazon_s3.tf index 89254e4..8906934 100644 --- a/terraform/aws/pipeline/secure/target_amazon_s3.tf +++ b/terraform/aws/pipeline/secure/target_amazon_s3.tf @@ -1,6 +1,6 @@ module "s3_loader_raw" { source = "snowplow-devops/s3-loader-kinesis-ec2/aws" - version = "0.5.0" + version = "0.5.1" accept_limited_use_license = var.accept_limited_use_license @@ -30,11 +30,13 @@ module "s3_loader_raw" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } module "s3_loader_bad" { source = "snowplow-devops/s3-loader-kinesis-ec2/aws" - version = "0.5.0" + version = "0.5.1" accept_limited_use_license = var.accept_limited_use_license @@ -67,11 +69,13 @@ module "s3_loader_bad" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } module "s3_loader_enriched" { source = "snowplow-devops/s3-loader-kinesis-ec2/aws" - version = "0.5.0" + version = "0.5.1" accept_limited_use_license = var.accept_limited_use_license @@ -103,4 +107,6 @@ module "s3_loader_enriched" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } diff --git a/terraform/aws/pipeline/secure/target_databricks.tf b/terraform/aws/pipeline/secure/target_databricks.tf index 33304c6..d0fbaa4 100644 --- a/terraform/aws/pipeline/secure/target_databricks.tf +++ b/terraform/aws/pipeline/secure/target_databricks.tf @@ -9,7 +9,7 @@ resource "aws_sqs_queue" "db_message_queue" { module "db_transformer_wrp" { source = "snowplow-devops/transformer-kinesis-ec2/aws" - version = "0.4.0" + version = "0.4.1" accept_limited_use_license = var.accept_limited_use_license @@ -46,11 +46,13 @@ module "db_transformer_wrp" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } module "db_loader" { source = "snowplow-devops/databricks-loader-ec2/aws" - version = "0.2.0" + version = "0.2.1" accept_limited_use_license = var.accept_limited_use_license @@ -86,4 +88,6 @@ module "db_loader" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } diff --git a/terraform/aws/pipeline/secure/target_postgres.tf b/terraform/aws/pipeline/secure/target_postgres.tf index 78b39f0..905876d 100644 --- a/terraform/aws/pipeline/secure/target_postgres.tf +++ b/terraform/aws/pipeline/secure/target_postgres.tf @@ -21,7 +21,7 @@ module "postgres_loader_rds" { module "postgres_loader_enriched" { source = "snowplow-devops/postgres-loader-kinesis-ec2/aws" - version = "0.5.0" + version = "0.5.1" accept_limited_use_license = var.accept_limited_use_license @@ -61,11 +61,13 @@ module "postgres_loader_enriched" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } module "postgres_loader_bad" { source = "snowplow-devops/postgres-loader-kinesis-ec2/aws" - version = "0.5.0" + version = "0.5.1" accept_limited_use_license = var.accept_limited_use_license @@ -105,4 +107,6 @@ module "postgres_loader_bad" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } diff --git a/terraform/aws/pipeline/secure/target_redshift.tf b/terraform/aws/pipeline/secure/target_redshift.tf index a4e8244..d311ec4 100644 --- a/terraform/aws/pipeline/secure/target_redshift.tf +++ b/terraform/aws/pipeline/secure/target_redshift.tf @@ -9,7 +9,7 @@ resource "aws_sqs_queue" "rs_message_queue" { module "rs_transformer_stsv" { source = "snowplow-devops/transformer-kinesis-ec2/aws" - version = "0.4.0" + version = "0.4.1" accept_limited_use_license = var.accept_limited_use_license @@ -46,11 +46,13 @@ module "rs_transformer_stsv" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } module "rs_loader" { source = "snowplow-devops/redshift-loader-ec2/aws" - version = "0.2.0" + version = "0.2.1" accept_limited_use_license = var.accept_limited_use_license @@ -86,4 +88,6 @@ module "rs_loader" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } diff --git a/terraform/aws/pipeline/secure/target_snowflake.tf b/terraform/aws/pipeline/secure/target_snowflake.tf index b7aa2fa..5abd898 100644 --- a/terraform/aws/pipeline/secure/target_snowflake.tf +++ b/terraform/aws/pipeline/secure/target_snowflake.tf @@ -9,7 +9,7 @@ resource "aws_sqs_queue" "sf_message_queue" { module "sf_transformer_wrj" { source = "snowplow-devops/transformer-kinesis-ec2/aws" - version = "0.4.0" + version = "0.4.1" accept_limited_use_license = var.accept_limited_use_license @@ -46,11 +46,13 @@ module "sf_transformer_wrj" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } module "sf_loader" { source = "snowplow-devops/snowflake-loader-ec2/aws" - version = "0.3.0" + version = "0.3.1" accept_limited_use_license = var.accept_limited_use_license @@ -87,4 +89,6 @@ module "sf_loader" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } diff --git a/terraform/aws/pipeline/secure/target_snowflake_streaming.tf b/terraform/aws/pipeline/secure/target_snowflake_streaming.tf index 858d856..59b0ff6 100644 --- a/terraform/aws/pipeline/secure/target_snowflake_streaming.tf +++ b/terraform/aws/pipeline/secure/target_snowflake_streaming.tf @@ -1,6 +1,6 @@ module "snowflake_streaming_loader_enriched" { source = "snowplow-devops/snowflake-streaming-loader-ec2/aws" - version = "0.1.0" + version = "0.1.2" accept_limited_use_license = var.accept_limited_use_license @@ -35,4 +35,6 @@ module "snowflake_streaming_loader_enriched" { cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days + + private_ecr_registry = var.private_ecr_registry } diff --git a/terraform/aws/pipeline/secure/terraform.tfvars b/terraform/aws/pipeline/secure/terraform.tfvars index b48e94f..4f45e3b 100644 --- a/terraform/aws/pipeline/secure/terraform.tfvars +++ b/terraform/aws/pipeline/secure/terraform.tfvars @@ -131,3 +131,6 @@ cloudwatch_logs_retention_days = 7 # Extra Tags to append to created resources (optional) tags = {} + +# Image Repositories +private_ecr_registry = "" diff --git a/terraform/aws/pipeline/secure/variables.tf b/terraform/aws/pipeline/secure/variables.tf index 912af35..24a40a3 100644 --- a/terraform/aws/pipeline/secure/variables.tf +++ b/terraform/aws/pipeline/secure/variables.tf @@ -126,6 +126,12 @@ variable "cloudwatch_logs_retention_days" { type = number } +variable "private_ecr_registry" { + description = "The URL of an ECR registry that the sub-account has access to (e.g. '000000000000.dkr.ecr.cn-north-1.amazonaws.com.cn/')" + type = string + default = "" +} + # --- Target: Amazon S3 variable "s3_raw_enabled" {
"certificate_arn": "",
"enabled": false
}