diff --git a/README.md b/README.md index 7316cbd..de2c8cf 100644 --- a/README.md +++ b/README.md @@ -52,10 +52,12 @@ module "transformer_kinesis" { source = "snowplow-devops/transformer-kinesis-ec2/aws" name = var.name + app_version = "5.8.0" vpc_id = var.vpc_id subnet_ids = var.subnet_ids stream_name = module.enriched_stream.name + config_b64 = "" s3_bucket_name = var.transformed_bucket s3_bucket_object_prefix = "transformed/good" window_period_min = 10 @@ -111,40 +113,27 @@ module "transformer_kinesis" { | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [name](#input\_name) | A name which will be pre-pended to the resources created | `string` | n/a | yes | +| [name](#input\_app\_version) | Version of transformer-kinesis | `string` | "5.8.0" | no | | [s3\_bucket\_name](#input\_s3\_bucket\_name) | The name of the S3 bucket events will be loaded into | `string` | n/a | yes | | [s3\_bucket\_object\_prefix](#input\_s3\_bucket\_object\_prefix) | An optional prefix under which Snowplow data will be saved | `string` | n/a | yes | | [ssh\_key\_name](#input\_ssh\_key\_name) | The name of the SSH key-pair to attach to all EC2 nodes deployed | `string` | n/a | yes | | [stream\_name](#input\_stream\_name) | The name of the input kinesis stream that the Transformer will pull data from | `string` | n/a | yes | | [subnet\_ids](#input\_subnet\_ids) | The list of subnets to deploy Transformer across | `list(string)` | n/a | yes | | [vpc\_id](#input\_vpc\_id) | The VPC to deploy Transformer within | `string` | n/a | yes | -| [window\_period\_min](#input\_window\_period\_min) | Frequency to emit loading finished message - 5,10,15,20,30,60 etc minutes | `number` | n/a | yes | | [amazon\_linux\_2\_ami\_id](#input\_amazon\_linux\_2\_ami\_id) | The AMI ID to use which must be based of of Amazon Linux 2; by default the latest community version is used | `string` | `""` | no | | [associate\_public\_ip\_address](#input\_associate\_public\_ip\_address) | Whether to assign a public ip address to this instance | `bool` | `true` | no | | [cloudwatch\_logs\_enabled](#input\_cloudwatch\_logs\_enabled) | Whether application logs should be reported to CloudWatch | `bool` | `true` | no | | [cloudwatch\_logs\_retention\_days](#input\_cloudwatch\_logs\_retention\_days) | The length of time in days to retain logs for | `number` | `7` | no | | [custom\_iglu\_resolvers](#input\_custom\_iglu\_resolvers) | The custom Iglu Resolvers that will be used by Transformer | 
list(object({
    name            = string
    priority        = number
    uri             = string
    api_key         = string
    vendor_prefixes = list(string)
  }))
| `[]` | no | | [default\_iglu\_resolvers](#input\_default\_iglu\_resolvers) | The default Iglu Resolvers that will be used by Transformer | 
list(object({
    name            = string
    priority        = number
    uri             = string
    api_key         = string
    vendor_prefixes = list(string)
  }))
| 
[
  {
    "api_key": "",
    "name": "Iglu Central",
    "priority": 10,
    "uri": "http://iglucentral.com",
    "vendor_prefixes": []
  },
  {
    "api_key": "",
    "name": "Iglu Central - Mirror 01",
    "priority": 20,
    "uri": "http://mirror01.iglucentral.com",
    "vendor_prefixes": []
  }
]
| no | -| [default\_shred\_format](#input\_default\_shred\_format) | Format used by default when format type is 'shred' (TSV or JSON) | `string` | `"TSV"` | no | | [iam\_permissions\_boundary](#input\_iam\_permissions\_boundary) | The permissions boundary ARN to set on IAM roles created | `string` | `""` | no | -| [initial\_position](#input\_initial\_position) | Where to start processing the input Kinesis Stream from (TRIM\_HORIZON or LATEST) | `string` | `"TRIM_HORIZON"` | no | | [instance\_type](#input\_instance\_type) | The instance type to use | `string` | `"t3a.small"` | no | | [java\_opts](#input\_java\_opts) | Custom JAVA Options | `string` | `"-Dorg.slf4j.simpleLogger.defaultLogLevel=info -XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=75"` | no | +| [config\_b64](#input\_config\_b64) | base64 encoded configuration | `string` | `` | yes | | [kcl\_read\_max\_capacity](#input\_kcl\_read\_max\_capacity) | The maximum READ capacity for the KCL DynamoDB table | `number` | `10` | no | | [kcl\_read\_min\_capacity](#input\_kcl\_read\_min\_capacity) | The minimum READ capacity for the KCL DynamoDB table | `number` | `1` | no | | [kcl\_write\_max\_capacity](#input\_kcl\_write\_max\_capacity) | The maximum WRITE capacity for the KCL DynamoDB table | `number` | `10` | no | | [kcl\_write\_min\_capacity](#input\_kcl\_write\_min\_capacity) | The minimum WRITE capacity for the KCL DynamoDB table | `number` | `1` | no | -| [schemas\_json](#input\_schemas\_json) | List of schemas to get shredded as JSON | `list(string)` | `[]` | no | -| [schemas\_skip](#input\_schemas\_skip) | List of schemas to not get shredded (and thus not loaded) | `list(string)` | `[]` | no | -| [schemas\_tsv](#input\_schemas\_tsv) | List of schemas to get shredded as TSV | `list(string)` | `[]` | no | -| [sns\_topic\_arn](#input\_sns\_topic\_arn) | The ARN of the SNS topic that Transformer will send the transforming complete message. Either `sqs_queue_name` or `sns_topic_arn` needs to be set | `string` | `""` | no | -| [sqs\_queue\_name](#input\_sqs\_queue\_name) | The name of the SQS queue that Transformer will send the transforming complete message. Either `sqs_queue_name` or `sns_topic_arn` needs to be set | `string` | `""` | no | -| [ssh\_ip\_allowlist](#input\_ssh\_ip\_allowlist) | The list of CIDR ranges to allow SSH traffic from | `list(any)` | 
[
  "0.0.0.0/0"
]
| no | -| [tags](#input\_tags) | The tags to append to this resource | `map(string)` | `{}` | no | -| [telemetry\_enabled](#input\_telemetry\_enabled) | Whether or not to send telemetry information back to Snowplow Analytics Ltd | `bool` | `true` | no | -| [transformation\_type](#input\_transformation\_type) | Type of the transformation (shred or widerow) | `string` | `"shred"` | no | -| [transformer\_compression](#input\_transformer\_compression) | Transformer output compression, GZIP or NONE | `string` | `"GZIP"` | no | -| [user\_provided\_id](#input\_user\_provided\_id) | An optional unique identifier to identify the telemetry events emitted by this stack | `string` | `""` | no | -| [widerow\_file\_format](#input\_widerow\_file\_format) | The output file\_format from the widerow transformation\_type selected (json or parquet) | `string` | `"json"` | no | ## Outputs diff --git a/main.tf b/main.tf index efe524b..a480986 100644 --- a/main.tf +++ b/main.tf @@ -2,13 +2,12 @@ locals { module_name = "transformer-kinesis-ec2" module_version = "0.3.4" - app_name = "transformer-kinesis" - app_version = "5.6.0" + app_name = "transformer-kinesis" local_tags = { Name = var.name app_name = local.app_name - app_version = local.app_version + app_version = var.app_version module_name = local.module_name module_version = local.module_version } @@ -68,7 +67,7 @@ module "telemetry" { cloud = "AWS" region = data.aws_region.current.name app_name = local.app_name - app_version = local.app_version + app_version = var.app_version module_name = local.module_name module_version = local.module_version } @@ -336,38 +335,10 @@ locals { iglu_resolver = templatefile("${path.module}/templates/iglu_resolver.json.tmpl", { resolvers = jsonencode(local.resolvers) }) - config = templatefile("${path.module}/templates/config.json.tmpl", { - app_name = var.name - stream_name = var.stream_name - region = data.aws_region.current.name - initial_position = var.initial_position - transformed_output = local.s3_path - compression = var.transformer_compression - window_period = "${var.window_period_min} minutes" - sqs_enabled = local.sqs_enabled - sqs_queue_name = var.sqs_queue_name - sns_topic_arn = var.sns_topic_arn - transformation_type = var.transformation_type - default_shred_format = var.default_shred_format - schemas_json = jsonencode(var.schemas_json) - schemas_tsv = jsonencode(var.schemas_tsv) - schemas_skip = jsonencode(var.schemas_skip) - widerow_file_format = var.widerow_file_format - - telemetry_disable = !var.telemetry_enabled - telemetry_collector_uri = join("", module.telemetry.*.collector_uri) - telemetry_collector_port = 443 - telemetry_secure = true - telemetry_user_provided_id = var.user_provided_id - telemetry_auto_gen_id = join("", module.telemetry.*.auto_generated_id) - telemetry_module_name = local.module_name - telemetry_module_version = local.module_version - }) - user_data = templatefile("${path.module}/templates/user-data.sh.tmpl", { - config_b64 = base64encode(local.config) + config_b64 = var.config_b64 iglu_resolver_b64 = base64encode(local.iglu_resolver) - version = local.app_version + version = var.app_version telemetry_script = join("", module.telemetry.*.amazon_linux_2_user_data) diff --git a/variables.tf b/variables.tf index 3eba6d6..0e9a25d 100644 --- a/variables.tf +++ b/variables.tf @@ -3,6 +3,12 @@ variable "name" { type = string } +variable "app_version" { + description = "Version of the transformer" + type = string + default = "5.8.0" +} + variable "vpc_id" { description = "The VPC to deploy Transformer within" type = string @@ -103,9 +109,8 @@ variable "stream_name" { type = string } -variable "initial_position" { - description = "Where to start processing the input Kinesis Stream from (TRIM_HORIZON or LATEST)" - default = "TRIM_HORIZON" +variable "config_b64" { + description = "Base64 encoded config" type = string } @@ -119,17 +124,6 @@ variable "s3_bucket_object_prefix" { type = string } -variable "transformer_compression" { - description = "Transformer output compression, GZIP or NONE" - default = "GZIP" - type = string -} - -variable "window_period_min" { - description = "Frequency to emit loading finished message - 5,10,15,20,30,60 etc minutes" - type = number -} - variable "sqs_queue_name" { description = "The name of the SQS queue that Transformer will send the transforming complete message. Either `sqs_queue_name` or `sns_topic_arn` needs to be set" default = "" @@ -142,42 +136,6 @@ variable "sns_topic_arn" { type = string } -variable "transformation_type" { - description = "Type of the transformation (shred or widerow)" - default = "shred" - type = string -} - -variable "default_shred_format" { - description = "Format used by default when format type is 'shred' (TSV or JSON)" - default = "TSV" - type = string -} - -variable "schemas_json" { - description = "List of schemas to get shredded as JSON" - default = [] - type = list(string) -} - -variable "schemas_tsv" { - description = "List of schemas to get shredded as TSV" - default = [] - type = list(string) -} - -variable "schemas_skip" { - description = "List of schemas to not get shredded (and thus not loaded)" - default = [] - type = list(string) -} - -variable "widerow_file_format" { - description = "The output file_format from the widerow transformation_type selected (json or parquet)" - default = "json" - type = string -} - # --- Iglu Resolver variable "default_iglu_resolvers" {