diff --git a/terraform/aws/pipeline/default/README.md b/terraform/aws/pipeline/default/README.md
index 694f17e..95d2148 100644
--- a/terraform/aws/pipeline/default/README.md
+++ b/terraform/aws/pipeline/default/README.md
@@ -36,6 +36,7 @@
| [s3\_pipeline\_bucket](#module\_s3\_pipeline\_bucket) | snowplow-devops/s3-bucket/aws | 0.2.0 |
| [sf\_loader](#module\_sf\_loader) | snowplow-devops/snowflake-loader-ec2/aws | 0.3.0 |
| [sf\_transformer\_wrj](#module\_sf\_transformer\_wrj) | snowplow-devops/transformer-kinesis-ec2/aws | 0.4.0 |
+| [snowflake\_streaming\_loader\_enriched](#module\_snowflake\_streaming\_loader\_enriched) | snowplow-devops/snowflake-streaming-loader-ec2/aws | 0.1.0 |
## Resources
@@ -97,6 +98,12 @@
| [snowflake\_loader\_user](#input\_snowflake\_loader\_user) | The Snowflake user used by Snowflake Loader | `string` | `""` | no |
| [snowflake\_region](#input\_snowflake\_region) | Region of Snowflake account | `string` | `""` | no |
| [snowflake\_schema](#input\_snowflake\_schema) | Snowflake schema name | `string` | `""` | no |
+| [snowflake\_streaming\_account\_url](#input\_snowflake\_streaming\_account\_url) | Snowflake account URL to use | `string` | `""` | no |
+| [snowflake\_streaming\_database](#input\_snowflake\_streaming\_database) | Snowflake database name | `string` | `""` | no |
+| [snowflake\_streaming\_enabled](#input\_snowflake\_streaming\_enabled) | Whether to enable loading into a Snowflake Database with a Streaming Loader | `bool` | `false` | no |
+| [snowflake\_streaming\_loader\_private\_key](#input\_snowflake\_streaming\_loader\_private\_key) | The private key to use for the loader user | `string` | `""` | no |
+| [snowflake\_streaming\_loader\_user](#input\_snowflake\_streaming\_loader\_user) | The Snowflake user used by Snowflake Streaming Loader | `string` | `""` | no |
+| [snowflake\_streaming\_schema](#input\_snowflake\_streaming\_schema) | Snowflake schema name | `string` | `""` | no |
| [snowflake\_transformer\_window\_period\_min](#input\_snowflake\_transformer\_window\_period\_min) | Frequency to emit transforming finished message - 5,10,15,20,30,60 etc minutes | `number` | `5` | no |
| [snowflake\_warehouse](#input\_snowflake\_warehouse) | Snowflake warehouse name | `string` | `""` | no |
| [ssl\_information](#input\_ssl\_information) | The ARN of an Amazon Certificate Manager certificate to bind to the load balancer |
object({
enabled = bool
certificate_arn = string
})
| {
"certificate_arn": "",
"enabled": false
}
| no |
diff --git a/terraform/aws/pipeline/default/target_snowflake_streaming.tf b/terraform/aws/pipeline/default/target_snowflake_streaming.tf
new file mode 100644
index 0000000..26f677d
--- /dev/null
+++ b/terraform/aws/pipeline/default/target_snowflake_streaming.tf
@@ -0,0 +1,36 @@
+module "snowflake_streaming_loader_enriched" {
+ source = "snowplow-devops/snowflake-streaming-loader-ec2/aws"
+ version = "0.1.0"
+
+ accept_limited_use_license = var.accept_limited_use_license
+
+ count = var.snowflake_streaming_enabled ? 1 : 0
+
+ name = "${var.prefix}-sf-streaming-enriched"
+ vpc_id = var.vpc_id
+ subnet_ids = var.public_subnet_ids
+
+ in_stream_name = module.enriched_stream.name
+ bad_stream_name = module.bad_1_stream.name
+
+ snowflake_account_url = var.snowflake_streaming_account_url
+ snowflake_loader_user = var.snowflake_streaming_loader_user
+ snowflake_private_key = var.snowflake_streaming_loader_private_key
+ snowflake_database = var.snowflake_streaming_database
+ snowflake_schema = var.snowflake_streaming_schema
+
+ ssh_key_name = aws_key_pair.pipeline.key_name
+ ssh_ip_allowlist = var.ssh_ip_allowlist
+
+ iam_permissions_boundary = var.iam_permissions_boundary
+
+ telemetry_enabled = var.telemetry_enabled
+ user_provided_id = var.user_provided_id
+
+ kcl_write_max_capacity = var.pipeline_kcl_write_max_capacity
+
+ tags = var.tags
+
+ cloudwatch_logs_enabled = var.cloudwatch_logs_enabled
+ cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days
+}
diff --git a/terraform/aws/pipeline/default/terraform.tfvars b/terraform/aws/pipeline/default/terraform.tfvars
index 1c48703..7f5506b 100644
--- a/terraform/aws/pipeline/default/terraform.tfvars
+++ b/terraform/aws/pipeline/default/terraform.tfvars
@@ -72,6 +72,17 @@ snowflake_warehouse = ""
# This controls how often data will be loading into Snowflake
snowflake_transformer_window_period_min = 1
+# --- Target: Snowflake Streaming
+# Follow the guide to get input values for the loader:
+# https://docs.snowplow.io/docs/getting-started-on-snowplow-open-source/quick-start-aws
+snowflake_streaming_enabled = false
+
+snowflake_streaming_account_url = ""
+snowflake_streaming_loader_user = ""
+snowflake_streaming_loader_private_key = ""
+snowflake_streaming_database = ""
+snowflake_streaming_schema = ""
+
# --- Target: Databricks
# Follow the guide to get input values for the loader:
# https://docs.snowplow.io/docs/getting-started-on-snowplow-open-source/quick-start-aws
diff --git a/terraform/aws/pipeline/default/variables.tf b/terraform/aws/pipeline/default/variables.tf
index 397caab..1f5e675 100644
--- a/terraform/aws/pipeline/default/variables.tf
+++ b/terraform/aws/pipeline/default/variables.tf
@@ -237,6 +237,45 @@ variable "snowflake_transformer_window_period_min" {
default = 5
}
+# --- Target: SnowflakeDB Streaming
+
+variable "snowflake_streaming_enabled" {
+ description = "Whether to enable loading into a Snowflake Database with a Streaming Loader"
+ default = false
+ type = bool
+}
+
+variable "snowflake_streaming_account_url" {
+ description = "Snowflake account URL to use"
+ type = string
+ default = ""
+}
+
+variable "snowflake_streaming_loader_user" {
+ description = "The Snowflake user used by Snowflake Streaming Loader"
+ type = string
+ default = ""
+}
+
+variable "snowflake_streaming_loader_private_key" {
+ description = "The private key to use for the loader user"
+ type = string
+ sensitive = true
+ default = ""
+}
+
+variable "snowflake_streaming_database" {
+ description = "Snowflake database name"
+ type = string
+ default = ""
+}
+
+variable "snowflake_streaming_schema" {
+ description = "Snowflake schema name"
+ type = string
+ default = ""
+}
+
# --- Target: Databricks
variable "databricks_enabled" {
diff --git a/terraform/aws/pipeline/secure/README.md b/terraform/aws/pipeline/secure/README.md
index 5773ec3..deaa851 100644
--- a/terraform/aws/pipeline/secure/README.md
+++ b/terraform/aws/pipeline/secure/README.md
@@ -36,6 +36,7 @@
| [s3\_pipeline\_bucket](#module\_s3\_pipeline\_bucket) | snowplow-devops/s3-bucket/aws | 0.2.0 |
| [sf\_loader](#module\_sf\_loader) | snowplow-devops/snowflake-loader-ec2/aws | 0.3.0 |
| [sf\_transformer\_wrj](#module\_sf\_transformer\_wrj) | snowplow-devops/transformer-kinesis-ec2/aws | 0.4.0 |
+| [snowflake\_streaming\_loader\_enriched](#module\_snowflake\_streaming\_loader\_enriched) | snowplow-devops/snowflake-streaming-loader-ec2/aws | 0.1.0 |
## Resources
@@ -97,6 +98,12 @@
| [snowflake\_loader\_user](#input\_snowflake\_loader\_user) | The Snowflake user used by Snowflake Loader | `string` | `""` | no |
| [snowflake\_region](#input\_snowflake\_region) | Region of Snowflake account | `string` | `""` | no |
| [snowflake\_schema](#input\_snowflake\_schema) | Snowflake schema name | `string` | `""` | no |
+| [snowflake\_streaming\_account\_url](#input\_snowflake\_streaming\_account\_url) | Snowflake account URL to use | `string` | `""` | no |
+| [snowflake\_streaming\_database](#input\_snowflake\_streaming\_database) | Snowflake database name | `string` | `""` | no |
+| [snowflake\_streaming\_enabled](#input\_snowflake\_streaming\_enabled) | Whether to enable loading into a Snowflake Database with a Streaming Loader | `bool` | `false` | no |
+| [snowflake\_streaming\_loader\_private\_key](#input\_snowflake\_streaming\_loader\_private\_key) | The private key to use for the loader user | `string` | `""` | no |
+| [snowflake\_streaming\_loader\_user](#input\_snowflake\_streaming\_loader\_user) | The Snowflake user used by Snowflake Streaming Loader | `string` | `""` | no |
+| [snowflake\_streaming\_schema](#input\_snowflake\_streaming\_schema) | Snowflake schema name | `string` | `""` | no |
| [snowflake\_transformer\_window\_period\_min](#input\_snowflake\_transformer\_window\_period\_min) | Frequency to emit transforming finished message - 5,10,15,20,30,60 etc minutes | `number` | `5` | no |
| [snowflake\_warehouse](#input\_snowflake\_warehouse) | Snowflake warehouse name | `string` | `""` | no |
| [ssl\_information](#input\_ssl\_information) | The ARN of an Amazon Certificate Manager certificate to bind to the load balancer | object({
enabled = bool
certificate_arn = string
})
| {
"certificate_arn": "",
"enabled": false
}
| no |
diff --git a/terraform/aws/pipeline/secure/target_snowflake_streaming.tf b/terraform/aws/pipeline/secure/target_snowflake_streaming.tf
new file mode 100644
index 0000000..858d856
--- /dev/null
+++ b/terraform/aws/pipeline/secure/target_snowflake_streaming.tf
@@ -0,0 +1,38 @@
+module "snowflake_streaming_loader_enriched" {
+ source = "snowplow-devops/snowflake-streaming-loader-ec2/aws"
+ version = "0.1.0"
+
+ accept_limited_use_license = var.accept_limited_use_license
+
+ count = var.snowflake_streaming_enabled ? 1 : 0
+
+ name = "${var.prefix}-sf-streaming-enriched"
+ vpc_id = var.vpc_id
+ subnet_ids = var.private_subnet_ids
+
+ in_stream_name = module.enriched_stream.name
+ bad_stream_name = module.bad_1_stream.name
+
+ snowflake_account_url = var.snowflake_streaming_account_url
+ snowflake_loader_user = var.snowflake_streaming_loader_user
+ snowflake_private_key = var.snowflake_streaming_loader_private_key
+ snowflake_database = var.snowflake_streaming_database
+ snowflake_schema = var.snowflake_streaming_schema
+
+ ssh_key_name = aws_key_pair.pipeline.key_name
+ ssh_ip_allowlist = var.ssh_ip_allowlist
+
+ iam_permissions_boundary = var.iam_permissions_boundary
+
+ telemetry_enabled = var.telemetry_enabled
+ user_provided_id = var.user_provided_id
+
+ kcl_write_max_capacity = var.pipeline_kcl_write_max_capacity
+
+ associate_public_ip_address = false
+
+ tags = var.tags
+
+ cloudwatch_logs_enabled = var.cloudwatch_logs_enabled
+ cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days
+}
diff --git a/terraform/aws/pipeline/secure/terraform.tfvars b/terraform/aws/pipeline/secure/terraform.tfvars
index 96e1a31..b48e94f 100644
--- a/terraform/aws/pipeline/secure/terraform.tfvars
+++ b/terraform/aws/pipeline/secure/terraform.tfvars
@@ -74,6 +74,17 @@ snowflake_warehouse = ""
# This controls how often data will be loading into Snowflake
snowflake_transformer_window_period_min = 1
+# --- Target: Snowflake Streaming
+# Follow the guide to get input values for the loader:
+# https://docs.snowplow.io/docs/getting-started-on-snowplow-open-source/quick-start-aws
+snowflake_streaming_enabled = false
+
+snowflake_streaming_account_url = ""
+snowflake_streaming_loader_user = ""
+snowflake_streaming_loader_private_key = ""
+snowflake_streaming_database = ""
+snowflake_streaming_schema = ""
+
# --- Target: Databricks
# Follow the guide to get input values for the loader:
# https://docs.snowplow.io/docs/getting-started-on-snowplow-open-source/quick-start-aws
diff --git a/terraform/aws/pipeline/secure/variables.tf b/terraform/aws/pipeline/secure/variables.tf
index 9387b97..912af35 100644
--- a/terraform/aws/pipeline/secure/variables.tf
+++ b/terraform/aws/pipeline/secure/variables.tf
@@ -236,6 +236,45 @@ variable "snowflake_transformer_window_period_min" {
default = 5
}
+# --- Target: SnowflakeDB Streaming
+
+variable "snowflake_streaming_enabled" {
+ description = "Whether to enable loading into a Snowflake Database with a Streaming Loader"
+ default = false
+ type = bool
+}
+
+variable "snowflake_streaming_account_url" {
+ description = "Snowflake account URL to use"
+ type = string
+ default = ""
+}
+
+variable "snowflake_streaming_loader_user" {
+ description = "The Snowflake user used by Snowflake Streaming Loader"
+ type = string
+ default = ""
+}
+
+variable "snowflake_streaming_loader_private_key" {
+ description = "The private key to use for the loader user"
+ type = string
+ sensitive = true
+ default = ""
+}
+
+variable "snowflake_streaming_database" {
+ description = "Snowflake database name"
+ type = string
+ default = ""
+}
+
+variable "snowflake_streaming_schema" {
+ description = "Snowflake schema name"
+ type = string
+ default = ""
+}
+
# --- Target: Databricks
variable "databricks_enabled" {