From 822e19dfdfbe45c21ee48650d3a55909b9c78689 Mon Sep 17 00:00:00 2001 From: ccuffari <75679158+ccuffari@users.noreply.github.com> Date: Thu, 10 Oct 2024 16:06:28 +0200 Subject: [PATCH 01/22] Aggiunta codice di test del modulo per Azure data factory --- .../DataFactory/environments/dev/backend.tf | 8 + .../DataFactory/environments/dev/main.tf | 21 +++ .../DataFactory/environments/dev/outputs.tf | 0 .../environments/dev/terraform.tfvars | 165 ++++++++++++++++++ .../DataFactory/environments/dev/variables.tf | 75 ++++++++ .../DataFactory/environments/dev/versions.tf | 10 ++ .../DataFactory/modules/data_factory/main.tf | 81 +++++++++ .../modules/data_factory/outputs.tf | 4 + .../modules/data_factory/variables.tf | 73 ++++++++ 9 files changed, 437 insertions(+) create mode 100644 src/_modules/test_users/DataFactory/environments/dev/backend.tf create mode 100644 src/_modules/test_users/DataFactory/environments/dev/main.tf create mode 100644 src/_modules/test_users/DataFactory/environments/dev/outputs.tf create mode 100644 src/_modules/test_users/DataFactory/environments/dev/terraform.tfvars create mode 100644 src/_modules/test_users/DataFactory/environments/dev/variables.tf create mode 100644 src/_modules/test_users/DataFactory/environments/dev/versions.tf create mode 100644 src/_modules/test_users/DataFactory/modules/data_factory/main.tf create mode 100644 src/_modules/test_users/DataFactory/modules/data_factory/outputs.tf create mode 100644 src/_modules/test_users/DataFactory/modules/data_factory/variables.tf diff --git a/src/_modules/test_users/DataFactory/environments/dev/backend.tf b/src/_modules/test_users/DataFactory/environments/dev/backend.tf new file mode 100644 index 000000000..64222a48a --- /dev/null +++ b/src/_modules/test_users/DataFactory/environments/dev/backend.tf @@ -0,0 +1,8 @@ +terraform { + backend "azurerm" { + resource_group_name = "dev-fasanorg" + storage_account_name = "stbipdevtest" + container_name = "bc-tf-bip-dev-test" + key = "terraform.tfstate" + } +} diff --git a/src/_modules/test_users/DataFactory/environments/dev/main.tf b/src/_modules/test_users/DataFactory/environments/dev/main.tf new file mode 100644 index 000000000..245649cba --- /dev/null +++ b/src/_modules/test_users/DataFactory/environments/dev/main.tf @@ -0,0 +1,21 @@ +# Main.tf per l'ambiente DEV +provider "azurerm" { + features {} + subscription_id = var.subscription_id +} + +data "azurerm_client_config" "current" {} + +module "data_factory" { + source = "../../modules/data_factory" + + data_factories = var.data_factories + pipelines = var.pipelines + azure_runtimes = var.azure_runtimes + self_hosted_runtimes = var.self_hosted_runtimes + linked_services = var.linked_services + datasets = var.datasets + storage_account_resource_groups = var.storage_account_resource_groups # Aggiungi questa riga +} + + diff --git a/src/_modules/test_users/DataFactory/environments/dev/outputs.tf b/src/_modules/test_users/DataFactory/environments/dev/outputs.tf new file mode 100644 index 000000000..e69de29bb diff --git a/src/_modules/test_users/DataFactory/environments/dev/terraform.tfvars b/src/_modules/test_users/DataFactory/environments/dev/terraform.tfvars new file mode 100644 index 000000000..4c5e9b725 --- /dev/null +++ b/src/_modules/test_users/DataFactory/environments/dev/terraform.tfvars @@ -0,0 +1,165 @@ +data_factories = { + dev_adf = { + name = "adfdevbiptest1" + location = "Italy North" + resource_group_name = "dev-fasanorg" + tags = { + Provider = "BIP" + Environment = "DEV" + Application = "Test" + Owners = "Cristian Felice Cuffari, Domenico Fasano" + } + } +} + +azure_runtimes = { + dev_runtime = { + name = "runtime-dev" + location = "West Europe" + data_factory = "dev_adf" + } +} + +self_hosted_runtimes = { + dev_self_hosted = { + name = "self-hosted-dev" + data_factory = "dev_adf" + } +} + +linked_services = { + dev_blob_service = { + name = "blob-storage-dev" + data_factory = "dev_adf" + type = "AzureBlobStorage" + type_properties_json = < details.storage_account if details.storage_account != "" } + name = each.value + resource_group_name = var.storage_account_resource_groups[each.key] +} + +# Linked service per Azure Data Factory +resource "azurerm_data_factory_linked_custom_service" "this" { + for_each = var.linked_services + name = each.value.name + data_factory_id = azurerm_data_factory.this[each.value.data_factory].id + type = each.value.type + + type_properties_json = < account.primary_connection_string } +} + diff --git a/src/_modules/test_users/DataFactory/modules/data_factory/variables.tf b/src/_modules/test_users/DataFactory/modules/data_factory/variables.tf new file mode 100644 index 000000000..62f25905e --- /dev/null +++ b/src/_modules/test_users/DataFactory/modules/data_factory/variables.tf @@ -0,0 +1,73 @@ +variable "data_factories" { + description = "Elenco delle Data Factory da creare" + type = map(object({ + name = string + location = string + resource_group_name = string + tags = map(string) + })) +} + +variable "pipelines" { + type = map(object({ + name = string + data_factory = string + variables = map(string) # Per le variabili della pipeline + activities_json = string # Il contenuto JSON dell'attività + })) +} + + + + +variable "azure_runtimes" { + description = "Elenco degli Azure Integration Runtimes per Data Factory" + type = map(object({ + name = string + location = string + data_factory = string + })) +} + +variable "self_hosted_runtimes" { + description = "Elenco dei Self-Hosted Integration Runtimes per Data Factory" + type = map(object({ + name = string + data_factory = string + })) +} + +variable "linked_services" { + description = "Elenco dei linked services da creare per Data Factory" + type = map(object({ + name = string + data_factory = string + type = string + type_properties_json = string + storage_account = string + })) +} + +variable "storage_account_resource_groups" { + description = "Resource groups for each storage account used in linked services" + type = map(string) +} + + +variable "datasets" { + description = "Elenco dei dataset da creare per Data Factory" + type = map(object({ + name = string + data_factory = string + type = string + linked_service = string + type_properties_json = string + description = optional(string) + annotations = optional(list(string)) + folder = optional(string) + parameters = optional(map(string)) + additional_properties = optional(map(string)) + schema_json = optional(string) + })) +} + From 2e1af1de71fc0af70a407b7681d4515589b54249 Mon Sep 17 00:00:00 2001 From: ccuffari <75679158+ccuffari@users.noreply.github.com> Date: Fri, 11 Oct 2024 09:39:26 +0200 Subject: [PATCH 02/22] File folder structure check --- src/_modules/data_factory/main.tf | 98 +++++++++++ src/_modules/data_factory/outputs.tf | 10 ++ src/_modules/data_factory/variables.tf | 113 ++++++++++++ .../DataFactory/environments/dev/backend.tf | 8 - .../DataFactory/environments/dev/main.tf | 21 --- .../DataFactory/environments/dev/outputs.tf | 0 .../environments/dev/terraform.tfvars | 165 ------------------ .../DataFactory/environments/dev/variables.tf | 75 -------- .../DataFactory/environments/dev/versions.tf | 10 -- .../DataFactory/modules/data_factory/main.tf | 81 --------- .../modules/data_factory/outputs.tf | 4 - .../modules/data_factory/variables.tf | 73 -------- 12 files changed, 221 insertions(+), 437 deletions(-) create mode 100644 src/_modules/data_factory/main.tf create mode 100644 src/_modules/data_factory/outputs.tf create mode 100644 src/_modules/data_factory/variables.tf delete mode 100644 src/_modules/test_users/DataFactory/environments/dev/backend.tf delete mode 100644 src/_modules/test_users/DataFactory/environments/dev/main.tf delete mode 100644 src/_modules/test_users/DataFactory/environments/dev/outputs.tf delete mode 100644 src/_modules/test_users/DataFactory/environments/dev/terraform.tfvars delete mode 100644 src/_modules/test_users/DataFactory/environments/dev/variables.tf delete mode 100644 src/_modules/test_users/DataFactory/environments/dev/versions.tf delete mode 100644 src/_modules/test_users/DataFactory/modules/data_factory/main.tf delete mode 100644 src/_modules/test_users/DataFactory/modules/data_factory/outputs.tf delete mode 100644 src/_modules/test_users/DataFactory/modules/data_factory/variables.tf diff --git a/src/_modules/data_factory/main.tf b/src/_modules/data_factory/main.tf new file mode 100644 index 000000000..652fc907b --- /dev/null +++ b/src/_modules/data_factory/main.tf @@ -0,0 +1,98 @@ +# Create Azure Data Factory instances +# Uses for_each to create multiple factories based on input variables +# Enables system-assigned managed identity for secure access to resources +resource "azurerm_data_factory" "this" { + for_each = var.data_factories + name = "adf${each.value.name}" + location = each.value.location + resource_group_name = each.value.resource_group_name + tags = each.value.tags + + identity { + type = "SystemAssigned" + } +} + +# Define Data Factory pipelines +# Creates pipelines with custom variables and activities +# Dependencies ensure datasets are created before pipelines +resource "azurerm_data_factory_pipeline" "this" { + for_each = var.pipelines + name = each.value.name + data_factory_id = azurerm_data_factory.this[each.value.data_factory].id + variables = each.value.variables + activities_json = each.value.activities_json + + depends_on = [ + azurerm_data_factory_custom_dataset.this + ] +} + +# Create Azure Integration Runtime +# Managed compute infrastructure for data movement and transformation +# Located in specific Azure regions +resource "azurerm_data_factory_integration_runtime_azure" "azure_runtime" { + for_each = var.azure_runtimes + name = "azure-runtime-${each.value.name}" + data_factory_id = azurerm_data_factory.this[each.value.data_factory].id + location = each.value.location +} + +# Create Self-hosted Integration Runtime +# Enables data movement and transformation in private networks +# Requires manual installation of runtime on premises +resource "azurerm_data_factory_integration_runtime_self_hosted" "self_hosted_runtime" { + for_each = var.self_hosted_runtimes + name = "self-hosted-runtime-${each.value.name}" + data_factory_id = azurerm_data_factory.this[each.value.data_factory].id +} + +# Retrieve existing Storage Account information +# Uses data source to get connection details for linked services +# Filters storage accounts based on provided configuration +data "azurerm_storage_account" "existing" { + for_each = { + for linked_service, details in var.linked_services : linked_service => details.storage_account + if details.storage_account != "" + } + name = each.value + resource_group_name = var.storage_account_resource_groups[each.key] +} + +# Create Linked Services for Data Factory +# Establishes connections to external data sources +# Uses storage account connection strings for authentication +resource "azurerm_data_factory_linked_custom_service" "this" { + for_each = var.linked_services + name = each.value.name + data_factory_id = azurerm_data_factory.this[each.value.data_factory].id + type = each.value.type + type_properties_json = < account.primary_connection_string } +} \ No newline at end of file diff --git a/src/_modules/data_factory/variables.tf b/src/_modules/data_factory/variables.tf new file mode 100644 index 000000000..7ee50706c --- /dev/null +++ b/src/_modules/data_factory/variables.tf @@ -0,0 +1,113 @@ +# Variable to define Azure Data Factories to be created +# Requires: +# - name: unique identifier for the Data Factory +# - location: Azure region where the Data Factory will be deployed +# - resource_group_name: target resource group +# - tags: map of tags for resource organization +variable "data_factories" { + description = "List of Data Factories to be created" + type = map(object({ + name = string + location = string + resource_group_name = string + tags = map(string) + })) +} + +# Variable to define Data Factory pipelines +# Requires: +# - name: pipeline identifier +# - data_factory: reference to parent Data Factory +# - variables: map of pipeline variables +# - activities_json: JSON definition of pipeline activities +variable "pipelines" { + description = "List of pipelines to be created in Data Factory" + type = map(object({ + name = string + data_factory = string + variables = map(string) # Pipeline variables + activities_json = string # Activities JSON content + })) +} + +# Variable to define Azure Integration Runtimes +# These are managed compute resources for data movement +# Requires: +# - name: runtime identifier +# - location: Azure region for runtime deployment +# - data_factory: reference to parent Data Factory +variable "azure_runtimes" { + description = "List of Azure Integration Runtimes for Data Factory" + type = map(object({ + name = string + location = string + data_factory = string + })) +} + +# Variable to define Self-Hosted Integration Runtimes +# These are on-premises or private network runtimes +# Requires: +# - name: runtime identifier +# - data_factory: reference to parent Data Factory +variable "self_hosted_runtimes" { + description = "List of Self-Hosted Integration Runtimes for Data Factory" + type = map(object({ + name = string + data_factory = string + })) +} + +# Variable to define Linked Services +# These establish connections to external data sources +# Requires: +# - name: service identifier +# - data_factory: reference to parent Data Factory +# - type: service type (e.g., AzureBlobStorage) +# - type_properties_json: connection properties in JSON format +# - storage_account: reference to associated storage account +variable "linked_services" { + description = "List of linked services to be created for Data Factory" + type = map(object({ + name = string + data_factory = string + type = string + type_properties_json = string + storage_account = string + })) +} + +# Variable to map storage accounts to their resource groups +# Used to lookup existing storage accounts for linked services +variable "storage_account_resource_groups" { + description = "Resource groups for each storage account used in linked services" + type = map(string) +} + +# Variable to define Data Factory datasets +# These represent data structures within data stores +# Requires: +# - name: dataset identifier +# - data_factory: reference to parent Data Factory +# - type: dataset type +# - linked_service: reference to associated linked service +# Optional: +# - description, annotations, folder, parameters +# - additional_properties for extended configuration +# - schema_json for data structure definition +variable "datasets" { + description = "List of datasets to be created for Data Factory" + type = map(object({ + name = string + data_factory = string + type = string + linked_service = string + type_properties_json = string + description = optional(string) + annotations = optional(list(string)) + folder = optional(string) + parameters = optional(map(string)) + additional_properties = optional(map(string)) + schema_json = optional(string) + })) +} \ No newline at end of file diff --git a/src/_modules/test_users/DataFactory/environments/dev/backend.tf b/src/_modules/test_users/DataFactory/environments/dev/backend.tf deleted file mode 100644 index 64222a48a..000000000 --- a/src/_modules/test_users/DataFactory/environments/dev/backend.tf +++ /dev/null @@ -1,8 +0,0 @@ -terraform { - backend "azurerm" { - resource_group_name = "dev-fasanorg" - storage_account_name = "stbipdevtest" - container_name = "bc-tf-bip-dev-test" - key = "terraform.tfstate" - } -} diff --git a/src/_modules/test_users/DataFactory/environments/dev/main.tf b/src/_modules/test_users/DataFactory/environments/dev/main.tf deleted file mode 100644 index 245649cba..000000000 --- a/src/_modules/test_users/DataFactory/environments/dev/main.tf +++ /dev/null @@ -1,21 +0,0 @@ -# Main.tf per l'ambiente DEV -provider "azurerm" { - features {} - subscription_id = var.subscription_id -} - -data "azurerm_client_config" "current" {} - -module "data_factory" { - source = "../../modules/data_factory" - - data_factories = var.data_factories - pipelines = var.pipelines - azure_runtimes = var.azure_runtimes - self_hosted_runtimes = var.self_hosted_runtimes - linked_services = var.linked_services - datasets = var.datasets - storage_account_resource_groups = var.storage_account_resource_groups # Aggiungi questa riga -} - - diff --git a/src/_modules/test_users/DataFactory/environments/dev/outputs.tf b/src/_modules/test_users/DataFactory/environments/dev/outputs.tf deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/_modules/test_users/DataFactory/environments/dev/terraform.tfvars b/src/_modules/test_users/DataFactory/environments/dev/terraform.tfvars deleted file mode 100644 index 4c5e9b725..000000000 --- a/src/_modules/test_users/DataFactory/environments/dev/terraform.tfvars +++ /dev/null @@ -1,165 +0,0 @@ -data_factories = { - dev_adf = { - name = "adfdevbiptest1" - location = "Italy North" - resource_group_name = "dev-fasanorg" - tags = { - Provider = "BIP" - Environment = "DEV" - Application = "Test" - Owners = "Cristian Felice Cuffari, Domenico Fasano" - } - } -} - -azure_runtimes = { - dev_runtime = { - name = "runtime-dev" - location = "West Europe" - data_factory = "dev_adf" - } -} - -self_hosted_runtimes = { - dev_self_hosted = { - name = "self-hosted-dev" - data_factory = "dev_adf" - } -} - -linked_services = { - dev_blob_service = { - name = "blob-storage-dev" - data_factory = "dev_adf" - type = "AzureBlobStorage" - type_properties_json = < details.storage_account if details.storage_account != "" } - name = each.value - resource_group_name = var.storage_account_resource_groups[each.key] -} - -# Linked service per Azure Data Factory -resource "azurerm_data_factory_linked_custom_service" "this" { - for_each = var.linked_services - name = each.value.name - data_factory_id = azurerm_data_factory.this[each.value.data_factory].id - type = each.value.type - - type_properties_json = < account.primary_connection_string } -} - diff --git a/src/_modules/test_users/DataFactory/modules/data_factory/variables.tf b/src/_modules/test_users/DataFactory/modules/data_factory/variables.tf deleted file mode 100644 index 62f25905e..000000000 --- a/src/_modules/test_users/DataFactory/modules/data_factory/variables.tf +++ /dev/null @@ -1,73 +0,0 @@ -variable "data_factories" { - description = "Elenco delle Data Factory da creare" - type = map(object({ - name = string - location = string - resource_group_name = string - tags = map(string) - })) -} - -variable "pipelines" { - type = map(object({ - name = string - data_factory = string - variables = map(string) # Per le variabili della pipeline - activities_json = string # Il contenuto JSON dell'attività - })) -} - - - - -variable "azure_runtimes" { - description = "Elenco degli Azure Integration Runtimes per Data Factory" - type = map(object({ - name = string - location = string - data_factory = string - })) -} - -variable "self_hosted_runtimes" { - description = "Elenco dei Self-Hosted Integration Runtimes per Data Factory" - type = map(object({ - name = string - data_factory = string - })) -} - -variable "linked_services" { - description = "Elenco dei linked services da creare per Data Factory" - type = map(object({ - name = string - data_factory = string - type = string - type_properties_json = string - storage_account = string - })) -} - -variable "storage_account_resource_groups" { - description = "Resource groups for each storage account used in linked services" - type = map(string) -} - - -variable "datasets" { - description = "Elenco dei dataset da creare per Data Factory" - type = map(object({ - name = string - data_factory = string - type = string - linked_service = string - type_properties_json = string - description = optional(string) - annotations = optional(list(string)) - folder = optional(string) - parameters = optional(map(string)) - additional_properties = optional(map(string)) - schema_json = optional(string) - })) -} - From 5bb58096f98e98b04586c1a76b23072f9a3c0154 Mon Sep 17 00:00:00 2001 From: ccuffari <75679158+ccuffari@users.noreply.github.com> Date: Tue, 15 Oct 2024 11:48:14 +0200 Subject: [PATCH 03/22] Update italynorth.tf --- src/_modules/data_factory/main.tf | 98 --------------- src/_modules/data_factory/outputs.tf | 10 -- src/_modules/data_factory/variables.tf | 113 ----------------- .../modules/data_factory/azure_runtimes.tf | 6 + .../modules/data_factory/data_factory.tf | 11 ++ .../data_factory/datasets_containers.tf | 23 ++++ .../modules/data_factory/datasets_tables.tf | 14 +++ .../linked_services_containers.tf | 9 ++ .../data_factory/linked_services_tables.tf | 7 ++ .../examples/modules/data_factory/locals.tf | 17 +++ .../examples/modules/data_factory/outputs.tf | 0 .../data_factory/pipeline_containers.tf | 65 ++++++++++ .../modules/data_factory/pipeline_tables.tf | 53 ++++++++ .../data_factory/self_hosted_runtime.tf | 5 + .../modules/data_factory/variables.tf | 100 +++++++++++++++ .../examples/prod/data.tf | 0 .../examples/prod/italynorth.tf | 115 ++++++++++++++++++ .../examples/prod/locals.tf | 14 +++ .../examples/prod/main.tf | 26 ++++ .../examples/prod/outputs.tf | 0 .../examples/prod/variables.tf | 9 ++ 21 files changed, 474 insertions(+), 221 deletions(-) delete mode 100644 src/_modules/data_factory/main.tf delete mode 100644 src/_modules/data_factory/outputs.tf delete mode 100644 src/_modules/data_factory/variables.tf create mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/azure_runtimes.tf create mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/data_factory.tf create mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/datasets_containers.tf create mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/datasets_tables.tf create mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/linked_services_containers.tf create mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/linked_services_tables.tf create mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/locals.tf create mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/outputs.tf create mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/pipeline_containers.tf create mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/pipeline_tables.tf create mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/self_hosted_runtime.tf create mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/variables.tf create mode 100644 src/_modules/data_factory_storage_account/examples/prod/data.tf create mode 100644 src/_modules/data_factory_storage_account/examples/prod/italynorth.tf create mode 100644 src/_modules/data_factory_storage_account/examples/prod/locals.tf create mode 100644 src/_modules/data_factory_storage_account/examples/prod/main.tf create mode 100644 src/_modules/data_factory_storage_account/examples/prod/outputs.tf create mode 100644 src/_modules/data_factory_storage_account/examples/prod/variables.tf diff --git a/src/_modules/data_factory/main.tf b/src/_modules/data_factory/main.tf deleted file mode 100644 index 652fc907b..000000000 --- a/src/_modules/data_factory/main.tf +++ /dev/null @@ -1,98 +0,0 @@ -# Create Azure Data Factory instances -# Uses for_each to create multiple factories based on input variables -# Enables system-assigned managed identity for secure access to resources -resource "azurerm_data_factory" "this" { - for_each = var.data_factories - name = "adf${each.value.name}" - location = each.value.location - resource_group_name = each.value.resource_group_name - tags = each.value.tags - - identity { - type = "SystemAssigned" - } -} - -# Define Data Factory pipelines -# Creates pipelines with custom variables and activities -# Dependencies ensure datasets are created before pipelines -resource "azurerm_data_factory_pipeline" "this" { - for_each = var.pipelines - name = each.value.name - data_factory_id = azurerm_data_factory.this[each.value.data_factory].id - variables = each.value.variables - activities_json = each.value.activities_json - - depends_on = [ - azurerm_data_factory_custom_dataset.this - ] -} - -# Create Azure Integration Runtime -# Managed compute infrastructure for data movement and transformation -# Located in specific Azure regions -resource "azurerm_data_factory_integration_runtime_azure" "azure_runtime" { - for_each = var.azure_runtimes - name = "azure-runtime-${each.value.name}" - data_factory_id = azurerm_data_factory.this[each.value.data_factory].id - location = each.value.location -} - -# Create Self-hosted Integration Runtime -# Enables data movement and transformation in private networks -# Requires manual installation of runtime on premises -resource "azurerm_data_factory_integration_runtime_self_hosted" "self_hosted_runtime" { - for_each = var.self_hosted_runtimes - name = "self-hosted-runtime-${each.value.name}" - data_factory_id = azurerm_data_factory.this[each.value.data_factory].id -} - -# Retrieve existing Storage Account information -# Uses data source to get connection details for linked services -# Filters storage accounts based on provided configuration -data "azurerm_storage_account" "existing" { - for_each = { - for linked_service, details in var.linked_services : linked_service => details.storage_account - if details.storage_account != "" - } - name = each.value - resource_group_name = var.storage_account_resource_groups[each.key] -} - -# Create Linked Services for Data Factory -# Establishes connections to external data sources -# Uses storage account connection strings for authentication -resource "azurerm_data_factory_linked_custom_service" "this" { - for_each = var.linked_services - name = each.value.name - data_factory_id = azurerm_data_factory.this[each.value.data_factory].id - type = each.value.type - type_properties_json = < account.primary_connection_string } -} \ No newline at end of file diff --git a/src/_modules/data_factory/variables.tf b/src/_modules/data_factory/variables.tf deleted file mode 100644 index 7ee50706c..000000000 --- a/src/_modules/data_factory/variables.tf +++ /dev/null @@ -1,113 +0,0 @@ -# Variable to define Azure Data Factories to be created -# Requires: -# - name: unique identifier for the Data Factory -# - location: Azure region where the Data Factory will be deployed -# - resource_group_name: target resource group -# - tags: map of tags for resource organization -variable "data_factories" { - description = "List of Data Factories to be created" - type = map(object({ - name = string - location = string - resource_group_name = string - tags = map(string) - })) -} - -# Variable to define Data Factory pipelines -# Requires: -# - name: pipeline identifier -# - data_factory: reference to parent Data Factory -# - variables: map of pipeline variables -# - activities_json: JSON definition of pipeline activities -variable "pipelines" { - description = "List of pipelines to be created in Data Factory" - type = map(object({ - name = string - data_factory = string - variables = map(string) # Pipeline variables - activities_json = string # Activities JSON content - })) -} - -# Variable to define Azure Integration Runtimes -# These are managed compute resources for data movement -# Requires: -# - name: runtime identifier -# - location: Azure region for runtime deployment -# - data_factory: reference to parent Data Factory -variable "azure_runtimes" { - description = "List of Azure Integration Runtimes for Data Factory" - type = map(object({ - name = string - location = string - data_factory = string - })) -} - -# Variable to define Self-Hosted Integration Runtimes -# These are on-premises or private network runtimes -# Requires: -# - name: runtime identifier -# - data_factory: reference to parent Data Factory -variable "self_hosted_runtimes" { - description = "List of Self-Hosted Integration Runtimes for Data Factory" - type = map(object({ - name = string - data_factory = string - })) -} - -# Variable to define Linked Services -# These establish connections to external data sources -# Requires: -# - name: service identifier -# - data_factory: reference to parent Data Factory -# - type: service type (e.g., AzureBlobStorage) -# - type_properties_json: connection properties in JSON format -# - storage_account: reference to associated storage account -variable "linked_services" { - description = "List of linked services to be created for Data Factory" - type = map(object({ - name = string - data_factory = string - type = string - type_properties_json = string - storage_account = string - })) -} - -# Variable to map storage accounts to their resource groups -# Used to lookup existing storage accounts for linked services -variable "storage_account_resource_groups" { - description = "Resource groups for each storage account used in linked services" - type = map(string) -} - -# Variable to define Data Factory datasets -# These represent data structures within data stores -# Requires: -# - name: dataset identifier -# - data_factory: reference to parent Data Factory -# - type: dataset type -# - linked_service: reference to associated linked service -# Optional: -# - description, annotations, folder, parameters -# - additional_properties for extended configuration -# - schema_json for data structure definition -variable "datasets" { - description = "List of datasets to be created for Data Factory" - type = map(object({ - name = string - data_factory = string - type = string - linked_service = string - type_properties_json = string - description = optional(string) - annotations = optional(list(string)) - folder = optional(string) - parameters = optional(map(string)) - additional_properties = optional(map(string)) - schema_json = optional(string) - })) -} \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/examples/modules/data_factory/azure_runtimes.tf b/src/_modules/data_factory_storage_account/examples/modules/data_factory/azure_runtimes.tf new file mode 100644 index 000000000..92ccd69d0 --- /dev/null +++ b/src/_modules/data_factory_storage_account/examples/modules/data_factory/azure_runtimes.tf @@ -0,0 +1,6 @@ +resource "azurerm_data_factory_integration_runtime_azure" "azure_runtime" { + for_each = var.azure_runtimes + name = "${local.prefix}-${local.env_short}-${local.region}-${local.domain}-${local.appname}-${local.azure_data_factory_azure_runtime}-${each.value.name}" + data_factory_id = azurerm_data_factory.data_factory[each.value.data_factory].id + location = each.value.location +} diff --git a/src/_modules/data_factory_storage_account/examples/modules/data_factory/data_factory.tf b/src/_modules/data_factory_storage_account/examples/modules/data_factory/data_factory.tf new file mode 100644 index 000000000..18e03674f --- /dev/null +++ b/src/_modules/data_factory_storage_account/examples/modules/data_factory/data_factory.tf @@ -0,0 +1,11 @@ +resource "azurerm_data_factory" "data_factory" { + for_each = var.data_factories + name = "${local.prefix}-${local.env_short}-${local.region}-${local.domain}-${local.appname}-${local.azure_data_factory}-${each.value.name}" + location = each.value.location + resource_group_name = each.value.resource_group_name + tags = each.value.tags + + identity { + type = "SystemAssigned" + } +} \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/examples/modules/data_factory/datasets_containers.tf b/src/_modules/data_factory_storage_account/examples/modules/data_factory/datasets_containers.tf new file mode 100644 index 000000000..9c1ed8254 --- /dev/null +++ b/src/_modules/data_factory_storage_account/examples/modules/data_factory/datasets_containers.tf @@ -0,0 +1,23 @@ +resource "azurerm_data_factory_custom_dataset" "dataset_container" { + for_each = var.datasets + name = "${local.prefix}-${local.env_short}-${local.region}-${local.domain}-${local.appname}-${local.azure_data_factory_dataset_container}-${each.value.name}" + data_factory_id = azurerm_data_factory.data_factory[each.value.data_factory].id + type = each.value.type + + linked_service { + name = azurerm_data_factory_linked_service_azure_blob_storage.linked_service_container[each.value.linked_service].name + parameters = each.value.parameters + } + + type_properties_json = < Date: Tue, 15 Oct 2024 19:11:26 +0200 Subject: [PATCH 04/22] feat: started refactoring --- .../data_factory_storage_account/data.tf | 9 ++ .../datasets_containers.tf | 21 ++++ .../datasets_tables.tf | 14 +++ .../modules/data_factory/azure_runtimes.tf | 6 - .../modules/data_factory/data_factory.tf | 11 -- .../data_factory/datasets_containers.tf | 23 ---- .../modules/data_factory/datasets_tables.tf | 14 --- .../linked_services_containers.tf | 9 -- .../data_factory/linked_services_tables.tf | 7 -- .../examples/modules/data_factory/locals.tf | 17 --- .../data_factory/pipeline_containers.tf | 65 ---------- .../modules/data_factory/pipeline_tables.tf | 53 -------- .../data_factory/self_hosted_runtime.tf | 5 - .../modules/data_factory/variables.tf | 100 --------------- .../examples/prod/italynorth.tf | 115 ------------------ .../examples/prod/locals.tf | 14 --- .../examples/prod/main.tf | 26 ---- .../examples/prod/variables.tf | 9 -- .../linked_services_containers.tf | 9 ++ .../linked_services_tables.tf | 7 ++ .../data_factory_storage_account/locals.tf | 3 + .../data_factory_storage_account/main.tf | 12 ++ .../modules/data_factory => }/outputs.tf | 0 .../pipeline_containers.tf | 65 ++++++++++ .../pipeline_tables.tf | 54 ++++++++ .../data_factory_storage_account/variables.tf | 32 +++++ src/migration/_modules/datafactory/main.tf | 19 +++ .../_modules/datafactory/variables.tf | 19 +++ .../examples => migration}/prod/data.tf | 0 src/migration/prod/italynorth.tf | 9 ++ src/migration/prod/locals.tf | 17 +++ src/migration/prod/main.tf | 20 +++ .../examples => migration}/prod/outputs.tf | 0 33 files changed, 310 insertions(+), 474 deletions(-) create mode 100644 src/_modules/data_factory_storage_account/data.tf create mode 100644 src/_modules/data_factory_storage_account/datasets_containers.tf create mode 100644 src/_modules/data_factory_storage_account/datasets_tables.tf delete mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/azure_runtimes.tf delete mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/data_factory.tf delete mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/datasets_containers.tf delete mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/datasets_tables.tf delete mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/linked_services_containers.tf delete mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/linked_services_tables.tf delete mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/locals.tf delete mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/pipeline_containers.tf delete mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/pipeline_tables.tf delete mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/self_hosted_runtime.tf delete mode 100644 src/_modules/data_factory_storage_account/examples/modules/data_factory/variables.tf delete mode 100644 src/_modules/data_factory_storage_account/examples/prod/italynorth.tf delete mode 100644 src/_modules/data_factory_storage_account/examples/prod/locals.tf delete mode 100644 src/_modules/data_factory_storage_account/examples/prod/main.tf delete mode 100644 src/_modules/data_factory_storage_account/examples/prod/variables.tf create mode 100644 src/_modules/data_factory_storage_account/linked_services_containers.tf create mode 100644 src/_modules/data_factory_storage_account/linked_services_tables.tf create mode 100644 src/_modules/data_factory_storage_account/locals.tf create mode 100644 src/_modules/data_factory_storage_account/main.tf rename src/_modules/data_factory_storage_account/{examples/modules/data_factory => }/outputs.tf (100%) create mode 100644 src/_modules/data_factory_storage_account/pipeline_containers.tf create mode 100644 src/_modules/data_factory_storage_account/pipeline_tables.tf create mode 100644 src/_modules/data_factory_storage_account/variables.tf create mode 100644 src/migration/_modules/datafactory/main.tf create mode 100644 src/migration/_modules/datafactory/variables.tf rename src/{_modules/data_factory_storage_account/examples => migration}/prod/data.tf (100%) create mode 100644 src/migration/prod/italynorth.tf create mode 100644 src/migration/prod/locals.tf create mode 100644 src/migration/prod/main.tf rename src/{_modules/data_factory_storage_account/examples => migration}/prod/outputs.tf (100%) diff --git a/src/_modules/data_factory_storage_account/data.tf b/src/_modules/data_factory_storage_account/data.tf new file mode 100644 index 000000000..c6f9c99eb --- /dev/null +++ b/src/_modules/data_factory_storage_account/data.tf @@ -0,0 +1,9 @@ +data "azurerm_storage_containers" "this" { + for_each = length(var.containers) == 0 ? [1] : [] + storage_account_id = var.source_storage_account.id +} + +data "azurerm_storage_account" "source" { + name = var.source_storage_account.name + resource_group_name = var.source_storage_account.resource_group_name +} \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/datasets_containers.tf b/src/_modules/data_factory_storage_account/datasets_containers.tf new file mode 100644 index 000000000..869b08bf9 --- /dev/null +++ b/src/_modules/data_factory_storage_account/datasets_containers.tf @@ -0,0 +1,21 @@ +resource "azurerm_data_factory_custom_dataset" "dataset_container" { + for_each = local.containers + name = "${module.naming_convention.prefix}-adf-${each.value.name}-blob-${module.naming_convention.suffix}" + data_factory_id = var.data_factory.id + type = "AzureBlob" + + linked_service { + name = azurerm_data_factory_linked_service_azure_blob_storage.linked_service_container[each.key].name + # parameters = each.value.parameters + } + + type_properties_json = jsonencode({ + linkedServiceName = { + referenceName = azurerm_data_factory_linked_service_azure_blob_storage.linked_service_container[each.key].name + type = "LinkedServiceReference" + } + type = "AzureBlob" + fileName = each.value.file_name + folderPath = each.value.folder_path + }) +} \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/datasets_tables.tf b/src/_modules/data_factory_storage_account/datasets_tables.tf new file mode 100644 index 000000000..72809d1f8 --- /dev/null +++ b/src/_modules/data_factory_storage_account/datasets_tables.tf @@ -0,0 +1,14 @@ +resource "azurerm_data_factory_custom_dataset" "dataset_table" { + for_each = local.tables + name = "${module.naming_convention.prefix}-adf-${each.value.name}-table-${module.naming_convention.suffix}" + data_factory_id = var.data_factory.id + type = "AzureTable" + + linked_service { + name = azurerm_data_factory_linked_service_azure_blob_storage.linked_service_container[each.key].name + } + + type_properties_json = jsonencode({ + tableName = each.value.name + }) +} \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/examples/modules/data_factory/azure_runtimes.tf b/src/_modules/data_factory_storage_account/examples/modules/data_factory/azure_runtimes.tf deleted file mode 100644 index 92ccd69d0..000000000 --- a/src/_modules/data_factory_storage_account/examples/modules/data_factory/azure_runtimes.tf +++ /dev/null @@ -1,6 +0,0 @@ -resource "azurerm_data_factory_integration_runtime_azure" "azure_runtime" { - for_each = var.azure_runtimes - name = "${local.prefix}-${local.env_short}-${local.region}-${local.domain}-${local.appname}-${local.azure_data_factory_azure_runtime}-${each.value.name}" - data_factory_id = azurerm_data_factory.data_factory[each.value.data_factory].id - location = each.value.location -} diff --git a/src/_modules/data_factory_storage_account/examples/modules/data_factory/data_factory.tf b/src/_modules/data_factory_storage_account/examples/modules/data_factory/data_factory.tf deleted file mode 100644 index 18e03674f..000000000 --- a/src/_modules/data_factory_storage_account/examples/modules/data_factory/data_factory.tf +++ /dev/null @@ -1,11 +0,0 @@ -resource "azurerm_data_factory" "data_factory" { - for_each = var.data_factories - name = "${local.prefix}-${local.env_short}-${local.region}-${local.domain}-${local.appname}-${local.azure_data_factory}-${each.value.name}" - location = each.value.location - resource_group_name = each.value.resource_group_name - tags = each.value.tags - - identity { - type = "SystemAssigned" - } -} \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/examples/modules/data_factory/datasets_containers.tf b/src/_modules/data_factory_storage_account/examples/modules/data_factory/datasets_containers.tf deleted file mode 100644 index 9c1ed8254..000000000 --- a/src/_modules/data_factory_storage_account/examples/modules/data_factory/datasets_containers.tf +++ /dev/null @@ -1,23 +0,0 @@ -resource "azurerm_data_factory_custom_dataset" "dataset_container" { - for_each = var.datasets - name = "${local.prefix}-${local.env_short}-${local.region}-${local.domain}-${local.appname}-${local.azure_data_factory_dataset_container}-${each.value.name}" - data_factory_id = azurerm_data_factory.data_factory[each.value.data_factory].id - type = each.value.type - - linked_service { - name = azurerm_data_factory_linked_service_azure_blob_storage.linked_service_container[each.value.linked_service].name - parameters = each.value.parameters - } - - type_properties_json = < 0 ? var.containers : [for container in data.azurerm_storage_containers.this[0].containers : container.name] +} \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/main.tf b/src/_modules/data_factory_storage_account/main.tf new file mode 100644 index 000000000..13dbda61f --- /dev/null +++ b/src/_modules/data_factory_storage_account/main.tf @@ -0,0 +1,12 @@ +module "naming_convention" { + source = "github.com/pagopa/dx//infra/modules/azure_naming_convention/?ref=main" + + environment = { + prefix = var.environment.prefix + env_short = var.environment.env_short + location = var.environment.location + domain = var.environment.domain + app_name = var.environment.app_name + instance_number = var.environment.instance_number + } +} diff --git a/src/_modules/data_factory_storage_account/examples/modules/data_factory/outputs.tf b/src/_modules/data_factory_storage_account/outputs.tf similarity index 100% rename from src/_modules/data_factory_storage_account/examples/modules/data_factory/outputs.tf rename to src/_modules/data_factory_storage_account/outputs.tf diff --git a/src/_modules/data_factory_storage_account/pipeline_containers.tf b/src/_modules/data_factory_storage_account/pipeline_containers.tf new file mode 100644 index 000000000..ac6e96c01 --- /dev/null +++ b/src/_modules/data_factory_storage_account/pipeline_containers.tf @@ -0,0 +1,65 @@ +resource "azurerm_data_factory_pipeline" "pipeline_container" { + for_each = local.containers + name = "${module.naming_convention.prefix}-adf-${each.value.name}-blob-${module.naming_convention.suffix}" + data_factory_id = var.data_factory.id + + variables = each.value.variables + + depends_on = [ + azurerm_data_factory_custom_dataset.dataset_container + ] + + activities_json = jsonencode( + [ + { + name = "CopyActivity" + type = "Copy" + dependsOn = [] + policy = { + timeout = "0.12:00:00" + retry = 0 + retryIntervalInSeconds = 30 + secureOutput = false + secureInput = false + } + userProperties = [] + typeProperties = { + source = { + type = "JsonSource" + storeSettings = { + type = "AzureBlobStorageReadSettings" + recursive = true + enablePartitionDiscovery = false + wildcardFileName = each.value.wildcard_file_name + } + formatSettings = { + type = "JsonReadSettings" + } + } + sink = { + type = "JsonSink" + storeSettings = { + type = "AzureBlobStorageWriteSettings" + } + formatSettings = { + type = "JsonWriteSettings" + } + } + enableStaging = false + } + inputs = [ + { + referenceName = each.value.input_dataset + type = "DatasetReference" + } + ] + outputs = [ + { + referenceName = each.value.output_dataset + type = "DatasetReference" + } + ] + } + ] + ) +} \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/pipeline_tables.tf b/src/_modules/data_factory_storage_account/pipeline_tables.tf new file mode 100644 index 000000000..83d03f5c6 --- /dev/null +++ b/src/_modules/data_factory_storage_account/pipeline_tables.tf @@ -0,0 +1,54 @@ +resource "azurerm_data_factory_pipeline" "pipeline_table" { + for_each = local.tables + name = "${module.naming_convention.prefix}-adf-${each.value.name}-table-${module.naming_convention.suffix}" + data_factory_id = var.data_factory.id + + variables = each.value.variables + + depends_on = [ + azurerm_data_factory_custom_dataset.dataset_table + ] + + activities_json = jsonencode( + [ + { + name = "CopyActivity" + type = "Copy" + dependsOn = [] + policy = { + timeout = "0.12:00:00" + retry = 0 + retryIntervalInSeconds = 30 + secureOutput = false + secureInput = false + } + userProperties = [] + typeProperties = { + source = { + type = "AzureTableSource" + azureTableSourceIgnoreTableNotFound = false + } + sink = { + type = "AzureTableSink" + writeBatchSize = 10000 + writeBatchTimeout = "00:00:30" + } + enableStaging = false + } + inputs = [ + { + referenceName = each.value.input_dataset + type = "DatasetReference" + } + ] + outputs = [ + { + referenceName = each.value.output_dataset + type = "DatasetReference" + } + ] + } + ] + ) + +} \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/variables.tf b/src/_modules/data_factory_storage_account/variables.tf new file mode 100644 index 000000000..fe831785a --- /dev/null +++ b/src/_modules/data_factory_storage_account/variables.tf @@ -0,0 +1,32 @@ +variable "environment" { + type = object({ + prefix = string + env_short = string + location = string + domain = optional(string) + app_name = string + instance_number = string + }) + + description = "Values which are used to generate resource names and location short names. They are all mandatory except for domain, which should not be used only in the case of a resource used by multiple domains." +} + +variable "data_factory" { + description = "Data Factory information." + type = map(object({ + id = string + name = string + location = string + resource_group_name = string + })) +} + +variable "containers" { + type = list(object({ + name = string + storage_account_name = string + container_name = string + })) + + description = "List of containers to migrate." +} \ No newline at end of file diff --git a/src/migration/_modules/datafactory/main.tf b/src/migration/_modules/datafactory/main.tf new file mode 100644 index 000000000..0d85af6a0 --- /dev/null +++ b/src/migration/_modules/datafactory/main.tf @@ -0,0 +1,19 @@ +# Create Azure Data Factory instances +# Enables system-assigned managed identity for secure access to resources +resource "azurerm_data_factory" "this" { + name = "${var.project}-migration-adf-01" + location = var.location + resource_group_name = var.resource_group_name + + identity { + type = "SystemAssigned" + } + + tags = var.tags +} + +resource "azurerm_data_factory_integration_runtime_azure" "azure_runtime" { + name = "${var.project}-adfir-${module.naming_convention.suffix}" + data_factory_id = azurerm_data_factory.this.id + location = var.location +} diff --git a/src/migration/_modules/datafactory/variables.tf b/src/migration/_modules/datafactory/variables.tf new file mode 100644 index 000000000..e1bab3ba3 --- /dev/null +++ b/src/migration/_modules/datafactory/variables.tf @@ -0,0 +1,19 @@ +variable "project" { + type = string + description = "IO prefix, short environment and short location" +} + +variable "location" { + type = string + description = "Azure region" +} + +variable "tags" { + type = map(any) + description = "Resource tags" +} + +variable "resource_group_name" { + type = string + description = "Resource group where create resources" +} diff --git a/src/_modules/data_factory_storage_account/examples/prod/data.tf b/src/migration/prod/data.tf similarity index 100% rename from src/_modules/data_factory_storage_account/examples/prod/data.tf rename to src/migration/prod/data.tf diff --git a/src/migration/prod/italynorth.tf b/src/migration/prod/italynorth.tf new file mode 100644 index 000000000..11753776b --- /dev/null +++ b/src/migration/prod/italynorth.tf @@ -0,0 +1,9 @@ +module "adf" { + source = "../_modules/datafactory" + + project = local.project_itn + location = "italynorth" + resource_group_name = azurerm_resource_group.github_runner.name + + tags = local.tags +} \ No newline at end of file diff --git a/src/migration/prod/locals.tf b/src/migration/prod/locals.tf new file mode 100644 index 000000000..f4d873141 --- /dev/null +++ b/src/migration/prod/locals.tf @@ -0,0 +1,17 @@ +locals { + prefix = "io" + env_short = "p" + location_short = { westeurope = "weu", italynorth = "itn", germanywestcentral = "gwc", northeurope = "neu" } + project_itn = "${local.prefix}-${local.env_short}-${local.location_short.italynorth}" + project_weu = "${local.prefix}-${local.env_short}-${local.location_short.westeurope}" + project_weu_legacy = "${local.prefix}-${local.env_short}" + secondary_project = "${local.prefix}-${local.env_short}-${local.location_short.germanywestcentral}" + + tags = { + CostCenter = "TS310 - PAGAMENTI & SERVIZI" + CreatedBy = "Terraform" + Environment = "Prod" + Owner = "IO" + Source = "https://github.com/pagopa/io-infra/blob/main/src/migration/prod" + } +} diff --git a/src/migration/prod/main.tf b/src/migration/prod/main.tf new file mode 100644 index 000000000..271442c57 --- /dev/null +++ b/src/migration/prod/main.tf @@ -0,0 +1,20 @@ +terraform { + + backend "azurerm" { + resource_group_name = "terraform-state-rg" + storage_account_name = "iopitntfst001" + container_name = "terraform-state" + key = "io-infra.migration.prod.italynorth.tfstate" + } + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "<= 3.112.0" + } + } +} + +provider "azurerm" { + features {} +} diff --git a/src/_modules/data_factory_storage_account/examples/prod/outputs.tf b/src/migration/prod/outputs.tf similarity index 100% rename from src/_modules/data_factory_storage_account/examples/prod/outputs.tf rename to src/migration/prod/outputs.tf From 4e11cf8852c5083a2b71a172e8a3485566a3dc17 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Tue, 15 Oct 2024 19:19:27 +0200 Subject: [PATCH 05/22] chore: what_to_migrate variable added --- .../linked_services_containers.tf | 2 +- .../linked_services_tables.tf | 2 +- .../data_factory_storage_account/locals.tf | 3 +- .../data_factory_storage_account/variables.tf | 39 +++++++++++++++---- 4 files changed, 35 insertions(+), 11 deletions(-) diff --git a/src/_modules/data_factory_storage_account/linked_services_containers.tf b/src/_modules/data_factory_storage_account/linked_services_containers.tf index e7f1b26f1..74acbac44 100644 --- a/src/_modules/data_factory_storage_account/linked_services_containers.tf +++ b/src/_modules/data_factory_storage_account/linked_services_containers.tf @@ -1,5 +1,5 @@ resource "azurerm_data_factory_linked_service_azure_blob_storage" "linked_service_container" { - for_each = var.source_types.blob ? [1] : [] + for_each = var.what_to_migrate.blob.enabled ? [1] : [] name = "${module.naming_convention.prefix}-adf-${var.source_storage_account.name}-st-${module.naming_convention.suffix}" data_factory_id = var.data_factory.id diff --git a/src/_modules/data_factory_storage_account/linked_services_tables.tf b/src/_modules/data_factory_storage_account/linked_services_tables.tf index 8000026ee..7c9b4ded8 100644 --- a/src/_modules/data_factory_storage_account/linked_services_tables.tf +++ b/src/_modules/data_factory_storage_account/linked_services_tables.tf @@ -1,5 +1,5 @@ resource "azurerm_data_factory_linked_service_azure_table_storage" "linked_service_table" { - for_each = var.source_types.table ? [1] : [] + for_each = var.what_to_migrate.table.enabled ? [1] : [] name = "${module.naming_convention.prefix}-adf-${var.source_storage_account.name}-st-${module.naming_convention.suffix}" data_factory_id = var.data_factory.id diff --git a/src/_modules/data_factory_storage_account/locals.tf b/src/_modules/data_factory_storage_account/locals.tf index e361fc923..f96fc704d 100644 --- a/src/_modules/data_factory_storage_account/locals.tf +++ b/src/_modules/data_factory_storage_account/locals.tf @@ -1,3 +1,4 @@ locals { - containers = length(var.containers) > 0 ? var.containers : [for container in data.azurerm_storage_containers.this[0].containers : container.name] + containers = length(var.what_to_migrate.blob.containers) > 0 ? var.what_to_migrate.blob.containers : [for container in data.azurerm_storage_containers.this[0].containers : container.name] + tables = var.what_to_migrate.table.tables } \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/variables.tf b/src/_modules/data_factory_storage_account/variables.tf index fe831785a..a1b76ed51 100644 --- a/src/_modules/data_factory_storage_account/variables.tf +++ b/src/_modules/data_factory_storage_account/variables.tf @@ -21,12 +21,35 @@ variable "data_factory" { })) } -variable "containers" { - type = list(object({ - name = string - storage_account_name = string - container_name = string - })) +variable "what_to_migrate" { + type = object({ + blob = optional(object( + { + enabled = bool + containers = optional(list(string), []) + }), + { enabled = false } + ) + table = optional(object( + { + enabled = bool + tables = list(string) + }), + { enabled = false } + ) + }) - description = "List of containers to migrate." -} \ No newline at end of file + # validate that at least one between blob and table is enabled + validation { + condition = anytrue([var.what_to_migrate.blob.enabled, var.what_to_migrate.table.enabled]) + error_message = "At least one between blob and table should be enabled." + } + + # validate that if table is enabled, at least one table is specified + validation { + condition = !(var.what_to_migrate.table.enabled && length(var.what_to_migrate.table.tables) == 0) + error_message = "If table is enabled, at least one table should be specified." + } + + description = "List of databases, file shares, containers and tables to migrate." +} From 7760c8febff4d1c3a45200a4be15bd7c80550859 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Wed, 16 Oct 2024 10:50:49 +0200 Subject: [PATCH 06/22] feat: added iam and fixed pipelines --- .../data_factory_storage_account/data.tf | 17 ++++++---- .../datasets_containers.tf | 8 ++--- .../data_factory_storage_account/iam.tf | 32 +++++++++++++++++++ .../linked_services_containers.tf | 16 ++++++++-- .../linked_services_tables.tf | 12 +++++-- .../data_factory_storage_account/outputs.tf | 10 ++++++ .../pipeline_containers.tf | 10 +++--- .../data_factory_storage_account/variables.tf | 14 ++++++++ 8 files changed, 97 insertions(+), 22 deletions(-) create mode 100644 src/_modules/data_factory_storage_account/iam.tf diff --git a/src/_modules/data_factory_storage_account/data.tf b/src/_modules/data_factory_storage_account/data.tf index c6f9c99eb..73a41e312 100644 --- a/src/_modules/data_factory_storage_account/data.tf +++ b/src/_modules/data_factory_storage_account/data.tf @@ -1,9 +1,14 @@ -data "azurerm_storage_containers" "this" { - for_each = length(var.containers) == 0 ? [1] : [] - storage_account_id = var.source_storage_account.id +data "azurerm_storage_account" "source" { + name = var.storage_accounts.source.name + resource_group_name = var.storage_accounts.source.resource_group_name } -data "azurerm_storage_account" "source" { - name = var.source_storage_account.name - resource_group_name = var.source_storage_account.resource_group_name +data "azurerm_storage_account" "target" { + name = var.storage_accounts.target.name + resource_group_name = var.storage_accounts.target.resource_group_name +} + +data "azurerm_storage_containers" "this" { + for_each = length(var.what_to_migrate.containers) == 0 ? [1] : [] + storage_account_id = data.azurerm_storage_account.source.id } \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/datasets_containers.tf b/src/_modules/data_factory_storage_account/datasets_containers.tf index 869b08bf9..2d5ab4aca 100644 --- a/src/_modules/data_factory_storage_account/datasets_containers.tf +++ b/src/_modules/data_factory_storage_account/datasets_containers.tf @@ -5,17 +5,15 @@ resource "azurerm_data_factory_custom_dataset" "dataset_container" { type = "AzureBlob" linked_service { - name = azurerm_data_factory_linked_service_azure_blob_storage.linked_service_container[each.key].name - # parameters = each.value.parameters + name = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_blob.name } type_properties_json = jsonencode({ linkedServiceName = { - referenceName = azurerm_data_factory_linked_service_azure_blob_storage.linked_service_container[each.key].name + referenceName = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_blob.name type = "LinkedServiceReference" } type = "AzureBlob" - fileName = each.value.file_name - folderPath = each.value.folder_path + folderPath = each.value.name }) } \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/iam.tf b/src/_modules/data_factory_storage_account/iam.tf new file mode 100644 index 000000000..001998cd9 --- /dev/null +++ b/src/_modules/data_factory_storage_account/iam.tf @@ -0,0 +1,32 @@ +module "roles" { + source = "github.com/pagopa/dx//infra/modules/azure_role_assignments?ref=main" + principal_id = var.function_app.user_func_02.principal_id + + storage_blob = var.what_to_migrate.blob.enabled ? [ + { + storage_account_name = var.storage_accounts.source.name + resource_group_name = var.storage_accounts.source.resource_group_name + role = "reader" + }, + { + storage_account_name = var.storage_accounts.target.name + resource_group_name = var.storage_accounts.target.resource_group_name + role = "writer" + } + ] : [] + + # ADF terraform resources still force to use connection strings for tables + # but it's possible to switch to managed identities from the portal + storage_table = var.what_to_migrate.table.enabled ? [ + { + storage_account_name = var.storage_accounts.source.name + resource_group_name = var.storage_accounts.source.resource_group_name + role = "reader" + }, + { + storage_account_name = var.storage_accounts.target.name + resource_group_name = var.storage_accounts.target.resource_group_name + role = "writer" + } + ] : [] +} \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/linked_services_containers.tf b/src/_modules/data_factory_storage_account/linked_services_containers.tf index 74acbac44..15cc89bc3 100644 --- a/src/_modules/data_factory_storage_account/linked_services_containers.tf +++ b/src/_modules/data_factory_storage_account/linked_services_containers.tf @@ -1,9 +1,19 @@ -resource "azurerm_data_factory_linked_service_azure_blob_storage" "linked_service_container" { +resource "azurerm_data_factory_linked_service_azure_blob_storage" "source_linked_service_blob" { for_each = var.what_to_migrate.blob.enabled ? [1] : [] - name = "${module.naming_convention.prefix}-adf-${var.source_storage_account.name}-st-${module.naming_convention.suffix}" + name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-blob-${module.naming_convention.suffix}" data_factory_id = var.data_factory.id - service_endpoint = "https://${var.source_storage_account.id}.blob.core.windows.net" + service_endpoint = "https://${data.azurerm_storage_account.source.id}.blob.core.windows.net" + + use_managed_identity = true +} + +resource "azurerm_data_factory_linked_service_azure_blob_storage" "target_linked_service_blob" { + for_each = var.what_to_migrate.blob.enabled ? [1] : [] + name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-blob-${module.naming_convention.suffix}" + data_factory_id = var.data_factory.id + + service_endpoint = "https://${data.azurerm_storage_account.target.id}.blob.core.windows.net" use_managed_identity = true } \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/linked_services_tables.tf b/src/_modules/data_factory_storage_account/linked_services_tables.tf index 7c9b4ded8..354693e56 100644 --- a/src/_modules/data_factory_storage_account/linked_services_tables.tf +++ b/src/_modules/data_factory_storage_account/linked_services_tables.tf @@ -1,7 +1,15 @@ -resource "azurerm_data_factory_linked_service_azure_table_storage" "linked_service_table" { +resource "azurerm_data_factory_linked_service_azure_table_storage" "source_linked_service_table" { for_each = var.what_to_migrate.table.enabled ? [1] : [] - name = "${module.naming_convention.prefix}-adf-${var.source_storage_account.name}-st-${module.naming_convention.suffix}" + name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-table-${module.naming_convention.suffix}" data_factory_id = var.data_factory.id connection_string = data.azurerm_storage_account.source.primary_table_endpoint +} + +resource "azurerm_data_factory_linked_service_azure_table_storage" "target_linked_service_table" { + for_each = var.what_to_migrate.table.enabled ? [1] : [] + name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-table-${module.naming_convention.suffix}" + data_factory_id = var.data_factory.id + + connection_string = data.azurerm_storage_account.target.primary_table_endpoint } \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/outputs.tf b/src/_modules/data_factory_storage_account/outputs.tf index e69de29bb..cc37a5ca2 100644 --- a/src/_modules/data_factory_storage_account/outputs.tf +++ b/src/_modules/data_factory_storage_account/outputs.tf @@ -0,0 +1,10 @@ +output "pipelines" { + value = { + for pipeline in concat(azurerm_data_factory_pipeline.pipeline_container, azurerm_data_factory_pipeline.pipeline_table) + : pipeline.name => { + id = pipeline.id + name = pipeline.name + url = "https://adf.azure.com/en/authoring/pipeline/${pipeline.name}?factory=${pipeline.data_factory_id}" + } + } +} \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/pipeline_containers.tf b/src/_modules/data_factory_storage_account/pipeline_containers.tf index ac6e96c01..9f0540b6f 100644 --- a/src/_modules/data_factory_storage_account/pipeline_containers.tf +++ b/src/_modules/data_factory_storage_account/pipeline_containers.tf @@ -3,8 +3,6 @@ resource "azurerm_data_factory_pipeline" "pipeline_container" { name = "${module.naming_convention.prefix}-adf-${each.value.name}-blob-${module.naming_convention.suffix}" data_factory_id = var.data_factory.id - variables = each.value.variables - depends_on = [ azurerm_data_factory_custom_dataset.dataset_container ] @@ -30,14 +28,14 @@ resource "azurerm_data_factory_pipeline" "pipeline_container" { type = "AzureBlobStorageReadSettings" recursive = true enablePartitionDiscovery = false - wildcardFileName = each.value.wildcard_file_name + wildcardFileName = "*" # Copy all files } formatSettings = { type = "JsonReadSettings" } } sink = { - type = "JsonSink" + type = "JsonSink" # Check for binary storeSettings = { type = "AzureBlobStorageWriteSettings" } @@ -49,13 +47,13 @@ resource "azurerm_data_factory_pipeline" "pipeline_container" { } inputs = [ { - referenceName = each.value.input_dataset + referenceName = azurerm_data_factory_custom_dataset.source_dataset_container type = "DatasetReference" } ] outputs = [ { - referenceName = each.value.output_dataset + referenceName = azurerm_data_factory_custom_dataset.target_dataset_container type = "DatasetReference" } ] diff --git a/src/_modules/data_factory_storage_account/variables.tf b/src/_modules/data_factory_storage_account/variables.tf index a1b76ed51..6d988c7b6 100644 --- a/src/_modules/data_factory_storage_account/variables.tf +++ b/src/_modules/data_factory_storage_account/variables.tf @@ -21,6 +21,20 @@ variable "data_factory" { })) } +variable "storage_accounts" { + type = object({ + source = object({ + name = string + resource_group_name = string + }) + + target = object({ + name = string + resource_group_name = string + }) + }) +} + variable "what_to_migrate" { type = object({ blob = optional(object( From f9aa68d61a042da82f69c10adc42b7c948c7f5e6 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Wed, 16 Oct 2024 11:04:43 +0200 Subject: [PATCH 07/22] feat: added missing datasets and checks --- .../data_factory_storage_account/data.tf | 2 +- .../datasets_containers.tf | 24 +++++++++++++++++-- .../datasets_tables.tf | 21 +++++++++++++--- .../data_factory_storage_account/locals.tf | 4 ++-- .../pipeline_containers.tf | 4 ++-- .../pipeline_tables.tf | 10 ++++---- 6 files changed, 49 insertions(+), 16 deletions(-) diff --git a/src/_modules/data_factory_storage_account/data.tf b/src/_modules/data_factory_storage_account/data.tf index 73a41e312..4c64f1b9a 100644 --- a/src/_modules/data_factory_storage_account/data.tf +++ b/src/_modules/data_factory_storage_account/data.tf @@ -9,6 +9,6 @@ data "azurerm_storage_account" "target" { } data "azurerm_storage_containers" "this" { - for_each = length(var.what_to_migrate.containers) == 0 ? [1] : [] + for_each = var.what_to_migrate.blob.enabled && length(var.what_to_migrate.blob.containers) == 0 ? [1] : [] storage_account_id = data.azurerm_storage_account.source.id } \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/datasets_containers.tf b/src/_modules/data_factory_storage_account/datasets_containers.tf index 2d5ab4aca..daa4c4d93 100644 --- a/src/_modules/data_factory_storage_account/datasets_containers.tf +++ b/src/_modules/data_factory_storage_account/datasets_containers.tf @@ -1,6 +1,6 @@ -resource "azurerm_data_factory_custom_dataset" "dataset_container" { +resource "azurerm_data_factory_custom_dataset" "source_dataset_container" { for_each = local.containers - name = "${module.naming_convention.prefix}-adf-${each.value.name}-blob-${module.naming_convention.suffix}" + name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value.name}-blob-${module.naming_convention.suffix}" data_factory_id = var.data_factory.id type = "AzureBlob" @@ -16,4 +16,24 @@ resource "azurerm_data_factory_custom_dataset" "dataset_container" { type = "AzureBlob" folderPath = each.value.name }) +} + +resource "azurerm_data_factory_custom_dataset" "target_dataset_container" { + for_each = local.containers + name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value.name}-blob-${module.naming_convention.suffix}" + data_factory_id = var.data_factory.id + type = "AzureBlob" + + linked_service { + name = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_blob.name + } + + type_properties_json = jsonencode({ + linkedServiceName = { + referenceName = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_blob.name + type = "LinkedServiceReference" + } + type = "AzureBlob" + folderPath = each.value.name + }) } \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/datasets_tables.tf b/src/_modules/data_factory_storage_account/datasets_tables.tf index 72809d1f8..ed10fc8a8 100644 --- a/src/_modules/data_factory_storage_account/datasets_tables.tf +++ b/src/_modules/data_factory_storage_account/datasets_tables.tf @@ -1,11 +1,26 @@ -resource "azurerm_data_factory_custom_dataset" "dataset_table" { +resource "azurerm_data_factory_custom_dataset" "source_dataset_table" { for_each = local.tables - name = "${module.naming_convention.prefix}-adf-${each.value.name}-table-${module.naming_convention.suffix}" + name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value.name}-table-${module.naming_convention.suffix}" data_factory_id = var.data_factory.id type = "AzureTable" linked_service { - name = azurerm_data_factory_linked_service_azure_blob_storage.linked_service_container[each.key].name + name = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_table.name + } + + type_properties_json = jsonencode({ + tableName = each.value.name + }) +} + +resource "azurerm_data_factory_custom_dataset" "target_dataset_table" { + for_each = local.tables + name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value.name}-table-${module.naming_convention.suffix}" + data_factory_id = var.data_factory.id + type = "AzureTable" + + linked_service { + name = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_table.name } type_properties_json = jsonencode({ diff --git a/src/_modules/data_factory_storage_account/locals.tf b/src/_modules/data_factory_storage_account/locals.tf index f96fc704d..181f0aa08 100644 --- a/src/_modules/data_factory_storage_account/locals.tf +++ b/src/_modules/data_factory_storage_account/locals.tf @@ -1,4 +1,4 @@ locals { - containers = length(var.what_to_migrate.blob.containers) > 0 ? var.what_to_migrate.blob.containers : [for container in data.azurerm_storage_containers.this[0].containers : container.name] - tables = var.what_to_migrate.table.tables + containers = var.what_to_migrate.blob.enabled ? length(var.what_to_migrate.blob.containers) > 0 ? var.what_to_migrate.blob.containers : [for container in data.azurerm_storage_containers.this[0].containers : container.name] : [] + tables = var.what_to_migrate.table.enabled ? var.what_to_migrate.table.tables : [] } \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/pipeline_containers.tf b/src/_modules/data_factory_storage_account/pipeline_containers.tf index 9f0540b6f..914485d66 100644 --- a/src/_modules/data_factory_storage_account/pipeline_containers.tf +++ b/src/_modules/data_factory_storage_account/pipeline_containers.tf @@ -1,6 +1,6 @@ resource "azurerm_data_factory_pipeline" "pipeline_container" { for_each = local.containers - name = "${module.naming_convention.prefix}-adf-${each.value.name}-blob-${module.naming_convention.suffix}" + name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value.name}-blob-${module.naming_convention.suffix}" data_factory_id = var.data_factory.id depends_on = [ @@ -35,7 +35,7 @@ resource "azurerm_data_factory_pipeline" "pipeline_container" { } } sink = { - type = "JsonSink" # Check for binary + type = "JsonSink" storeSettings = { type = "AzureBlobStorageWriteSettings" } diff --git a/src/_modules/data_factory_storage_account/pipeline_tables.tf b/src/_modules/data_factory_storage_account/pipeline_tables.tf index 83d03f5c6..5ea2a11c4 100644 --- a/src/_modules/data_factory_storage_account/pipeline_tables.tf +++ b/src/_modules/data_factory_storage_account/pipeline_tables.tf @@ -1,10 +1,8 @@ resource "azurerm_data_factory_pipeline" "pipeline_table" { for_each = local.tables - name = "${module.naming_convention.prefix}-adf-${each.value.name}-table-${module.naming_convention.suffix}" + name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value.name}-table-${module.naming_convention.suffix}" data_factory_id = var.data_factory.id - variables = each.value.variables - depends_on = [ azurerm_data_factory_custom_dataset.dataset_table ] @@ -31,19 +29,19 @@ resource "azurerm_data_factory_pipeline" "pipeline_table" { sink = { type = "AzureTableSink" writeBatchSize = 10000 - writeBatchTimeout = "00:00:30" + writeBatchTimeout = "00:02:00" } enableStaging = false } inputs = [ { - referenceName = each.value.input_dataset + referenceName = azurerm_data_factory_custom_dataset.source_dataset_table type = "DatasetReference" } ] outputs = [ { - referenceName = each.value.output_dataset + referenceName = azurerm_data_factory_custom_dataset.target_dataset_table type = "DatasetReference" } ] From 9d2c460d00a2e0ec9e7a4abe62cad0b88937012a Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Wed, 16 Oct 2024 11:28:23 +0200 Subject: [PATCH 08/22] fix: changed folder structure and ran pre-commit --- .../data_factory_storage_account/data.tf | 2 +- .../data_factory_storage_account/locals.tf | 2 +- .../data_factory_storage_account/outputs.tf | 14 ++--- .../data_factory_storage_account/variables.tf | 23 +++----- src/migration/_modules/datafactory/main.tf | 19 ------ .../_modules/datafactory/variables.tf | 19 ------ src/migration/prod/README.md | 37 ++++++++++++ src/migration/prod/italynorth.tf | 58 +++++++++++++++++-- 8 files changed, 109 insertions(+), 65 deletions(-) delete mode 100644 src/migration/_modules/datafactory/main.tf delete mode 100644 src/migration/_modules/datafactory/variables.tf create mode 100644 src/migration/prod/README.md diff --git a/src/_modules/data_factory_storage_account/data.tf b/src/_modules/data_factory_storage_account/data.tf index 4c64f1b9a..0de9bf352 100644 --- a/src/_modules/data_factory_storage_account/data.tf +++ b/src/_modules/data_factory_storage_account/data.tf @@ -9,6 +9,6 @@ data "azurerm_storage_account" "target" { } data "azurerm_storage_containers" "this" { - for_each = var.what_to_migrate.blob.enabled && length(var.what_to_migrate.blob.containers) == 0 ? [1] : [] + for_each = var.what_to_migrate.blob.enabled && length(var.what_to_migrate.blob.containers) == 0 ? [1] : [] storage_account_id = data.azurerm_storage_account.source.id } \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/locals.tf b/src/_modules/data_factory_storage_account/locals.tf index 181f0aa08..db6f13ac2 100644 --- a/src/_modules/data_factory_storage_account/locals.tf +++ b/src/_modules/data_factory_storage_account/locals.tf @@ -1,4 +1,4 @@ locals { containers = var.what_to_migrate.blob.enabled ? length(var.what_to_migrate.blob.containers) > 0 ? var.what_to_migrate.blob.containers : [for container in data.azurerm_storage_containers.this[0].containers : container.name] : [] - tables = var.what_to_migrate.table.enabled ? var.what_to_migrate.table.tables : [] + tables = var.what_to_migrate.table.enabled ? var.what_to_migrate.table.tables : [] } \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/outputs.tf b/src/_modules/data_factory_storage_account/outputs.tf index cc37a5ca2..22c1d86d8 100644 --- a/src/_modules/data_factory_storage_account/outputs.tf +++ b/src/_modules/data_factory_storage_account/outputs.tf @@ -1,10 +1,10 @@ output "pipelines" { - value = { - for pipeline in concat(azurerm_data_factory_pipeline.pipeline_container, azurerm_data_factory_pipeline.pipeline_table) - : pipeline.name => { - id = pipeline.id - name = pipeline.name - url = "https://adf.azure.com/en/authoring/pipeline/${pipeline.name}?factory=${pipeline.data_factory_id}" - } + value = { + for pipeline in concat(azurerm_data_factory_pipeline.pipeline_container, azurerm_data_factory_pipeline.pipeline_table) + : pipeline.name => { + id = pipeline.id + name = pipeline.name + url = "https://adf.azure.com/en/authoring/pipeline/${pipeline.name}?factory=${pipeline.data_factory_id}" } + } } \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/variables.tf b/src/_modules/data_factory_storage_account/variables.tf index 6d988c7b6..c6dca9a79 100644 --- a/src/_modules/data_factory_storage_account/variables.tf +++ b/src/_modules/data_factory_storage_account/variables.tf @@ -11,25 +11,20 @@ variable "environment" { description = "Values which are used to generate resource names and location short names. They are all mandatory except for domain, which should not be used only in the case of a resource used by multiple domains." } -variable "data_factory" { - description = "Data Factory information." - type = map(object({ - id = string - name = string - location = string - resource_group_name = string - })) +variable "data_factory_id" { + description = "Data Factory id where to create resources." + type = string } variable "storage_accounts" { type = object({ source = object({ - name = string + name = string resource_group_name = string }) target = object({ - name = string + name = string resource_group_name = string }) }) @@ -39,16 +34,16 @@ variable "what_to_migrate" { type = object({ blob = optional(object( { - enabled = bool + enabled = bool containers = optional(list(string), []) - }), + }), { enabled = false } ) table = optional(object( { enabled = bool - tables = list(string) - }), + tables = list(string) + }), { enabled = false } ) }) diff --git a/src/migration/_modules/datafactory/main.tf b/src/migration/_modules/datafactory/main.tf deleted file mode 100644 index 0d85af6a0..000000000 --- a/src/migration/_modules/datafactory/main.tf +++ /dev/null @@ -1,19 +0,0 @@ -# Create Azure Data Factory instances -# Enables system-assigned managed identity for secure access to resources -resource "azurerm_data_factory" "this" { - name = "${var.project}-migration-adf-01" - location = var.location - resource_group_name = var.resource_group_name - - identity { - type = "SystemAssigned" - } - - tags = var.tags -} - -resource "azurerm_data_factory_integration_runtime_azure" "azure_runtime" { - name = "${var.project}-adfir-${module.naming_convention.suffix}" - data_factory_id = azurerm_data_factory.this.id - location = var.location -} diff --git a/src/migration/_modules/datafactory/variables.tf b/src/migration/_modules/datafactory/variables.tf deleted file mode 100644 index e1bab3ba3..000000000 --- a/src/migration/_modules/datafactory/variables.tf +++ /dev/null @@ -1,19 +0,0 @@ -variable "project" { - type = string - description = "IO prefix, short environment and short location" -} - -variable "location" { - type = string - description = "Azure region" -} - -variable "tags" { - type = map(any) - description = "Resource tags" -} - -variable "resource_group_name" { - type = string - description = "Resource group where create resources" -} diff --git a/src/migration/prod/README.md b/src/migration/prod/README.md new file mode 100644 index 000000000..ec20557ab --- /dev/null +++ b/src/migration/prod/README.md @@ -0,0 +1,37 @@ +# prod + + +## Requirements + +| Name | Version | +|------|---------| +| [azurerm](#requirement\_azurerm) | <= 3.112.0 | + +## Providers + +| Name | Version | +|------|---------| +| [azurerm](#provider\_azurerm) | <= 3.112.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [migrate\_storage\_accounts](#module\_migrate\_storage\_accounts) | ../../_modules/data_factory_storage_account | n/a | + +## Resources + +| Name | Type | +|------|------| +| [azurerm_data_factory.this](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/data_factory) | resource | +| [azurerm_data_factory_integration_runtime_azure.azure_runtime](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/data_factory_integration_runtime_azure) | resource | +| [azurerm_resource_group.migration](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | + +## Inputs + +No inputs. + +## Outputs + +No outputs. + diff --git a/src/migration/prod/italynorth.tf b/src/migration/prod/italynorth.tf index 11753776b..bf8384b0e 100644 --- a/src/migration/prod/italynorth.tf +++ b/src/migration/prod/italynorth.tf @@ -1,9 +1,59 @@ -module "adf" { - source = "../_modules/datafactory" +resource "azurerm_resource_group" "migration" { + name = "${local.project_itn}-migration-rg-01" + location = "italynorth" - project = local.project_itn + tags = local.tags +} + +# Create Azure Data Factory instances +# Enables system-assigned managed identity for secure access to resources +resource "azurerm_data_factory" "this" { + name = "${local.project_itn}-migration-adf-01" location = "italynorth" - resource_group_name = azurerm_resource_group.github_runner.name + resource_group_name = azurerm_resource_group.migration.name + + identity { + type = "SystemAssigned" + } tags = local.tags +} + +resource "azurerm_data_factory_integration_runtime_azure" "azure_runtime" { + name = "${local.project_itn}-migration-adfir-01" + location = "italynorth" + data_factory_id = azurerm_data_factory.this.id +} + +module "migrate_storage_accounts" { + for_each = local.storage_accounts + source = "../../_modules/data_factory_storage_account" + + environment = { + prefix = local.prefix + env_short = local.env_short + location = "italynorth" + domain = "eng" + app_name = "mig" + instance_number = "01" + } + + data_factory_id = azurerm_data_factory.this.id + + storage_accounts = { + source = each.value.source + target = each.value.target + } + + what_to_migrate = { + blob = { + enabled = each.value.blob.enabled + containers = each.value.blob.containers + } + + table = { + enabled = each.value.table.enabled + tables = each.value.table.tables + } + } } \ No newline at end of file From 0c4c0a8226175b509932c5c96c8593d813f42910 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Wed, 16 Oct 2024 12:14:05 +0200 Subject: [PATCH 09/22] feat: added the example list --- .../data_factory_storage_account/data.tf | 8 +++- .../datasets_containers.tf | 12 +++--- .../datasets_tables.tf | 8 ++-- .../data_factory_storage_account/iam.tf | 2 +- .../linked_services_containers.tf | 8 ++-- .../linked_services_tables.tf | 8 ++-- .../data_factory_storage_account/locals.tf | 4 +- .../data_factory_storage_account/main.tf | 9 ++++ .../data_factory_storage_account/network.tf | 27 ++++++++++++ .../pipeline_containers.tf | 6 +-- .../pipeline_tables.tf | 6 +-- .../data_factory_storage_account/variables.tf | 15 ++++--- src/migration/prod/.terraform.lock.hcl | 42 +++++++++++++++++++ src/migration/prod/README.md | 3 +- src/migration/prod/italynorth.tf | 24 ++++------- src/migration/prod/locals.tf | 18 ++++++++ src/migration/prod/main.tf | 7 ++++ 17 files changed, 151 insertions(+), 56 deletions(-) create mode 100644 src/_modules/data_factory_storage_account/network.tf create mode 100644 src/migration/prod/.terraform.lock.hcl diff --git a/src/_modules/data_factory_storage_account/data.tf b/src/_modules/data_factory_storage_account/data.tf index 0de9bf352..fb8fda7eb 100644 --- a/src/_modules/data_factory_storage_account/data.tf +++ b/src/_modules/data_factory_storage_account/data.tf @@ -9,6 +9,12 @@ data "azurerm_storage_account" "target" { } data "azurerm_storage_containers" "this" { - for_each = var.what_to_migrate.blob.enabled && length(var.what_to_migrate.blob.containers) == 0 ? [1] : [] + count = var.what_to_migrate.blob.enabled && length(var.what_to_migrate.blob.containers) == 0 ? 1 : 0 storage_account_id = data.azurerm_storage_account.source.id +} + +data "azapi_resource_list" "tables" { + type = "Microsoft.Storage/storageAccounts/tableServices/tables@2021-09-01" + parent_id = "${data.azurerm_storage_account.source.id}/tableServices/default" + response_export_values = ["*"] } \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/datasets_containers.tf b/src/_modules/data_factory_storage_account/datasets_containers.tf index daa4c4d93..0dfe0d28a 100644 --- a/src/_modules/data_factory_storage_account/datasets_containers.tf +++ b/src/_modules/data_factory_storage_account/datasets_containers.tf @@ -1,16 +1,16 @@ resource "azurerm_data_factory_custom_dataset" "source_dataset_container" { for_each = local.containers name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value.name}-blob-${module.naming_convention.suffix}" - data_factory_id = var.data_factory.id + data_factory_id = var.data_factory_id type = "AzureBlob" linked_service { - name = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_blob.name + name = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_blob[0].name } type_properties_json = jsonencode({ linkedServiceName = { - referenceName = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_blob.name + referenceName = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_blob[0].name type = "LinkedServiceReference" } type = "AzureBlob" @@ -21,16 +21,16 @@ resource "azurerm_data_factory_custom_dataset" "source_dataset_container" { resource "azurerm_data_factory_custom_dataset" "target_dataset_container" { for_each = local.containers name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value.name}-blob-${module.naming_convention.suffix}" - data_factory_id = var.data_factory.id + data_factory_id = var.data_factory_id type = "AzureBlob" linked_service { - name = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_blob.name + name = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_blob[0].name } type_properties_json = jsonencode({ linkedServiceName = { - referenceName = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_blob.name + referenceName = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_blob[0].name type = "LinkedServiceReference" } type = "AzureBlob" diff --git a/src/_modules/data_factory_storage_account/datasets_tables.tf b/src/_modules/data_factory_storage_account/datasets_tables.tf index ed10fc8a8..8e7cfd854 100644 --- a/src/_modules/data_factory_storage_account/datasets_tables.tf +++ b/src/_modules/data_factory_storage_account/datasets_tables.tf @@ -1,11 +1,11 @@ resource "azurerm_data_factory_custom_dataset" "source_dataset_table" { for_each = local.tables name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value.name}-table-${module.naming_convention.suffix}" - data_factory_id = var.data_factory.id + data_factory_id = var.data_factory_id type = "AzureTable" linked_service { - name = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_table.name + name = azurerm_data_factory_linked_service_azure_table_storage.source_linked_service_table[0].name } type_properties_json = jsonencode({ @@ -16,11 +16,11 @@ resource "azurerm_data_factory_custom_dataset" "source_dataset_table" { resource "azurerm_data_factory_custom_dataset" "target_dataset_table" { for_each = local.tables name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value.name}-table-${module.naming_convention.suffix}" - data_factory_id = var.data_factory.id + data_factory_id = var.data_factory_id type = "AzureTable" linked_service { - name = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_table.name + name = azurerm_data_factory_linked_service_azure_table_storage.target_linked_service_table[0].name } type_properties_json = jsonencode({ diff --git a/src/_modules/data_factory_storage_account/iam.tf b/src/_modules/data_factory_storage_account/iam.tf index 001998cd9..f57446493 100644 --- a/src/_modules/data_factory_storage_account/iam.tf +++ b/src/_modules/data_factory_storage_account/iam.tf @@ -1,6 +1,6 @@ module "roles" { source = "github.com/pagopa/dx//infra/modules/azure_role_assignments?ref=main" - principal_id = var.function_app.user_func_02.principal_id + principal_id = var.data_factory_principal_id storage_blob = var.what_to_migrate.blob.enabled ? [ { diff --git a/src/_modules/data_factory_storage_account/linked_services_containers.tf b/src/_modules/data_factory_storage_account/linked_services_containers.tf index 15cc89bc3..4ef912d27 100644 --- a/src/_modules/data_factory_storage_account/linked_services_containers.tf +++ b/src/_modules/data_factory_storage_account/linked_services_containers.tf @@ -1,7 +1,7 @@ resource "azurerm_data_factory_linked_service_azure_blob_storage" "source_linked_service_blob" { - for_each = var.what_to_migrate.blob.enabled ? [1] : [] + count = var.what_to_migrate.blob.enabled ? 1 : 0 name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-blob-${module.naming_convention.suffix}" - data_factory_id = var.data_factory.id + data_factory_id = var.data_factory_id service_endpoint = "https://${data.azurerm_storage_account.source.id}.blob.core.windows.net" @@ -9,9 +9,9 @@ resource "azurerm_data_factory_linked_service_azure_blob_storage" "source_linked } resource "azurerm_data_factory_linked_service_azure_blob_storage" "target_linked_service_blob" { - for_each = var.what_to_migrate.blob.enabled ? [1] : [] + count = var.what_to_migrate.blob.enabled ? 1 : 0 name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-blob-${module.naming_convention.suffix}" - data_factory_id = var.data_factory.id + data_factory_id = var.data_factory_id service_endpoint = "https://${data.azurerm_storage_account.target.id}.blob.core.windows.net" diff --git a/src/_modules/data_factory_storage_account/linked_services_tables.tf b/src/_modules/data_factory_storage_account/linked_services_tables.tf index 354693e56..a5bc9922d 100644 --- a/src/_modules/data_factory_storage_account/linked_services_tables.tf +++ b/src/_modules/data_factory_storage_account/linked_services_tables.tf @@ -1,15 +1,15 @@ resource "azurerm_data_factory_linked_service_azure_table_storage" "source_linked_service_table" { - for_each = var.what_to_migrate.table.enabled ? [1] : [] + count = var.what_to_migrate.table.enabled ? 1 : 0 name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-table-${module.naming_convention.suffix}" - data_factory_id = var.data_factory.id + data_factory_id = var.data_factory_id connection_string = data.azurerm_storage_account.source.primary_table_endpoint } resource "azurerm_data_factory_linked_service_azure_table_storage" "target_linked_service_table" { - for_each = var.what_to_migrate.table.enabled ? [1] : [] + count = var.what_to_migrate.table.enabled ? 1 : 0 name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-table-${module.naming_convention.suffix}" - data_factory_id = var.data_factory.id + data_factory_id = var.data_factory_id connection_string = data.azurerm_storage_account.target.primary_table_endpoint } \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/locals.tf b/src/_modules/data_factory_storage_account/locals.tf index db6f13ac2..e2ebfe7ca 100644 --- a/src/_modules/data_factory_storage_account/locals.tf +++ b/src/_modules/data_factory_storage_account/locals.tf @@ -1,4 +1,4 @@ locals { - containers = var.what_to_migrate.blob.enabled ? length(var.what_to_migrate.blob.containers) > 0 ? var.what_to_migrate.blob.containers : [for container in data.azurerm_storage_containers.this[0].containers : container.name] : [] - tables = var.what_to_migrate.table.enabled ? var.what_to_migrate.table.tables : [] + containers = var.what_to_migrate.blob.enabled ? (length(var.what_to_migrate.blob.containers) > 0 ? var.what_to_migrate.blob.containers : [for container in data.azurerm_storage_containers.this[0].containers : container.name]) : [] + tables = var.what_to_migrate.table.enabled ? (length(var.what_to_migrate.table.tables) > 0 ? var.what_to_migrate.table.tables : [for table in jsondecode(data.azapi_resource_list.tables.output).value : table.TableName]) : [] } \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/main.tf b/src/_modules/data_factory_storage_account/main.tf index 13dbda61f..70b9d5218 100644 --- a/src/_modules/data_factory_storage_account/main.tf +++ b/src/_modules/data_factory_storage_account/main.tf @@ -1,3 +1,12 @@ +terraform { + required_providers { + azapi = { + source = "Azure/azapi" + version = "<= 1.15.0" + } + } +} + module "naming_convention" { source = "github.com/pagopa/dx//infra/modules/azure_naming_convention/?ref=main" diff --git a/src/_modules/data_factory_storage_account/network.tf b/src/_modules/data_factory_storage_account/network.tf new file mode 100644 index 000000000..84fc0b287 --- /dev/null +++ b/src/_modules/data_factory_storage_account/network.tf @@ -0,0 +1,27 @@ +resource "azurerm_data_factory_managed_private_endpoint" "blob_source" { + name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-blob-${module.naming_convention.suffix}" + data_factory_id = var.data_factory_id + target_resource_id = data.azurerm_storage_account.source.id + subresource_name = "blob" +} + +resource "azurerm_data_factory_managed_private_endpoint" "blob_target" { + name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-blob-${module.naming_convention.suffix}" + data_factory_id = var.data_factory_id + target_resource_id = data.azurerm_storage_account.target.id + subresource_name = "blob" +} + +resource "azurerm_data_factory_managed_private_endpoint" "table_source" { + name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-table-${module.naming_convention.suffix}" + data_factory_id = var.data_factory_id + target_resource_id = data.azurerm_storage_account.source.id + subresource_name = "table" +} + +resource "azurerm_data_factory_managed_private_endpoint" "table_target" { + name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-table-${module.naming_convention.suffix}" + data_factory_id = var.data_factory_id + target_resource_id = data.azurerm_storage_account.target.id + subresource_name = "table" +} \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/pipeline_containers.tf b/src/_modules/data_factory_storage_account/pipeline_containers.tf index 914485d66..e399e75c3 100644 --- a/src/_modules/data_factory_storage_account/pipeline_containers.tf +++ b/src/_modules/data_factory_storage_account/pipeline_containers.tf @@ -1,11 +1,7 @@ resource "azurerm_data_factory_pipeline" "pipeline_container" { for_each = local.containers name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value.name}-blob-${module.naming_convention.suffix}" - data_factory_id = var.data_factory.id - - depends_on = [ - azurerm_data_factory_custom_dataset.dataset_container - ] + data_factory_id = var.data_factory_id activities_json = jsonencode( [ diff --git a/src/_modules/data_factory_storage_account/pipeline_tables.tf b/src/_modules/data_factory_storage_account/pipeline_tables.tf index 5ea2a11c4..56a2b6963 100644 --- a/src/_modules/data_factory_storage_account/pipeline_tables.tf +++ b/src/_modules/data_factory_storage_account/pipeline_tables.tf @@ -1,11 +1,7 @@ resource "azurerm_data_factory_pipeline" "pipeline_table" { for_each = local.tables name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value.name}-table-${module.naming_convention.suffix}" - data_factory_id = var.data_factory.id - - depends_on = [ - azurerm_data_factory_custom_dataset.dataset_table - ] + data_factory_id = var.data_factory_id activities_json = jsonencode( [ diff --git a/src/_modules/data_factory_storage_account/variables.tf b/src/_modules/data_factory_storage_account/variables.tf index c6dca9a79..e9bccd2ca 100644 --- a/src/_modules/data_factory_storage_account/variables.tf +++ b/src/_modules/data_factory_storage_account/variables.tf @@ -16,6 +16,11 @@ variable "data_factory_id" { type = string } +variable "data_factory_principal_id" { + description = "Data Factory principal id to grant access to." + type = string +} + variable "storage_accounts" { type = object({ source = object({ @@ -42,7 +47,7 @@ variable "what_to_migrate" { table = optional(object( { enabled = bool - tables = list(string) + tables = optional(list(string), []) }), { enabled = false } ) @@ -54,11 +59,5 @@ variable "what_to_migrate" { error_message = "At least one between blob and table should be enabled." } - # validate that if table is enabled, at least one table is specified - validation { - condition = !(var.what_to_migrate.table.enabled && length(var.what_to_migrate.table.tables) == 0) - error_message = "If table is enabled, at least one table should be specified." - } - - description = "List of databases, file shares, containers and tables to migrate." + description = "List of storage account containers and tables to migrate." } diff --git a/src/migration/prod/.terraform.lock.hcl b/src/migration/prod/.terraform.lock.hcl new file mode 100644 index 000000000..f5d1455e4 --- /dev/null +++ b/src/migration/prod/.terraform.lock.hcl @@ -0,0 +1,42 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/azure/azapi" { + version = "1.15.0" + constraints = "<= 1.15.0" + hashes = [ + "h1:pO/phGY+TxMEKQ+ffYj+vUIvG5A1tno/sZYDb/yyA/w=", + "zh:0627a8bc77254debc25dc0c7b62e055138217c97b03221e593c3c56dc7550671", + "zh:2fe045f07070ef75d0bec4b0595a74c14394daa838ddb964e2fd23cc98c40c34", + "zh:343009f39c957883b2c06145a5954e524c70f93585f943f1ea3d28ef6995d0d0", + "zh:53fe9ab54485aaebc9b91e27a10bce2729a1c95b1399079e631dc6bb9e3f27dc", + "zh:63c407e7dc04d178d4798c17ad489d9cc92f7d1941d7f4a3f560b95908b6107b", + "zh:7d6fc2b432b264f036bb80ab2b2ba67f80a5d98da8a8c322aa097833dad598c9", + "zh:7ec49c0a8799d469eb6e2a1f856693f9862f1b73f5ed70adc1b346e5a4c6458d", + "zh:889704f10319d301d677539d788fc82a7c73608ab78cb93e1280ac2be39e6e00", + "zh:90b4b07405b7cde9ebae3b034cb5bb5dd18484d1b95bd250f905451f1e86ac3f", + "zh:92aa9c241a8cb2a6d81ad47bc007c119f8b818464a960ebaf39008766c361e6b", + "zh:f28fbd0a2c59e239b53067bc1adc691be444876bcb2d4f78d310f549724da6e0", + "zh:ffb15e0ddfa505d0e9b75341570199076ae574887124f398162b1ead9376b25f", + ] +} + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "3.112.0" + constraints = "~> 3.106, <= 3.112.0" + hashes = [ + "h1:0N5R6MrdL9HT1COS6CDkFdwoo2PpUoJFeO7i68mMhsI=", + "zh:341c22454d24a75792aa99fbbc0c156f368534b7bb04eef4701b85995c7526a4", + "zh:3708656d75061c92f7208cc731b946c991ad343a443f8ff0ef082f077b7580b9", + "zh:38ca06f9f45705c648f04f272bd9483397693ea8da6db788cd7955f49ab79d6b", + "zh:3f305adb5ee0032e0ea68d198a089ecfd0127092930e99fa51377a250292b592", + "zh:4ae2fc6065164a819f576f705e634ebf5059f983149a41dad909719fea96145a", + "zh:5d376ac7dd71898a94038d6b6b8036dfec4c0216d832ec1135c855bf3e58eb5f", + "zh:63d2ff296d3aee5787e12c759a6a3d5aa15a574456aebbe11b833f01adf3faef", + "zh:8ad8746741f7f0ac10da6f1d105f26ebeb6e4d944f58ba749e86d7c9a67da3db", + "zh:abec182594ee8a21d72a5f23d3aa7fa45247488539fce6ed648c9c255d8bf972", + "zh:bf704b400be4181333b38c0306949f26326a9aa5ae68b4167e2fb8ee7fb13618", + "zh:c072938f8695f725fc5fbe986a54890f00d520cce570006390dc5bbc51b2a4ea", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/src/migration/prod/README.md b/src/migration/prod/README.md index ec20557ab..d13cbd421 100644 --- a/src/migration/prod/README.md +++ b/src/migration/prod/README.md @@ -5,13 +5,14 @@ | Name | Version | |------|---------| +| [azapi](#requirement\_azapi) | <= 1.15.0 | | [azurerm](#requirement\_azurerm) | <= 3.112.0 | ## Providers | Name | Version | |------|---------| -| [azurerm](#provider\_azurerm) | <= 3.112.0 | +| [azurerm](#provider\_azurerm) | 3.112.0 | ## Modules diff --git a/src/migration/prod/italynorth.tf b/src/migration/prod/italynorth.tf index bf8384b0e..804af7c8e 100644 --- a/src/migration/prod/italynorth.tf +++ b/src/migration/prod/italynorth.tf @@ -1,5 +1,5 @@ resource "azurerm_resource_group" "migration" { - name = "${local.project_itn}-migration-rg-01" + name = "${local.project_itn}-${local.environment.domain}-${local.environment.app_name}-rg-01" location = "italynorth" tags = local.tags @@ -8,7 +8,7 @@ resource "azurerm_resource_group" "migration" { # Create Azure Data Factory instances # Enables system-assigned managed identity for secure access to resources resource "azurerm_data_factory" "this" { - name = "${local.project_itn}-migration-adf-01" + name = "${local.project_itn}-${local.environment.domain}-${local.environment.app_name}-adf-01" location = "italynorth" resource_group_name = azurerm_resource_group.migration.name @@ -20,25 +20,19 @@ resource "azurerm_data_factory" "this" { } resource "azurerm_data_factory_integration_runtime_azure" "azure_runtime" { - name = "${local.project_itn}-migration-adfir-01" + name = "${local.project_itn}-${local.environment.domain}-${local.environment.app_name}-adfir-01" location = "italynorth" data_factory_id = azurerm_data_factory.this.id } module "migrate_storage_accounts" { - for_each = local.storage_accounts + for_each = { for migration in local.storage_accounts : "${migration.source.name}|${migration.target.name}" => migration } source = "../../_modules/data_factory_storage_account" - environment = { - prefix = local.prefix - env_short = local.env_short - location = "italynorth" - domain = "eng" - app_name = "mig" - instance_number = "01" - } + environment = local.environment - data_factory_id = azurerm_data_factory.this.id + data_factory_id = azurerm_data_factory.this.id + data_factory_principal_id = azurerm_data_factory.this.identity[0].principal_id storage_accounts = { source = each.value.source @@ -48,12 +42,12 @@ module "migrate_storage_accounts" { what_to_migrate = { blob = { enabled = each.value.blob.enabled - containers = each.value.blob.containers + containers = try(each.value.blob.containers, []) } table = { enabled = each.value.table.enabled - tables = each.value.table.tables + tables = try(each.value.table.tables, []) } } } \ No newline at end of file diff --git a/src/migration/prod/locals.tf b/src/migration/prod/locals.tf index f4d873141..a1fb140f9 100644 --- a/src/migration/prod/locals.tf +++ b/src/migration/prod/locals.tf @@ -7,6 +7,15 @@ locals { project_weu_legacy = "${local.prefix}-${local.env_short}" secondary_project = "${local.prefix}-${local.env_short}-${local.location_short.germanywestcentral}" + environment = { + prefix = local.prefix + env_short = local.env_short + location = "italynorth" + domain = "eng" + app_name = "migitn" + instance_number = "01" + } + tags = { CostCenter = "TS310 - PAGAMENTI & SERVIZI" CreatedBy = "Terraform" @@ -14,4 +23,13 @@ locals { Owner = "IO" Source = "https://github.com/pagopa/io-infra/blob/main/src/migration/prod" } + + storage_accounts = [ + { + source = { name = "iopweuabc", resource_group_name = "abc" } + target = { name = "iopitnabc", resource_group_name = "abc" } + blob = { enabled = true } + table = { enabled = true } + } + ] } diff --git a/src/migration/prod/main.tf b/src/migration/prod/main.tf index 271442c57..535d0df02 100644 --- a/src/migration/prod/main.tf +++ b/src/migration/prod/main.tf @@ -12,9 +12,16 @@ terraform { source = "hashicorp/azurerm" version = "<= 3.112.0" } + + azapi = { + source = "Azure/azapi" + version = "<= 1.15.0" + } } } provider "azurerm" { features {} } + +provider "azapi" {} From 2e0da2fc01f1aced5f9a63aa1e3db16f65b7e991 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Wed, 16 Oct 2024 12:38:27 +0200 Subject: [PATCH 10/22] fix: errors and tested --- .../datasets_containers.tf | 12 ++++---- .../datasets_tables.tf | 12 ++++---- .../data_factory_storage_account/locals.tf | 4 ++- .../data_factory_storage_account/outputs.tf | 2 +- .../pipeline_containers.tf | 4 +-- .../pipeline_tables.tf | 4 +-- src/migration/prod/.terraform.lock.hcl | 28 +++++++++---------- src/migration/prod/data.tf | 0 src/migration/prod/locals.tf | 19 +++++++++---- src/migration/prod/main.tf | 2 +- 10 files changed, 48 insertions(+), 39 deletions(-) delete mode 100644 src/migration/prod/data.tf diff --git a/src/_modules/data_factory_storage_account/datasets_containers.tf b/src/_modules/data_factory_storage_account/datasets_containers.tf index 0dfe0d28a..6f945d726 100644 --- a/src/_modules/data_factory_storage_account/datasets_containers.tf +++ b/src/_modules/data_factory_storage_account/datasets_containers.tf @@ -1,6 +1,6 @@ resource "azurerm_data_factory_custom_dataset" "source_dataset_container" { - for_each = local.containers - name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value.name}-blob-${module.naming_convention.suffix}" + for_each = toset(local.containers) + name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-blob-${module.naming_convention.suffix}" data_factory_id = var.data_factory_id type = "AzureBlob" @@ -14,13 +14,13 @@ resource "azurerm_data_factory_custom_dataset" "source_dataset_container" { type = "LinkedServiceReference" } type = "AzureBlob" - folderPath = each.value.name + folderPath = each.value }) } resource "azurerm_data_factory_custom_dataset" "target_dataset_container" { - for_each = local.containers - name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value.name}-blob-${module.naming_convention.suffix}" + for_each = toset(local.containers) + name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value}-blob-${module.naming_convention.suffix}" data_factory_id = var.data_factory_id type = "AzureBlob" @@ -34,6 +34,6 @@ resource "azurerm_data_factory_custom_dataset" "target_dataset_container" { type = "LinkedServiceReference" } type = "AzureBlob" - folderPath = each.value.name + folderPath = each.value }) } \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/datasets_tables.tf b/src/_modules/data_factory_storage_account/datasets_tables.tf index 8e7cfd854..56ed466f0 100644 --- a/src/_modules/data_factory_storage_account/datasets_tables.tf +++ b/src/_modules/data_factory_storage_account/datasets_tables.tf @@ -1,6 +1,6 @@ resource "azurerm_data_factory_custom_dataset" "source_dataset_table" { - for_each = local.tables - name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value.name}-table-${module.naming_convention.suffix}" + for_each = toset(local.tables) + name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-table-${module.naming_convention.suffix}" data_factory_id = var.data_factory_id type = "AzureTable" @@ -9,13 +9,13 @@ resource "azurerm_data_factory_custom_dataset" "source_dataset_table" { } type_properties_json = jsonencode({ - tableName = each.value.name + tableName = each.value }) } resource "azurerm_data_factory_custom_dataset" "target_dataset_table" { - for_each = local.tables - name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value.name}-table-${module.naming_convention.suffix}" + for_each = toset(local.tables) + name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value}-table-${module.naming_convention.suffix}" data_factory_id = var.data_factory_id type = "AzureTable" @@ -24,6 +24,6 @@ resource "azurerm_data_factory_custom_dataset" "target_dataset_table" { } type_properties_json = jsonencode({ - tableName = each.value.name + tableName = each.value }) } \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/locals.tf b/src/_modules/data_factory_storage_account/locals.tf index e2ebfe7ca..892bfe213 100644 --- a/src/_modules/data_factory_storage_account/locals.tf +++ b/src/_modules/data_factory_storage_account/locals.tf @@ -1,4 +1,6 @@ locals { containers = var.what_to_migrate.blob.enabled ? (length(var.what_to_migrate.blob.containers) > 0 ? var.what_to_migrate.blob.containers : [for container in data.azurerm_storage_containers.this[0].containers : container.name]) : [] - tables = var.what_to_migrate.table.enabled ? (length(var.what_to_migrate.table.tables) > 0 ? var.what_to_migrate.table.tables : [for table in jsondecode(data.azapi_resource_list.tables.output).value : table.TableName]) : [] + + azapi_tables = jsondecode(data.azapi_resource_list.tables.output) + tables = var.what_to_migrate.table.enabled ? (length(var.what_to_migrate.table.tables) > 0 ? var.what_to_migrate.table.tables : [for table in local.azapi_tables.value : table.name]) : [] } \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/outputs.tf b/src/_modules/data_factory_storage_account/outputs.tf index 22c1d86d8..2153f8777 100644 --- a/src/_modules/data_factory_storage_account/outputs.tf +++ b/src/_modules/data_factory_storage_account/outputs.tf @@ -1,6 +1,6 @@ output "pipelines" { value = { - for pipeline in concat(azurerm_data_factory_pipeline.pipeline_container, azurerm_data_factory_pipeline.pipeline_table) + for pipeline in merge(azurerm_data_factory_pipeline.pipeline_container, azurerm_data_factory_pipeline.pipeline_table) : pipeline.name => { id = pipeline.id name = pipeline.name diff --git a/src/_modules/data_factory_storage_account/pipeline_containers.tf b/src/_modules/data_factory_storage_account/pipeline_containers.tf index e399e75c3..0a5fa77d6 100644 --- a/src/_modules/data_factory_storage_account/pipeline_containers.tf +++ b/src/_modules/data_factory_storage_account/pipeline_containers.tf @@ -1,6 +1,6 @@ resource "azurerm_data_factory_pipeline" "pipeline_container" { - for_each = local.containers - name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value.name}-blob-${module.naming_convention.suffix}" + for_each = toset(local.containers) + name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-blob-${module.naming_convention.suffix}" data_factory_id = var.data_factory_id activities_json = jsonencode( diff --git a/src/_modules/data_factory_storage_account/pipeline_tables.tf b/src/_modules/data_factory_storage_account/pipeline_tables.tf index 56a2b6963..364ad691c 100644 --- a/src/_modules/data_factory_storage_account/pipeline_tables.tf +++ b/src/_modules/data_factory_storage_account/pipeline_tables.tf @@ -1,6 +1,6 @@ resource "azurerm_data_factory_pipeline" "pipeline_table" { - for_each = local.tables - name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value.name}-table-${module.naming_convention.suffix}" + for_each = toset(local.tables) + name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-table-${module.naming_convention.suffix}" data_factory_id = var.data_factory_id activities_json = jsonencode( diff --git a/src/migration/prod/.terraform.lock.hcl b/src/migration/prod/.terraform.lock.hcl index f5d1455e4..18e707f28 100644 --- a/src/migration/prod/.terraform.lock.hcl +++ b/src/migration/prod/.terraform.lock.hcl @@ -22,21 +22,21 @@ provider "registry.terraform.io/azure/azapi" { } provider "registry.terraform.io/hashicorp/azurerm" { - version = "3.112.0" - constraints = "~> 3.106, <= 3.112.0" + version = "3.116.0" + constraints = "~> 3.106, <= 3.116.0" hashes = [ - "h1:0N5R6MrdL9HT1COS6CDkFdwoo2PpUoJFeO7i68mMhsI=", - "zh:341c22454d24a75792aa99fbbc0c156f368534b7bb04eef4701b85995c7526a4", - "zh:3708656d75061c92f7208cc731b946c991ad343a443f8ff0ef082f077b7580b9", - "zh:38ca06f9f45705c648f04f272bd9483397693ea8da6db788cd7955f49ab79d6b", - "zh:3f305adb5ee0032e0ea68d198a089ecfd0127092930e99fa51377a250292b592", - "zh:4ae2fc6065164a819f576f705e634ebf5059f983149a41dad909719fea96145a", - "zh:5d376ac7dd71898a94038d6b6b8036dfec4c0216d832ec1135c855bf3e58eb5f", - "zh:63d2ff296d3aee5787e12c759a6a3d5aa15a574456aebbe11b833f01adf3faef", - "zh:8ad8746741f7f0ac10da6f1d105f26ebeb6e4d944f58ba749e86d7c9a67da3db", - "zh:abec182594ee8a21d72a5f23d3aa7fa45247488539fce6ed648c9c255d8bf972", - "zh:bf704b400be4181333b38c0306949f26326a9aa5ae68b4167e2fb8ee7fb13618", - "zh:c072938f8695f725fc5fbe986a54890f00d520cce570006390dc5bbc51b2a4ea", + "h1:BCR3NIorFSvGG3v/+JOiiw3VM4PkChLO4m84wzD9NDo=", + "zh:02b6606aff025fc2a962b3e568e000300abe959adac987183c24dac8eb057f4d", + "zh:2a23a8ce24ff9e885925ffee0c3ea7eadba7a702541d05869275778aa47bdea7", + "zh:57d10746384baeca4d5c56e88872727cdc150f437b8c5e14f0542127f7475e24", + "zh:59e3ebde1a2e1e094c671e179f231ead60684390dbf02d2b1b7fe67a228daa1a", + "zh:5f1f5c7d09efa2ee8ddf21bd9efbbf8286f6e90047556bef305c062fa0ac5880", + "zh:a40646aee3c9907276dab926e6123a8d70b1e56174836d4c59a9992034f88d70", + "zh:c21d40461bc5836cf56ad3d93d2fc47f61138574a55e972ad5ff1cb73bab66dc", + "zh:c56fb91a5ae66153ba0f737a26da1b3d4f88fdef7d41c63e06c5772d93b26953", + "zh:d1e60e85f51d12fc150aeab8e31d3f18f859c32f927f99deb5b74cb1e10087aa", + "zh:ed35e727e7d79e687cd3d148f52b442961ede286e7c5b4da1dcd9f0128009466", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f6d2a4e7c58f44e7d04a4a9c73f35ed452f412c97c85def68c4b52814cbe03ab", ] } diff --git a/src/migration/prod/data.tf b/src/migration/prod/data.tf deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/migration/prod/locals.tf b/src/migration/prod/locals.tf index a1fb140f9..b1bfc75a3 100644 --- a/src/migration/prod/locals.tf +++ b/src/migration/prod/locals.tf @@ -25,11 +25,18 @@ locals { } storage_accounts = [ - { - source = { name = "iopweuabc", resource_group_name = "abc" } - target = { name = "iopitnabc", resource_group_name = "abc" } - blob = { enabled = true } - table = { enabled = true } - } + # Copy both containers and tables + # { + # source = { name = "stdevbiptest1", resource_group_name = "RG-BIP-DEV-TEST" } + # target = { name = "stbipdevtest1", resource_group_name = "dev-fasanorg" } + # }, + # + # Copy only selected containers and tables + # { + # source = { name = "stdevbiptest1", resource_group_name = "RG-BIP-DEV-TEST" } + # target = { name = "stbipdevtest1", resource_group_name = "dev-fasanorg" } + # blob = {enabled = true, containers = ["c1", "c2", "c3"]} + # table = {enabled = true, tables = ["t1", "t2", "t3"]} + # } ] } diff --git a/src/migration/prod/main.tf b/src/migration/prod/main.tf index 535d0df02..fcca7f12c 100644 --- a/src/migration/prod/main.tf +++ b/src/migration/prod/main.tf @@ -10,7 +10,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "<= 3.112.0" + version = "<= 3.116.0" } azapi = { From a28f762e4b6382a5a470e1e104eb585c9d40d309 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Wed, 16 Oct 2024 12:41:03 +0200 Subject: [PATCH 11/22] chore: ran pre-commit --- src/_modules/data_factory_storage_account/locals.tf | 2 +- src/migration/prod/README.md | 4 ++-- src/migration/prod/locals.tf | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/_modules/data_factory_storage_account/locals.tf b/src/_modules/data_factory_storage_account/locals.tf index 892bfe213..2ec34c586 100644 --- a/src/_modules/data_factory_storage_account/locals.tf +++ b/src/_modules/data_factory_storage_account/locals.tf @@ -2,5 +2,5 @@ locals { containers = var.what_to_migrate.blob.enabled ? (length(var.what_to_migrate.blob.containers) > 0 ? var.what_to_migrate.blob.containers : [for container in data.azurerm_storage_containers.this[0].containers : container.name]) : [] azapi_tables = jsondecode(data.azapi_resource_list.tables.output) - tables = var.what_to_migrate.table.enabled ? (length(var.what_to_migrate.table.tables) > 0 ? var.what_to_migrate.table.tables : [for table in local.azapi_tables.value : table.name]) : [] + tables = var.what_to_migrate.table.enabled ? (length(var.what_to_migrate.table.tables) > 0 ? var.what_to_migrate.table.tables : [for table in local.azapi_tables.value : table.name]) : [] } \ No newline at end of file diff --git a/src/migration/prod/README.md b/src/migration/prod/README.md index d13cbd421..b20548be9 100644 --- a/src/migration/prod/README.md +++ b/src/migration/prod/README.md @@ -6,13 +6,13 @@ | Name | Version | |------|---------| | [azapi](#requirement\_azapi) | <= 1.15.0 | -| [azurerm](#requirement\_azurerm) | <= 3.112.0 | +| [azurerm](#requirement\_azurerm) | <= 3.116.0 | ## Providers | Name | Version | |------|---------| -| [azurerm](#provider\_azurerm) | 3.112.0 | +| [azurerm](#provider\_azurerm) | 3.116.0 | ## Modules diff --git a/src/migration/prod/locals.tf b/src/migration/prod/locals.tf index b1bfc75a3..b6ec2f32f 100644 --- a/src/migration/prod/locals.tf +++ b/src/migration/prod/locals.tf @@ -33,8 +33,8 @@ locals { # # Copy only selected containers and tables # { - # source = { name = "stdevbiptest1", resource_group_name = "RG-BIP-DEV-TEST" } - # target = { name = "stbipdevtest1", resource_group_name = "dev-fasanorg" } + # source = { name = "stdevbiptest1", resource_group_name = "RG-BIP-DEV-TEST" } + # target = { name = "stbipdevtest1", resource_group_name = "dev-fasanorg" } # blob = {enabled = true, containers = ["c1", "c2", "c3"]} # table = {enabled = true, tables = ["t1", "t2", "t3"]} # } From 7f8190908ebdb3c6ac070342167ff5aa5159ab80 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Wed, 16 Oct 2024 12:44:22 +0200 Subject: [PATCH 12/22] feat: using instance number from local --- src/migration/prod/italynorth.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/migration/prod/italynorth.tf b/src/migration/prod/italynorth.tf index 804af7c8e..e7278fb1b 100644 --- a/src/migration/prod/italynorth.tf +++ b/src/migration/prod/italynorth.tf @@ -1,5 +1,5 @@ resource "azurerm_resource_group" "migration" { - name = "${local.project_itn}-${local.environment.domain}-${local.environment.app_name}-rg-01" + name = "${local.project_itn}-${local.environment.domain}-${local.environment.app_name}-rg-${local.environment.instance_number}" location = "italynorth" tags = local.tags @@ -8,7 +8,7 @@ resource "azurerm_resource_group" "migration" { # Create Azure Data Factory instances # Enables system-assigned managed identity for secure access to resources resource "azurerm_data_factory" "this" { - name = "${local.project_itn}-${local.environment.domain}-${local.environment.app_name}-adf-01" + name = "${local.project_itn}-${local.environment.domain}-${local.environment.app_name}-adf-${local.environment.instance_number}" location = "italynorth" resource_group_name = azurerm_resource_group.migration.name @@ -20,7 +20,7 @@ resource "azurerm_data_factory" "this" { } resource "azurerm_data_factory_integration_runtime_azure" "azure_runtime" { - name = "${local.project_itn}-${local.environment.domain}-${local.environment.app_name}-adfir-01" + name = "${local.project_itn}-${local.environment.domain}-${local.environment.app_name}-adfir-${local.environment.instance_number}" location = "italynorth" data_factory_id = azurerm_data_factory.this.id } From 09d17e661d3474051ca152a142b14c82e5d7fe9e Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Wed, 16 Oct 2024 14:18:36 +0200 Subject: [PATCH 13/22] feat: adf public_network_enabled set to false --- src/migration/prod/italynorth.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/migration/prod/italynorth.tf b/src/migration/prod/italynorth.tf index e7278fb1b..d3c104b5e 100644 --- a/src/migration/prod/italynorth.tf +++ b/src/migration/prod/italynorth.tf @@ -12,6 +12,8 @@ resource "azurerm_data_factory" "this" { location = "italynorth" resource_group_name = azurerm_resource_group.migration.name + public_network_enabled = false + identity { type = "SystemAssigned" } From 7200a5a888f6c3c9177fd7fcc0f7ac312b47958a Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Wed, 16 Oct 2024 14:27:48 +0200 Subject: [PATCH 14/22] feat: added managed virtual network support to adf --- src/migration/prod/italynorth.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/migration/prod/italynorth.tf b/src/migration/prod/italynorth.tf index d3c104b5e..5c5bfc9a4 100644 --- a/src/migration/prod/italynorth.tf +++ b/src/migration/prod/italynorth.tf @@ -12,7 +12,8 @@ resource "azurerm_data_factory" "this" { location = "italynorth" resource_group_name = azurerm_resource_group.migration.name - public_network_enabled = false + public_network_enabled = false + managed_virtual_network_enabled = true identity { type = "SystemAssigned" From 59cac9f11bc921a658ca18b4dd236d8a76010e5e Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Wed, 16 Oct 2024 15:25:54 +0200 Subject: [PATCH 15/22] fix: changed from jsonsource to binarysource --- .../data_factory_storage_account/pipeline_containers.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/_modules/data_factory_storage_account/pipeline_containers.tf b/src/_modules/data_factory_storage_account/pipeline_containers.tf index 0a5fa77d6..cc8a75058 100644 --- a/src/_modules/data_factory_storage_account/pipeline_containers.tf +++ b/src/_modules/data_factory_storage_account/pipeline_containers.tf @@ -19,7 +19,7 @@ resource "azurerm_data_factory_pipeline" "pipeline_container" { userProperties = [] typeProperties = { source = { - type = "JsonSource" + type = "BinarySource" storeSettings = { type = "AzureBlobStorageReadSettings" recursive = true @@ -27,16 +27,16 @@ resource "azurerm_data_factory_pipeline" "pipeline_container" { wildcardFileName = "*" # Copy all files } formatSettings = { - type = "JsonReadSettings" + type = "" } } sink = { - type = "JsonSink" + type = "BinarySink" storeSettings = { type = "AzureBlobStorageWriteSettings" } formatSettings = { - type = "JsonWriteSettings" + type = "" } } enableStaging = false From 0e714ca59908e82c624177a74793da117c58d0ce Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 17 Oct 2024 10:18:46 +0200 Subject: [PATCH 16/22] fix: references to resources --- src/_modules/data_factory_storage_account/network.tf | 4 ++++ .../data_factory_storage_account/pipeline_containers.tf | 4 ++-- .../data_factory_storage_account/pipeline_tables.tf | 4 ++-- src/_modules/data_factory_storage_account/variables.tf | 8 ++++---- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/_modules/data_factory_storage_account/network.tf b/src/_modules/data_factory_storage_account/network.tf index 84fc0b287..fffdc0751 100644 --- a/src/_modules/data_factory_storage_account/network.tf +++ b/src/_modules/data_factory_storage_account/network.tf @@ -1,4 +1,5 @@ resource "azurerm_data_factory_managed_private_endpoint" "blob_source" { + count = var.what_to_migrate.blob.enabled ? 1 : 0 name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-blob-${module.naming_convention.suffix}" data_factory_id = var.data_factory_id target_resource_id = data.azurerm_storage_account.source.id @@ -6,6 +7,7 @@ resource "azurerm_data_factory_managed_private_endpoint" "blob_source" { } resource "azurerm_data_factory_managed_private_endpoint" "blob_target" { + count = var.what_to_migrate.blob.enabled ? 1 : 0 name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-blob-${module.naming_convention.suffix}" data_factory_id = var.data_factory_id target_resource_id = data.azurerm_storage_account.target.id @@ -13,6 +15,7 @@ resource "azurerm_data_factory_managed_private_endpoint" "blob_target" { } resource "azurerm_data_factory_managed_private_endpoint" "table_source" { + count = var.what_to_migrate.table.enabled ? 1 : 0 name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-table-${module.naming_convention.suffix}" data_factory_id = var.data_factory_id target_resource_id = data.azurerm_storage_account.source.id @@ -20,6 +23,7 @@ resource "azurerm_data_factory_managed_private_endpoint" "table_source" { } resource "azurerm_data_factory_managed_private_endpoint" "table_target" { + count = var.what_to_migrate.table.enabled ? 1 : 0 name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-table-${module.naming_convention.suffix}" data_factory_id = var.data_factory_id target_resource_id = data.azurerm_storage_account.target.id diff --git a/src/_modules/data_factory_storage_account/pipeline_containers.tf b/src/_modules/data_factory_storage_account/pipeline_containers.tf index cc8a75058..96b0b75c0 100644 --- a/src/_modules/data_factory_storage_account/pipeline_containers.tf +++ b/src/_modules/data_factory_storage_account/pipeline_containers.tf @@ -43,13 +43,13 @@ resource "azurerm_data_factory_pipeline" "pipeline_container" { } inputs = [ { - referenceName = azurerm_data_factory_custom_dataset.source_dataset_container + referenceName = azurerm_data_factory_custom_dataset.source_dataset_container[each.value] type = "DatasetReference" } ] outputs = [ { - referenceName = azurerm_data_factory_custom_dataset.target_dataset_container + referenceName = azurerm_data_factory_custom_dataset.target_dataset_container[each.value] type = "DatasetReference" } ] diff --git a/src/_modules/data_factory_storage_account/pipeline_tables.tf b/src/_modules/data_factory_storage_account/pipeline_tables.tf index 364ad691c..b795e5be0 100644 --- a/src/_modules/data_factory_storage_account/pipeline_tables.tf +++ b/src/_modules/data_factory_storage_account/pipeline_tables.tf @@ -31,13 +31,13 @@ resource "azurerm_data_factory_pipeline" "pipeline_table" { } inputs = [ { - referenceName = azurerm_data_factory_custom_dataset.source_dataset_table + referenceName = azurerm_data_factory_custom_dataset.source_dataset_table[each.value] type = "DatasetReference" } ] outputs = [ { - referenceName = azurerm_data_factory_custom_dataset.target_dataset_table + referenceName = azurerm_data_factory_custom_dataset.target_dataset_table[each.value] type = "DatasetReference" } ] diff --git a/src/_modules/data_factory_storage_account/variables.tf b/src/_modules/data_factory_storage_account/variables.tf index e9bccd2ca..dcdc4210a 100644 --- a/src/_modules/data_factory_storage_account/variables.tf +++ b/src/_modules/data_factory_storage_account/variables.tf @@ -39,17 +39,17 @@ variable "what_to_migrate" { type = object({ blob = optional(object( { - enabled = bool + enabled = optional(bool, true) containers = optional(list(string), []) }), - { enabled = false } + { enabled = true, containers = [] } ) table = optional(object( { - enabled = bool + enabled = optional(bool, true) tables = optional(list(string), []) }), - { enabled = false } + { enabled = true, tables = [] } ) }) From 041e94b1e988ae5fd1ef00b3544b3216abd00c36 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 17 Oct 2024 11:46:23 +0200 Subject: [PATCH 17/22] feat: subdivide datafactory resources in folders --- .../datasets_containers.tf | 6 +++-- .../datasets_tables.tf | 7 +++--- .../data_factory_storage_account/iam.tf | 10 ++++++++- .../linked_services_containers.tf | 4 ++-- .../linked_services_tables.tf | 4 ++-- .../pipeline_containers.tf | 7 +++--- .../pipeline_tables.tf | 22 ++++++++++++++----- src/migration/prod/README.md | 5 ++++- src/migration/prod/italynorth.tf | 10 ++++----- src/migration/prod/locals.tf | 13 +++++++++-- src/migration/prod/main.tf | 12 +++++----- src/migration/prod/outputs.tf | 11 ++++++++++ 12 files changed, 78 insertions(+), 33 deletions(-) diff --git a/src/_modules/data_factory_storage_account/datasets_containers.tf b/src/_modules/data_factory_storage_account/datasets_containers.tf index 6f945d726..33b505ce4 100644 --- a/src/_modules/data_factory_storage_account/datasets_containers.tf +++ b/src/_modules/data_factory_storage_account/datasets_containers.tf @@ -1,8 +1,9 @@ resource "azurerm_data_factory_custom_dataset" "source_dataset_container" { for_each = toset(local.containers) - name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-blob-${module.naming_convention.suffix}" + name = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-blob-${module.naming_convention.suffix}", "-", "_") data_factory_id = var.data_factory_id type = "AzureBlob" + folder = "${var.storage_accounts.source.name}/source/blob" linked_service { name = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_blob[0].name @@ -20,9 +21,10 @@ resource "azurerm_data_factory_custom_dataset" "source_dataset_container" { resource "azurerm_data_factory_custom_dataset" "target_dataset_container" { for_each = toset(local.containers) - name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value}-blob-${module.naming_convention.suffix}" + name = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value}-blob-${module.naming_convention.suffix}", "-", "_") data_factory_id = var.data_factory_id type = "AzureBlob" + folder = "${var.storage_accounts.source.name}/target/blob" linked_service { name = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_blob[0].name diff --git a/src/_modules/data_factory_storage_account/datasets_tables.tf b/src/_modules/data_factory_storage_account/datasets_tables.tf index 56ed466f0..85923189e 100644 --- a/src/_modules/data_factory_storage_account/datasets_tables.tf +++ b/src/_modules/data_factory_storage_account/datasets_tables.tf @@ -1,8 +1,9 @@ resource "azurerm_data_factory_custom_dataset" "source_dataset_table" { for_each = toset(local.tables) - name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-table-${module.naming_convention.suffix}" + name = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-table-${module.naming_convention.suffix}", "-", "_") data_factory_id = var.data_factory_id type = "AzureTable" + folder = "${var.storage_accounts.source.name}/source/table" linked_service { name = azurerm_data_factory_linked_service_azure_table_storage.source_linked_service_table[0].name @@ -15,10 +16,10 @@ resource "azurerm_data_factory_custom_dataset" "source_dataset_table" { resource "azurerm_data_factory_custom_dataset" "target_dataset_table" { for_each = toset(local.tables) - name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value}-table-${module.naming_convention.suffix}" + name = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value}-table-${module.naming_convention.suffix}", "-", "_") data_factory_id = var.data_factory_id type = "AzureTable" - + folder = "${var.storage_accounts.source.name}/target/table" linked_service { name = azurerm_data_factory_linked_service_azure_table_storage.target_linked_service_table[0].name } diff --git a/src/_modules/data_factory_storage_account/iam.tf b/src/_modules/data_factory_storage_account/iam.tf index f57446493..45bb6c83e 100644 --- a/src/_modules/data_factory_storage_account/iam.tf +++ b/src/_modules/data_factory_storage_account/iam.tf @@ -29,4 +29,12 @@ module "roles" { role = "writer" } ] : [] -} \ No newline at end of file +} + +# add to var.data_factory_principal_id the Storage Account Contributor role using the azurerm_role_assignment resourcew +resource "azurerm_role_assignment" "storage_account_contributor" { + count = var.what_to_migrate.table.enabled ? 1 : 0 + scope = data.azurerm_storage_account.target.id + role_definition_name = "Storage Account Contributor" + principal_id = var.data_factory_principal_id +} diff --git a/src/_modules/data_factory_storage_account/linked_services_containers.tf b/src/_modules/data_factory_storage_account/linked_services_containers.tf index 4ef912d27..3a16233e4 100644 --- a/src/_modules/data_factory_storage_account/linked_services_containers.tf +++ b/src/_modules/data_factory_storage_account/linked_services_containers.tf @@ -3,7 +3,7 @@ resource "azurerm_data_factory_linked_service_azure_blob_storage" "source_linked name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-blob-${module.naming_convention.suffix}" data_factory_id = var.data_factory_id - service_endpoint = "https://${data.azurerm_storage_account.source.id}.blob.core.windows.net" + service_endpoint = "https://${data.azurerm_storage_account.source.name}.blob.core.windows.net" use_managed_identity = true } @@ -13,7 +13,7 @@ resource "azurerm_data_factory_linked_service_azure_blob_storage" "target_linked name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-blob-${module.naming_convention.suffix}" data_factory_id = var.data_factory_id - service_endpoint = "https://${data.azurerm_storage_account.target.id}.blob.core.windows.net" + service_endpoint = "https://${data.azurerm_storage_account.target.name}.blob.core.windows.net" use_managed_identity = true } \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/linked_services_tables.tf b/src/_modules/data_factory_storage_account/linked_services_tables.tf index a5bc9922d..c343b3a71 100644 --- a/src/_modules/data_factory_storage_account/linked_services_tables.tf +++ b/src/_modules/data_factory_storage_account/linked_services_tables.tf @@ -3,7 +3,7 @@ resource "azurerm_data_factory_linked_service_azure_table_storage" "source_linke name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-table-${module.naming_convention.suffix}" data_factory_id = var.data_factory_id - connection_string = data.azurerm_storage_account.source.primary_table_endpoint + connection_string = data.azurerm_storage_account.source.primary_connection_string } resource "azurerm_data_factory_linked_service_azure_table_storage" "target_linked_service_table" { @@ -11,5 +11,5 @@ resource "azurerm_data_factory_linked_service_azure_table_storage" "target_linke name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-table-${module.naming_convention.suffix}" data_factory_id = var.data_factory_id - connection_string = data.azurerm_storage_account.target.primary_table_endpoint + connection_string = data.azurerm_storage_account.target.primary_connection_string } \ No newline at end of file diff --git a/src/_modules/data_factory_storage_account/pipeline_containers.tf b/src/_modules/data_factory_storage_account/pipeline_containers.tf index 96b0b75c0..dc6a3d996 100644 --- a/src/_modules/data_factory_storage_account/pipeline_containers.tf +++ b/src/_modules/data_factory_storage_account/pipeline_containers.tf @@ -1,7 +1,8 @@ resource "azurerm_data_factory_pipeline" "pipeline_container" { for_each = toset(local.containers) - name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-blob-${module.naming_convention.suffix}" + name = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-blob-${module.naming_convention.suffix}", "-", "_") data_factory_id = var.data_factory_id + folder = "${var.storage_accounts.source.name}/blob" activities_json = jsonencode( [ @@ -43,13 +44,13 @@ resource "azurerm_data_factory_pipeline" "pipeline_container" { } inputs = [ { - referenceName = azurerm_data_factory_custom_dataset.source_dataset_container[each.value] + referenceName = azurerm_data_factory_custom_dataset.source_dataset_container[each.value].name type = "DatasetReference" } ] outputs = [ { - referenceName = azurerm_data_factory_custom_dataset.target_dataset_container[each.value] + referenceName = azurerm_data_factory_custom_dataset.target_dataset_container[each.value].name type = "DatasetReference" } ] diff --git a/src/_modules/data_factory_storage_account/pipeline_tables.tf b/src/_modules/data_factory_storage_account/pipeline_tables.tf index b795e5be0..b6d52fef2 100644 --- a/src/_modules/data_factory_storage_account/pipeline_tables.tf +++ b/src/_modules/data_factory_storage_account/pipeline_tables.tf @@ -1,7 +1,8 @@ resource "azurerm_data_factory_pipeline" "pipeline_table" { for_each = toset(local.tables) - name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-table-${module.naming_convention.suffix}" + name = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-table-${module.naming_convention.suffix}", "-", "_") data_factory_id = var.data_factory_id + folder = "${var.storage_accounts.source.name}/table" activities_json = jsonencode( [ @@ -23,21 +24,30 @@ resource "azurerm_data_factory_pipeline" "pipeline_table" { azureTableSourceIgnoreTableNotFound = false } sink = { - type = "AzureTableSink" - writeBatchSize = 10000 - writeBatchTimeout = "00:02:00" + type = "AzureTableSink" + writeBatchSize = 10000 + writeBatchTimeout = "00:02:00" + azureTableInsertType = "merge", + azureTablePartitionKeyName = { + value = "PartitionKey", + type = "Expression" + }, + azureTableRowKeyName = { + value = "RowKey", + type = "Expression" + }, } enableStaging = false } inputs = [ { - referenceName = azurerm_data_factory_custom_dataset.source_dataset_table[each.value] + referenceName = azurerm_data_factory_custom_dataset.source_dataset_table[each.value].name type = "DatasetReference" } ] outputs = [ { - referenceName = azurerm_data_factory_custom_dataset.target_dataset_table[each.value] + referenceName = azurerm_data_factory_custom_dataset.target_dataset_table[each.value].name type = "DatasetReference" } ] diff --git a/src/migration/prod/README.md b/src/migration/prod/README.md index b20548be9..37297372f 100644 --- a/src/migration/prod/README.md +++ b/src/migration/prod/README.md @@ -34,5 +34,8 @@ No inputs. ## Outputs -No outputs. +| Name | Description | +|------|-------------| +| [data\_factory](#output\_data\_factory) | n/a | +| [data\_factory\_st\_pipelines](#output\_data\_factory\_st\_pipelines) | n/a | diff --git a/src/migration/prod/italynorth.tf b/src/migration/prod/italynorth.tf index 5c5bfc9a4..2d777fb85 100644 --- a/src/migration/prod/italynorth.tf +++ b/src/migration/prod/italynorth.tf @@ -1,5 +1,5 @@ resource "azurerm_resource_group" "migration" { - name = "${local.project_itn}-${local.environment.domain}-${local.environment.app_name}-rg-${local.environment.instance_number}" + name = "${local.project_itn}-${local.environment.app_name}-rg-${local.environment.instance_number}" location = "italynorth" tags = local.tags @@ -8,7 +8,7 @@ resource "azurerm_resource_group" "migration" { # Create Azure Data Factory instances # Enables system-assigned managed identity for secure access to resources resource "azurerm_data_factory" "this" { - name = "${local.project_itn}-${local.environment.domain}-${local.environment.app_name}-adf-${local.environment.instance_number}" + name = "${local.project_itn}-${local.environment.app_name}-adf-${local.environment.instance_number}" location = "italynorth" resource_group_name = azurerm_resource_group.migration.name @@ -23,7 +23,7 @@ resource "azurerm_data_factory" "this" { } resource "azurerm_data_factory_integration_runtime_azure" "azure_runtime" { - name = "${local.project_itn}-${local.environment.domain}-${local.environment.app_name}-adfir-${local.environment.instance_number}" + name = "${local.project_itn}-${local.environment.app_name}-adfir-${local.environment.instance_number}" location = "italynorth" data_factory_id = azurerm_data_factory.this.id } @@ -44,12 +44,12 @@ module "migrate_storage_accounts" { what_to_migrate = { blob = { - enabled = each.value.blob.enabled + enabled = try(each.value.blob.enabled, true) containers = try(each.value.blob.containers, []) } table = { - enabled = each.value.table.enabled + enabled = try(each.value.table.enabled, true) tables = try(each.value.table.tables, []) } } diff --git a/src/migration/prod/locals.tf b/src/migration/prod/locals.tf index b6ec2f32f..5717536bb 100644 --- a/src/migration/prod/locals.tf +++ b/src/migration/prod/locals.tf @@ -11,8 +11,7 @@ locals { prefix = local.prefix env_short = local.env_short location = "italynorth" - domain = "eng" - app_name = "migitn" + app_name = "migration" instance_number = "01" } @@ -25,6 +24,16 @@ locals { } storage_accounts = [ + { + source = { + name = "stdevbiptest1" + resource_group_name = "RG-BIP-DEV-TEST" + } + target = { + name = "stbipdevtest" + resource_group_name = "dev-fasanorg" + } + } # Copy both containers and tables # { # source = { name = "stdevbiptest1", resource_group_name = "RG-BIP-DEV-TEST" } diff --git a/src/migration/prod/main.tf b/src/migration/prod/main.tf index fcca7f12c..a32d099f8 100644 --- a/src/migration/prod/main.tf +++ b/src/migration/prod/main.tf @@ -1,11 +1,11 @@ terraform { - backend "azurerm" { - resource_group_name = "terraform-state-rg" - storage_account_name = "iopitntfst001" - container_name = "terraform-state" - key = "io-infra.migration.prod.italynorth.tfstate" - } + # backend "azurerm" { + # resource_group_name = "terraform-state-rg" + # storage_account_name = "iopitntfst001" + # container_name = "terraform-state" + # key = "io-infra.migration.prod.italynorth.tfstate" + # } required_providers { azurerm = { diff --git a/src/migration/prod/outputs.tf b/src/migration/prod/outputs.tf index e69de29bb..cc35cc082 100644 --- a/src/migration/prod/outputs.tf +++ b/src/migration/prod/outputs.tf @@ -0,0 +1,11 @@ +output "data_factory" { + value = { + id = azurerm_data_factory.this.id + name = azurerm_data_factory.this.name + resource_group_name = azurerm_data_factory.this.resource_group_name + } +} + +output "data_factory_st_pipelines" { + value = { for migration in local.storage_accounts : "${migration.source.name}|${migration.target.name}" => module.migrate_storage_accounts["${migration.source.name}|${migration.target.name}"].pipelines } +} \ No newline at end of file From 088d4686236b3c5e5c00a232a5746894395289b8 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 17 Oct 2024 12:56:48 +0200 Subject: [PATCH 18/22] feat: substitute dollar sign with underscore in adf resource names --- .../data_factory_storage_account/datasets_containers.tf | 8 ++++---- .../data_factory_storage_account/datasets_tables.tf | 8 ++++---- .../data_factory_storage_account/pipeline_containers.tf | 4 ++-- .../data_factory_storage_account/pipeline_tables.tf | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/_modules/data_factory_storage_account/datasets_containers.tf b/src/_modules/data_factory_storage_account/datasets_containers.tf index 33b505ce4..6e4080cc6 100644 --- a/src/_modules/data_factory_storage_account/datasets_containers.tf +++ b/src/_modules/data_factory_storage_account/datasets_containers.tf @@ -1,9 +1,9 @@ resource "azurerm_data_factory_custom_dataset" "source_dataset_container" { for_each = toset(local.containers) - name = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-blob-${module.naming_convention.suffix}", "-", "_") + name = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-blob-${module.naming_convention.suffix}", "/[$-]/", "_") data_factory_id = var.data_factory_id type = "AzureBlob" - folder = "${var.storage_accounts.source.name}/source/blob" + folder = "${var.storage_accounts.source.name}/source/blob" linked_service { name = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_blob[0].name @@ -21,10 +21,10 @@ resource "azurerm_data_factory_custom_dataset" "source_dataset_container" { resource "azurerm_data_factory_custom_dataset" "target_dataset_container" { for_each = toset(local.containers) - name = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value}-blob-${module.naming_convention.suffix}", "-", "_") + name = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value}-blob-${module.naming_convention.suffix}", "/[$-]/", "_") data_factory_id = var.data_factory_id type = "AzureBlob" - folder = "${var.storage_accounts.source.name}/target/blob" + folder = "${var.storage_accounts.source.name}/target/blob" linked_service { name = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_blob[0].name diff --git a/src/_modules/data_factory_storage_account/datasets_tables.tf b/src/_modules/data_factory_storage_account/datasets_tables.tf index 85923189e..17a0dd8c3 100644 --- a/src/_modules/data_factory_storage_account/datasets_tables.tf +++ b/src/_modules/data_factory_storage_account/datasets_tables.tf @@ -1,9 +1,9 @@ resource "azurerm_data_factory_custom_dataset" "source_dataset_table" { for_each = toset(local.tables) - name = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-table-${module.naming_convention.suffix}", "-", "_") + name = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-table-${module.naming_convention.suffix}", "/[$-]/", "_") data_factory_id = var.data_factory_id type = "AzureTable" - folder = "${var.storage_accounts.source.name}/source/table" + folder = "${var.storage_accounts.source.name}/source/table" linked_service { name = azurerm_data_factory_linked_service_azure_table_storage.source_linked_service_table[0].name @@ -16,10 +16,10 @@ resource "azurerm_data_factory_custom_dataset" "source_dataset_table" { resource "azurerm_data_factory_custom_dataset" "target_dataset_table" { for_each = toset(local.tables) - name = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value}-table-${module.naming_convention.suffix}", "-", "_") + name = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value}-table-${module.naming_convention.suffix}", "/[$-]/", "_") data_factory_id = var.data_factory_id type = "AzureTable" - folder = "${var.storage_accounts.source.name}/target/table" + folder = "${var.storage_accounts.source.name}/target/table" linked_service { name = azurerm_data_factory_linked_service_azure_table_storage.target_linked_service_table[0].name } diff --git a/src/_modules/data_factory_storage_account/pipeline_containers.tf b/src/_modules/data_factory_storage_account/pipeline_containers.tf index dc6a3d996..7828c686f 100644 --- a/src/_modules/data_factory_storage_account/pipeline_containers.tf +++ b/src/_modules/data_factory_storage_account/pipeline_containers.tf @@ -1,8 +1,8 @@ resource "azurerm_data_factory_pipeline" "pipeline_container" { for_each = toset(local.containers) - name = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-blob-${module.naming_convention.suffix}", "-", "_") + name = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-blob-${module.naming_convention.suffix}", "/[$-]/", "_") data_factory_id = var.data_factory_id - folder = "${var.storage_accounts.source.name}/blob" + folder = "${var.storage_accounts.source.name}/blob" activities_json = jsonencode( [ diff --git a/src/_modules/data_factory_storage_account/pipeline_tables.tf b/src/_modules/data_factory_storage_account/pipeline_tables.tf index b6d52fef2..54f086b77 100644 --- a/src/_modules/data_factory_storage_account/pipeline_tables.tf +++ b/src/_modules/data_factory_storage_account/pipeline_tables.tf @@ -1,8 +1,8 @@ resource "azurerm_data_factory_pipeline" "pipeline_table" { for_each = toset(local.tables) - name = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-table-${module.naming_convention.suffix}", "-", "_") + name = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-table-${module.naming_convention.suffix}", "/[$-]/", "_") data_factory_id = var.data_factory_id - folder = "${var.storage_accounts.source.name}/table" + folder = "${var.storage_accounts.source.name}/table" activities_json = jsonencode( [ From 1190c2bea7a293ba127ed26dfbb6d106d98ba0bf Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 17 Oct 2024 12:58:09 +0200 Subject: [PATCH 19/22] fix: re added state configuration --- src/migration/prod/main.tf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/migration/prod/main.tf b/src/migration/prod/main.tf index a32d099f8..fcca7f12c 100644 --- a/src/migration/prod/main.tf +++ b/src/migration/prod/main.tf @@ -1,11 +1,11 @@ terraform { - # backend "azurerm" { - # resource_group_name = "terraform-state-rg" - # storage_account_name = "iopitntfst001" - # container_name = "terraform-state" - # key = "io-infra.migration.prod.italynorth.tfstate" - # } + backend "azurerm" { + resource_group_name = "terraform-state-rg" + storage_account_name = "iopitntfst001" + container_name = "terraform-state" + key = "io-infra.migration.prod.italynorth.tfstate" + } required_providers { azurerm = { From f43ca19b9995fe83421e6133c8e7b3d11e68a78d Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 17 Oct 2024 13:25:03 +0200 Subject: [PATCH 20/22] fix: comment --- src/_modules/data_factory_storage_account/iam.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_modules/data_factory_storage_account/iam.tf b/src/_modules/data_factory_storage_account/iam.tf index 45bb6c83e..6f68a6c00 100644 --- a/src/_modules/data_factory_storage_account/iam.tf +++ b/src/_modules/data_factory_storage_account/iam.tf @@ -31,7 +31,7 @@ module "roles" { ] : [] } -# add to var.data_factory_principal_id the Storage Account Contributor role using the azurerm_role_assignment resourcew +# Permission needed to allow Data Factory to create tables in the target storage account resource "azurerm_role_assignment" "storage_account_contributor" { count = var.what_to_migrate.table.enabled ? 1 : 0 scope = data.azurerm_storage_account.target.id From 3f68237602ec243ba1e1bc099686f702f3a669a7 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 17 Oct 2024 14:38:20 +0200 Subject: [PATCH 21/22] fix: deleted test storage accounts --- src/migration/prod/locals.tf | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/migration/prod/locals.tf b/src/migration/prod/locals.tf index 5717536bb..5cd76e9d7 100644 --- a/src/migration/prod/locals.tf +++ b/src/migration/prod/locals.tf @@ -24,16 +24,6 @@ locals { } storage_accounts = [ - { - source = { - name = "stdevbiptest1" - resource_group_name = "RG-BIP-DEV-TEST" - } - target = { - name = "stbipdevtest" - resource_group_name = "dev-fasanorg" - } - } # Copy both containers and tables # { # source = { name = "stdevbiptest1", resource_group_name = "RG-BIP-DEV-TEST" } From 02bc83b1cfaeb7bc4df4c4052e1f018d06697845 Mon Sep 17 00:00:00 2001 From: Mario Mupo Date: Thu, 17 Oct 2024 18:29:16 +0200 Subject: [PATCH 22/22] fix: updated lock with platforms --- src/migration/prod/.terraform.lock.hcl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/migration/prod/.terraform.lock.hcl b/src/migration/prod/.terraform.lock.hcl index 18e707f28..54894b893 100644 --- a/src/migration/prod/.terraform.lock.hcl +++ b/src/migration/prod/.terraform.lock.hcl @@ -5,6 +5,9 @@ provider "registry.terraform.io/azure/azapi" { version = "1.15.0" constraints = "<= 1.15.0" hashes = [ + "h1:5aoSqVISTygtAD42asvbglV/bMqjMvTA2RmuuPz87Ic=", + "h1:W41dtPI1BFKkDtLbKWLxGJ4L5ntAFZ5BJYZT+04+jk4=", + "h1:Y7ruMuPh8UJRTRl4rm+cdpGtmURx2taqiuqfYaH3o48=", "h1:pO/phGY+TxMEKQ+ffYj+vUIvG5A1tno/sZYDb/yyA/w=", "zh:0627a8bc77254debc25dc0c7b62e055138217c97b03221e593c3c56dc7550671", "zh:2fe045f07070ef75d0bec4b0595a74c14394daa838ddb964e2fd23cc98c40c34", @@ -25,7 +28,10 @@ provider "registry.terraform.io/hashicorp/azurerm" { version = "3.116.0" constraints = "~> 3.106, <= 3.116.0" hashes = [ + "h1:2QbjtN4oMXzdA++Nvrj/wSmWZTPgXKOSFGGQCLEMrb4=", "h1:BCR3NIorFSvGG3v/+JOiiw3VM4PkChLO4m84wzD9NDo=", + "h1:SJM/KQDW9blKFmLMaupsZVYtcZ0fYpjLHEriMgCBGCY=", + "h1:jwwbQ09fH1RdcNsknt1AkvfSUbULsl7nZQn6S8fabFI=", "zh:02b6606aff025fc2a962b3e568e000300abe959adac987183c24dac8eb057f4d", "zh:2a23a8ce24ff9e885925ffee0c3ea7eadba7a702541d05869275778aa47bdea7", "zh:57d10746384baeca4d5c56e88872727cdc150f437b8c5e14f0542127f7475e24",