feat: added the example list

src/_modules/data_factory_storage_account/data.tf

@@ -9,6 +9,12 @@ data "azurerm_storage_account" "target" {
 }
 
 data "azurerm_storage_containers" "this" {
-  for_each           = var.what_to_migrate.blob.enabled && length(var.what_to_migrate.blob.containers) == 0 ? [1] : []
+  count              = var.what_to_migrate.blob.enabled && length(var.what_to_migrate.blob.containers) == 0 ? 1 : 0
   storage_account_id = data.azurerm_storage_account.source.id
+}
+
+data "azapi_resource_list" "tables" {
+  type                   = "Microsoft.Storage/storageAccounts/tableServices/tables@2021-09-01"
+  parent_id              = "${data.azurerm_storage_account.source.id}/tableServices/default"
+  response_export_values = ["*"]
 }

src/_modules/data_factory_storage_account/datasets_containers.tf

@@ -1,16 +1,16 @@
 resource "azurerm_data_factory_custom_dataset" "source_dataset_container" {
   for_each        = local.containers
   name            = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value.name}-blob-${module.naming_convention.suffix}"
-  data_factory_id = var.data_factory.id
+  data_factory_id = var.data_factory_id
   type            = "AzureBlob"
 
   linked_service {
-    name = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_blob.name
+    name = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_blob[0].name
   }
 
   type_properties_json = jsonencode({
     linkedServiceName = {
-      referenceName = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_blob.name
+      referenceName = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_blob[0].name
       type          = "LinkedServiceReference"
     }
     type       = "AzureBlob"
@@ -21,16 +21,16 @@ resource "azurerm_data_factory_custom_dataset" "source_dataset_container" {
 resource "azurerm_data_factory_custom_dataset" "target_dataset_container" {
   for_each        = local.containers
   name            = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value.name}-blob-${module.naming_convention.suffix}"
-  data_factory_id = var.data_factory.id
+  data_factory_id = var.data_factory_id
   type            = "AzureBlob"
 
   linked_service {
-    name = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_blob.name
+    name = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_blob[0].name
   }
 
   type_properties_json = jsonencode({
     linkedServiceName = {
-      referenceName = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_blob.name
+      referenceName = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_blob[0].name
       type          = "LinkedServiceReference"
     }
     type       = "AzureBlob"

src/_modules/data_factory_storage_account/datasets_tables.tf

@@ -1,11 +1,11 @@
 resource "azurerm_data_factory_custom_dataset" "source_dataset_table" {
   for_each        = local.tables
   name            = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value.name}-table-${module.naming_convention.suffix}"
-  data_factory_id = var.data_factory.id
+  data_factory_id = var.data_factory_id
   type            = "AzureTable"
 
   linked_service {
-    name = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_table.name
+    name = azurerm_data_factory_linked_service_azure_table_storage.source_linked_service_table[0].name
   }
 
   type_properties_json = jsonencode({
@@ -16,11 +16,11 @@ resource "azurerm_data_factory_custom_dataset" "source_dataset_table" {
 resource "azurerm_data_factory_custom_dataset" "target_dataset_table" {
   for_each        = local.tables
   name            = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value.name}-table-${module.naming_convention.suffix}"
-  data_factory_id = var.data_factory.id
+  data_factory_id = var.data_factory_id
   type            = "AzureTable"
 
   linked_service {
-    name = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_table.name
+    name = azurerm_data_factory_linked_service_azure_table_storage.target_linked_service_table[0].name
   }
 
   type_properties_json = jsonencode({

src/_modules/data_factory_storage_account/iam.tf

@@ -1,6 +1,6 @@
 module "roles" {
   source       = "github.com/pagopa/dx//infra/modules/azure_role_assignments?ref=main"
-  principal_id = var.function_app.user_func_02.principal_id
+  principal_id = var.data_factory_principal_id
 
   storage_blob = var.what_to_migrate.blob.enabled ? [
     {

src/_modules/data_factory_storage_account/linked_services_containers.tf

@@ -1,17 +1,17 @@
 resource "azurerm_data_factory_linked_service_azure_blob_storage" "source_linked_service_blob" {
-  for_each        = var.what_to_migrate.blob.enabled ? [1] : []
+  count           = var.what_to_migrate.blob.enabled ? 1 : 0
   name            = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-blob-${module.naming_convention.suffix}"
-  data_factory_id = var.data_factory.id
+  data_factory_id = var.data_factory_id
 
   service_endpoint = "https://${data.azurerm_storage_account.source.id}.blob.core.windows.net"
 
   use_managed_identity = true
 }
 
 resource "azurerm_data_factory_linked_service_azure_blob_storage" "target_linked_service_blob" {
-  for_each        = var.what_to_migrate.blob.enabled ? [1] : []
+  count           = var.what_to_migrate.blob.enabled ? 1 : 0
   name            = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-blob-${module.naming_convention.suffix}"
-  data_factory_id = var.data_factory.id
+  data_factory_id = var.data_factory_id
 
   service_endpoint = "https://${data.azurerm_storage_account.target.id}.blob.core.windows.net"
 

src/_modules/data_factory_storage_account/linked_services_tables.tf

@@ -1,15 +1,15 @@
 resource "azurerm_data_factory_linked_service_azure_table_storage" "source_linked_service_table" {
-  for_each        = var.what_to_migrate.table.enabled ? [1] : []
+  count           = var.what_to_migrate.table.enabled ? 1 : 0
   name            = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-table-${module.naming_convention.suffix}"
-  data_factory_id = var.data_factory.id
+  data_factory_id = var.data_factory_id
 
   connection_string = data.azurerm_storage_account.source.primary_table_endpoint
 }
 
 resource "azurerm_data_factory_linked_service_azure_table_storage" "target_linked_service_table" {
-  for_each        = var.what_to_migrate.table.enabled ? [1] : []
+  count           = var.what_to_migrate.table.enabled ? 1 : 0
   name            = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-table-${module.naming_convention.suffix}"
-  data_factory_id = var.data_factory.id
+  data_factory_id = var.data_factory_id
 
   connection_string = data.azurerm_storage_account.target.primary_table_endpoint
 }

src/_modules/data_factory_storage_account/locals.tf

@@ -1,4 +1,4 @@
 locals {
-  containers = var.what_to_migrate.blob.enabled ? length(var.what_to_migrate.blob.containers) > 0 ? var.what_to_migrate.blob.containers : [for container in data.azurerm_storage_containers.this[0].containers : container.name] : []
-  tables     = var.what_to_migrate.table.enabled ? var.what_to_migrate.table.tables : []
+  containers = var.what_to_migrate.blob.enabled ? (length(var.what_to_migrate.blob.containers) > 0 ? var.what_to_migrate.blob.containers : [for container in data.azurerm_storage_containers.this[0].containers : container.name]) : []
+  tables     = var.what_to_migrate.table.enabled ? (length(var.what_to_migrate.table.tables) > 0 ? var.what_to_migrate.table.tables : [for table in jsondecode(data.azapi_resource_list.tables.output).value : table.TableName]) : []
 }

src/_modules/data_factory_storage_account/main.tf

@@ -1,3 +1,12 @@
+terraform {
+  required_providers {
+    azapi = {
+      source  = "Azure/azapi"
+      version = "<= 1.15.0"
+    }
+  }
+}
+
 module "naming_convention" {
   source = "github.com/pagopa/dx//infra/modules/azure_naming_convention/?ref=main"
 

src/_modules/data_factory_storage_account/network.tf

@@ -0,0 +1,27 @@
+resource "azurerm_data_factory_managed_private_endpoint" "blob_source" {
+  name               = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-blob-${module.naming_convention.suffix}"
+  data_factory_id    = var.data_factory_id
+  target_resource_id = data.azurerm_storage_account.source.id
+  subresource_name   = "blob"
+}
+
+resource "azurerm_data_factory_managed_private_endpoint" "blob_target" {
+  name               = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-blob-${module.naming_convention.suffix}"
+  data_factory_id    = var.data_factory_id
+  target_resource_id = data.azurerm_storage_account.target.id
+  subresource_name   = "blob"
+}
+
+resource "azurerm_data_factory_managed_private_endpoint" "table_source" {
+  name               = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-table-${module.naming_convention.suffix}"
+  data_factory_id    = var.data_factory_id
+  target_resource_id = data.azurerm_storage_account.source.id
+  subresource_name   = "table"
+}
+
+resource "azurerm_data_factory_managed_private_endpoint" "table_target" {
+  name               = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-table-${module.naming_convention.suffix}"
+  data_factory_id    = var.data_factory_id
+  target_resource_id = data.azurerm_storage_account.target.id
+  subresource_name   = "table"
+}

src/_modules/data_factory_storage_account/pipeline_containers.tf

@@ -1,11 +1,7 @@
 resource "azurerm_data_factory_pipeline" "pipeline_container" {
   for_each        = local.containers
   name            = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value.name}-blob-${module.naming_convention.suffix}"
-  data_factory_id = var.data_factory.id
-
-  depends_on = [
-    azurerm_data_factory_custom_dataset.dataset_container
-  ]
+  data_factory_id = var.data_factory_id
 
   activities_json = jsonencode(
     [

src/_modules/data_factory_storage_account/pipeline_tables.tf

@@ -1,11 +1,7 @@
 resource "azurerm_data_factory_pipeline" "pipeline_table" {
   for_each        = local.tables
   name            = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value.name}-table-${module.naming_convention.suffix}"
-  data_factory_id = var.data_factory.id
-
-  depends_on = [
-    azurerm_data_factory_custom_dataset.dataset_table
-  ]
+  data_factory_id = var.data_factory_id
 
   activities_json = jsonencode(
     [

src/_modules/data_factory_storage_account/variables.tf

@@ -16,6 +16,11 @@ variable "data_factory_id" {
   type        = string
 }
 
+variable "data_factory_principal_id" {
+  description = "Data Factory principal id to grant access to."
+  type        = string
+}
+
 variable "storage_accounts" {
   type = object({
     source = object({
@@ -42,7 +47,7 @@ variable "what_to_migrate" {
     table = optional(object(
       {
         enabled = bool
-        tables  = list(string)
+        tables  = optional(list(string), [])
       }),
       { enabled = false }
     )
@@ -54,11 +59,5 @@ variable "what_to_migrate" {
     error_message = "At least one between blob and table should be enabled."
   }
 
-  # validate that if table is enabled, at least one table is specified
-  validation {
-    condition     = !(var.what_to_migrate.table.enabled && length(var.what_to_migrate.table.tables) == 0)
-    error_message = "If table is enabled, at least one table should be specified."
-  }
-
-  description = "List of databases, file shares, containers and tables to migrate."
+  description = "List of storage account containers and tables to migrate."
 }

src/migration/prod/README.md

@@ -5,13 +5,14 @@
 
 | Name | Version |
 |------|---------|
+| <a name="requirement_azapi"></a> [azapi](#requirement\_azapi) | <= 1.15.0 |
 | <a name="requirement_azurerm"></a> [azurerm](#requirement\_azurerm) | <= 3.112.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_azurerm"></a> [azurerm](#provider\_azurerm) | <= 3.112.0 |
+| <a name="provider_azurerm"></a> [azurerm](#provider\_azurerm) | 3.112.0 |
 
 ## Modules
 

src/migration/prod/italynorth.tf

@@ -1,5 +1,5 @@
 resource "azurerm_resource_group" "migration" {
-  name     = "${local.project_itn}-migration-rg-01"
+  name     = "${local.project_itn}-${local.environment.domain}-${local.environment.app_name}-rg-01"
   location = "italynorth"
 
   tags = local.tags
@@ -8,7 +8,7 @@ resource "azurerm_resource_group" "migration" {
 # Create Azure Data Factory instances
 # Enables system-assigned managed identity for secure access to resources
 resource "azurerm_data_factory" "this" {
-  name                = "${local.project_itn}-migration-adf-01"
+  name                = "${local.project_itn}-${local.environment.domain}-${local.environment.app_name}-adf-01"
   location            = "italynorth"
   resource_group_name = azurerm_resource_group.migration.name
 
@@ -20,25 +20,19 @@ resource "azurerm_data_factory" "this" {
 }
 
 resource "azurerm_data_factory_integration_runtime_azure" "azure_runtime" {
-  name            = "${local.project_itn}-migration-adfir-01"
+  name            = "${local.project_itn}-${local.environment.domain}-${local.environment.app_name}-adfir-01"
   location        = "italynorth"
   data_factory_id = azurerm_data_factory.this.id
 }
 
 module "migrate_storage_accounts" {
-  for_each = local.storage_accounts
+  for_each = { for migration in local.storage_accounts : "${migration.source.name}|${migration.target.name}" => migration }
   source   = "../../_modules/data_factory_storage_account"
 
-  environment = {
-    prefix          = local.prefix
-    env_short       = local.env_short
-    location        = "italynorth"
-    domain          = "eng"
-    app_name        = "mig"
-    instance_number = "01"
-  }
+  environment = local.environment
 
-  data_factory_id = azurerm_data_factory.this.id
+  data_factory_id           = azurerm_data_factory.this.id
+  data_factory_principal_id = azurerm_data_factory.this.identity[0].principal_id
 
   storage_accounts = {
     source = each.value.source
@@ -48,12 +42,12 @@ module "migrate_storage_accounts" {
   what_to_migrate = {
     blob = {
       enabled    = each.value.blob.enabled
-      containers = each.value.blob.containers
+      containers = try(each.value.blob.containers, [])
     }
 
     table = {
       enabled = each.value.table.enabled
-      tables  = each.value.table.tables
+      tables  = try(each.value.table.tables, [])
     }
   }
 }

src/migration/prod/locals.tf

@@ -7,11 +7,29 @@ locals {
   project_weu_legacy = "${local.prefix}-${local.env_short}"
   secondary_project  = "${local.prefix}-${local.env_short}-${local.location_short.germanywestcentral}"
 
+  environment = {
+    prefix          = local.prefix
+    env_short       = local.env_short
+    location        = "italynorth"
+    domain          = "eng"
+    app_name        = "migitn"
+    instance_number = "01"
+  }
+
   tags = {
     CostCenter  = "TS310 - PAGAMENTI & SERVIZI"
     CreatedBy   = "Terraform"
     Environment = "Prod"
     Owner       = "IO"
     Source      = "https://github.com/pagopa/io-infra/blob/main/src/migration/prod"
   }
+
+  storage_accounts = [
+    {
+      source = { name = "iopweuabc", resource_group_name = "abc" }
+      target = { name = "iopitnabc", resource_group_name = "abc" }
+      blob   = { enabled = true }
+      table  = { enabled = true }
+    }
+  ]
 }