diff --git a/README.md b/README.md index 5d7e8f51..2afb9647 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,40 @@ private_dns_rg_name = "myResourceGroup" ``` **If you don't pass these params, we will automatically create the network resources for you.** +## Storage account +### We create/use the following storage accounts +- Logic app storage account - Stores the logic app configuration. Created by our module. +- Deployment storage account - Stores the deployment states (cluster and also NFS if configured). Created by our module if not provided. +- Weka OBS storage account - Created by our function app if OBS is configured and OBS storage account is not provided. + +### Storage account networking options +```hcl +variable "storage_account_public_network_access" { + type = string + description = "Public network access to the storage accounts." + default = "Enabled" + + validation { + condition = contains(["Enabled", "Disabled", "EnabledForVnet"], var.storage_account_public_network_access) + error_message = "Allowed values: [\"Enabled\", \"Disabled\", \"EnabledForVnet\"]." + } +} +``` +- `Enabled`: By default, the storage account is created with public network access enabled. +- `EnabledForVnet`: The storage account is created with public network access enabled, but only for the specified virtual network. + - Access should be enabled for the vnet, function app subnet delegation. + - File share is required (can provide existing via `deployment_file_share_name` or it will be auto-created in case if `storage_account_allowed_ips` are provided). + - `storage_account_allowed_ips`: required to allow creating the logic app storage account with the required config and function app file share. + - if `storage_account_allowed_ips` if not provided, scale down and autoscaling will not be supported and the file share needs to be created by the user. + - OBS storage account: if created by our module only the regular [OBS](#OBS) config is required. If provided by the user needs to have the Vnet enabled. +- `Disabled`: The storage account is created with public network access disabled. + - Scale down and autoscaling is not supported. + - Pre created deployment storage account is required. + - File share is required (`deployment_file_share_name`). + - Blob and file endpoints and private links are required. It can be created by our module if `create_storage_account_private_links` is provided or by the user. In case if there are existing private endpoints and `create_storage_account_private_links` is not set, `storage_blob_private_dns_zone_name` can be also set to specify private DNS zone for blob resource (uses Azure-recommended name as default value). + - OBS storage account: if created by our module only the regular [OBS](#OBS) config is required. If provided by the user, blob and file endpoints and private links are required. +They can be created by our module if `create_storage_account_private_links` is provided. + ## Usage example ```hcl provider "azurerm" { @@ -295,18 +329,16 @@ proxy_url = VALUE | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.4.6 | -| [azurerm](#requirement\_azurerm) | ~>3.75.0 | +| [azurerm](#requirement\_azurerm) | ~>3.114.0 | | [local](#requirement\_local) | ~>2.4.0 | -| [random](#requirement\_random) | ~>3.5.1 | | [tls](#requirement\_tls) | ~>4.0.4 | ## Providers | Name | Version | |------|---------| -| [azurerm](#provider\_azurerm) | ~>3.75.0 | +| [azurerm](#provider\_azurerm) | ~>3.114.0 | | [local](#provider\_local) | ~>2.4.0 | -| [random](#provider\_random) | ~>3.5.1 | | [tls](#provider\_tls) | ~>4.0.4 | ## Modules @@ -314,7 +346,10 @@ proxy_url = VALUE | Name | Source | Version | |------|--------|---------| | [clients](#module\_clients) | ./modules/clients | n/a | +| [function\_app\_subnet\_delegation](#module\_function\_app\_subnet\_delegation) | ./modules/subnet_delegation | n/a | | [iam](#module\_iam) | ./modules/iam | n/a | +| [logic\_app\_subnet\_delegation](#module\_logic\_app\_subnet\_delegation) | ./modules/subnet_delegation | n/a | +| [logicapp](#module\_logicapp) | ./modules/logic_app | n/a | | [network](#module\_network) | ./modules/network | n/a | | [nfs\_protocol\_gateways](#module\_nfs\_protocol\_gateways) | ./modules/protocol_gateways | n/a | | [peering](#module\_peering) | ./modules/peering_vnets | n/a | @@ -327,13 +362,13 @@ proxy_url = VALUE |------|------| | [azurerm_application_insights.application_insights](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/application_insights) | resource | | [azurerm_key_vault.key_vault](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault) | resource | -| [azurerm_key_vault_access_policy.function_app_get_secret_permission](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | +| [azurerm_key_vault_access_policy.function_app_secret_permissions](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | | [azurerm_key_vault_access_policy.key_vault_access_policy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | -| [azurerm_key_vault_access_policy.standard_logic_app_get_secret_permission](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | | [azurerm_key_vault_secret.function_app_default_key](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | | [azurerm_key_vault_secret.get_weka_io_token](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | | [azurerm_key_vault_secret.private_ssh_keys](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | | [azurerm_key_vault_secret.public_ssh_keys](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_key_vault_secret.weka_deployment_password](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | | [azurerm_key_vault_secret.weka_password_secret](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | | [azurerm_lb.backend_lb](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/lb) | resource | | [azurerm_lb.ui_lb](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/lb) | resource | @@ -345,7 +380,6 @@ proxy_url = VALUE | [azurerm_lb_rule.ui_lb_rule](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/lb_rule) | resource | | [azurerm_linux_function_app.function_app](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/linux_function_app) | resource | | [azurerm_log_analytics_workspace.la_workspace](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/log_analytics_workspace) | resource | -| [azurerm_logic_app_standard.logic_app_standard](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/logic_app_standard) | resource | | [azurerm_monitor_diagnostic_setting.function_diagnostic_setting](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_diagnostic_setting) | resource | | [azurerm_monitor_diagnostic_setting.insights_diagnostic_setting](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_diagnostic_setting) | resource | | [azurerm_private_dns_a_record.dns_a_record_backend_lb](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/private_dns_a_record) | resource | @@ -354,39 +388,36 @@ proxy_url = VALUE | [azurerm_private_dns_resolver_forwarding_rule.resolver_forwarding_rule](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/private_dns_resolver_forwarding_rule) | resource | | [azurerm_private_dns_resolver_outbound_endpoint.outbound_endpoint](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/private_dns_resolver_outbound_endpoint) | resource | | [azurerm_private_dns_resolver_virtual_network_link.dns_forwarding_virtual_network_link](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/private_dns_resolver_virtual_network_link) | resource | +| [azurerm_private_dns_zone.blob](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/private_dns_zone) | resource | +| [azurerm_private_dns_zone.file](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/private_dns_zone) | resource | +| [azurerm_private_dns_zone_virtual_network_link.blob_privatelink](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/private_dns_zone_virtual_network_link) | resource | +| [azurerm_private_dns_zone_virtual_network_link.file_privatelink](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/private_dns_zone_virtual_network_link) | resource | +| [azurerm_private_endpoint.blob_endpoint](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/private_endpoint) | resource | +| [azurerm_private_endpoint.file_endpoint](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/private_endpoint) | resource | +| [azurerm_private_endpoint.weka_obs_blob_endpoint](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/private_endpoint) | resource | | [azurerm_proximity_placement_group.ppg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/proximity_placement_group) | resource | | [azurerm_public_ip.backend_ip](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/public_ip) | resource | | [azurerm_public_ip.ui_ip](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/public_ip) | resource | | [azurerm_service_plan.app_service_plan](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/service_plan) | resource | -| [azurerm_service_plan.logicapp_service_plan](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/service_plan) | resource | | [azurerm_storage_account.deployment_sa](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_account) | resource | | [azurerm_storage_account.logicapp](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_account) | resource | | [azurerm_storage_blob.nfs_state](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_blob) | resource | | [azurerm_storage_blob.state](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_blob) | resource | -| [azurerm_storage_blob.vmss_config](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_blob) | resource | | [azurerm_storage_container.deployment](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_container) | resource | | [azurerm_storage_container.nfs_deployment](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_container) | resource | -| [azurerm_storage_share_directory.share_directory_scale_down](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_share_directory) | resource | -| [azurerm_storage_share_directory.share_directory_scale_up](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_share_directory) | resource | -| [azurerm_storage_share_file.connections_share_file](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_share_file) | resource | -| [azurerm_storage_share_file.scale_down_share_file](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_share_file) | resource | -| [azurerm_storage_share_file.scale_up_share_file](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_share_file) | resource | +| [azurerm_storage_share.function_app_share](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_share) | resource | | [azurerm_subnet.dns_resolver_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet) | resource | -| [azurerm_subnet.logicapp_subnet_delegation](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet) | resource | -| [azurerm_subnet.subnet_delegation](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet) | resource | -| [local_file.connections_workflow_file](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.private_key](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.public_key](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [local_file.scale_down_workflow_file](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [local_file.scale_up_workflow_file](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [random_password.weka_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | | [tls_private_key.ssh_key](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource | | [azurerm_application_insights.application_insights](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/application_insights) | data source | | [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source | | [azurerm_function_app_host_keys.function_keys](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/function_app_host_keys) | data source | +| [azurerm_private_dns_zone.blob](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/private_dns_zone) | data source | | [azurerm_resource_group.rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | | [azurerm_storage_account.deployment_blob](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/storage_account) | data source | -| [azurerm_storage_share.storage_share](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/storage_share) | data source | +| [azurerm_storage_account.weka_obs](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/storage_account) | data source | +| [azurerm_storage_account_blob_container_sas.function_app_code_sas](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/storage_account_blob_container_sas) | data source | | [azurerm_subnet.subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subnet) | data source | | [azurerm_virtual_network.vnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/virtual_network) | data source | @@ -409,37 +440,40 @@ proxy_url = VALUE | [clients\_custom\_data](#input\_clients\_custom\_data) | Custom data to pass to the client instances | `string` | `""` | no | | [clients\_number](#input\_clients\_number) | The number of client virtual machines to deploy. | `number` | `0` | no | | [clients\_use\_dpdk](#input\_clients\_use\_dpdk) | Mount weka clients in DPDK mode | `bool` | `true` | no | +| [clients\_use\_vmss](#input\_clients\_use\_vmss) | Use VMSS for clients | `bool` | `false` | no | | [cluster\_name](#input\_cluster\_name) | Cluster name | `string` | `"poc"` | no | | [cluster\_size](#input\_cluster\_size) | The number of virtual machines to deploy. | `number` | `6` | no | | [clusterization\_target](#input\_clusterization\_target) | The clusterization target | `number` | `null` | no | | [containers\_config\_map](#input\_containers\_config\_map) | Maps the number of objects and memory size per machine type. | 
map(object({
    compute  = number
    drive    = number
    frontend = number
    nvme     = number
    nics     = number
    memory   = list(string)
  }))
| 
{
  "Standard_L16as_v3": {
    "compute": 4,
    "drive": 2,
    "frontend": 1,
    "memory": [
      "72GB",
      "73GB"
    ],
    "nics": 8,
    "nvme": 2
  },
  "Standard_L16s_v3": {
    "compute": 4,
    "drive": 2,
    "frontend": 1,
    "memory": [
      "79GB",
      "72GB"
    ],
    "nics": 8,
    "nvme": 2
  },
  "Standard_L32as_v3": {
    "compute": 4,
    "drive": 2,
    "frontend": 1,
    "memory": [
      "190GB",
      "190GB"
    ],
    "nics": 8,
    "nvme": 4
  },
  "Standard_L32s_v3": {
    "compute": 4,
    "drive": 2,
    "frontend": 1,
    "memory": [
      "197GB",
      "189GB"
    ],
    "nics": 8,
    "nvme": 4
  },
  "Standard_L48as_v3": {
    "compute": 3,
    "drive": 3,
    "frontend": 1,
    "memory": [
      "308GB",
      "308GB"
    ],
    "nics": 8,
    "nvme": 6
  },
  "Standard_L48s_v3": {
    "compute": 3,
    "drive": 3,
    "frontend": 1,
    "memory": [
      "314GB",
      "306GB"
    ],
    "nics": 8,
    "nvme": 6
  },
  "Standard_L64as_v3": {
    "compute": 4,
    "drive": 2,
    "frontend": 1,
    "memory": [
      "384GB",
      "384GB"
    ],
    "nics": 8,
    "nvme": 8
  },
  "Standard_L64s_v3": {
    "compute": 4,
    "drive": 2,
    "frontend": 1,
    "memory": [
      "357GB",
      "384GB"
    ],
    "nics": 8,
    "nvme": 8
  },
  "Standard_L80as_v3": {
    "compute": 4,
    "drive": 2,
    "frontend": 1,
    "memory": [
      "384GB",
      "384GB"
    ],
    "nics": 8,
    "nvme": 8
  },
  "Standard_L80s_v3": {
    "compute": 4,
    "drive": 2,
    "frontend": 1,
    "memory": [
      "384GB",
      "384GB"
    ],
    "nics": 8,
    "nvme": 8
  },
  "Standard_L8as_v3": {
    "compute": 1,
    "drive": 1,
    "frontend": 1,
    "memory": [
      "29GB",
      "29GB"
    ],
    "nics": 4,
    "nvme": 1
  },
  "Standard_L8s_v3": {
    "compute": 1,
    "drive": 1,
    "frontend": 1,
    "memory": [
      "33GB",
      "31GB"
    ],
    "nics": 4,
    "nvme": 1
  }
}
| no | | [create\_lb](#input\_create\_lb) | Create backend and UI load balancers for weka cluster. | `bool` | `true` | no | | [create\_nat\_gateway](#input\_create\_nat\_gateway) | NAT needs to be created when no public ip is assigned to the backend, to allow internet access | `bool` | `false` | no | +| [create\_storage\_account\_private\_links](#input\_create\_storage\_account\_private\_links) | Create private links for storage accounts (needed in case if public network access for the storage account is disabled). | `bool` | `false` | no | | [debug\_down\_backends\_removal\_timeout](#input\_debug\_down\_backends\_removal\_timeout) | Don't change this value without consulting weka support team. Timeout for removing down backends. Valid time units are ns, us (or µs), ms, s, m, h. | `string` | `"3h"` | no | | [default\_disk\_size](#input\_default\_disk\_size) | The default disk size. | `number` | `48` | no | | [deployment\_container\_name](#input\_deployment\_container\_name) | Name of exising deployment container | `string` | `""` | no | -| [deployment\_storage\_account\_access\_key](#input\_deployment\_storage\_account\_access\_key) | The access key of the existing Blob object store container. | `string` | `""` | no | +| [deployment\_file\_share\_name](#input\_deployment\_file\_share\_name) | Name of exising deployment file share. Will use '-share' name if not provided. | `string` | `""` | no | +| [deployment\_function\_app\_code\_blob](#input\_deployment\_function\_app\_code\_blob) | The path to the function app code blob file. | `string` | `""` | no | | [deployment\_storage\_account\_name](#input\_deployment\_storage\_account\_name) | Name of exising deployment storage account | `string` | `""` | no | | [enable\_application\_insights](#input\_enable\_application\_insights) | Enable Application Insights. | `bool` | `true` | no | | [function\_access\_restriction\_enabled](#input\_function\_access\_restriction\_enabled) | Allow public access, Access restrictions apply to inbound access to internal vent | `bool` | `false` | no | -| [function\_app\_dist](#input\_function\_app\_dist) | Function app code dist | `string` | `"release"` | no | +| [function\_app\_dist](#input\_function\_app\_dist) | Function app code dist | `string` | `"dev"` | no | | [function\_app\_identity\_name](#input\_function\_app\_identity\_name) | The user assigned identity name for the function app (if empty - new one is created). | `string` | `""` | no | | [function\_app\_log\_level](#input\_function\_app\_log\_level) | Log level for function app (from -1 to 5). See https://github.com/rs/zerolog#leveled-logging | `number` | `1` | no | | [function\_app\_storage\_account\_container\_prefix](#input\_function\_app\_storage\_account\_container\_prefix) | Weka storage account container name prefix | `string` | `"weka-tf-functions-deployment-"` | no | | [function\_app\_storage\_account\_prefix](#input\_function\_app\_storage\_account\_prefix) | Weka storage account name prefix | `string` | `"weka"` | no | | [function\_app\_subnet\_delegation\_cidr](#input\_function\_app\_subnet\_delegation\_cidr) | Subnet delegation enables you to designate a specific subnet for an Azure PaaS service. | `string` | `"10.0.1.0/25"` | no | | [function\_app\_subnet\_delegation\_id](#input\_function\_app\_subnet\_delegation\_id) | Required to specify if subnet\_name were used to specify pre-defined subnets for weka. Function subnet delegation requires an additional subnet, and in the case of pre-defined networking this one also should be pre-defined | `string` | `""` | no | -| [function\_app\_version](#input\_function\_app\_version) | Function app code version (hash) | `string` | `"5464597f9be93b3c954324b1811ace7a"` | no | +| [function\_app\_version](#input\_function\_app\_version) | Function app code version (hash) | `string` | `"70129b9f8d813e6f87aeed9be4764327"` | no | | [get\_weka\_io\_token](#input\_get\_weka\_io\_token) | The token to download the Weka release from get.weka.io. | `string` | `""` | no | | [hotspare](#input\_hotspare) | Number of hotspares to set on weka cluster. Refer to https://docs.weka.io/overview/ssd-capacity-management#hot-spare | `number` | `1` | no | | [install\_cluster\_dpdk](#input\_install\_cluster\_dpdk) | Install weka cluster with DPDK | `bool` | `true` | no | | [install\_weka\_url](#input\_install\_weka\_url) | The URL of the Weka release download tar file. | `string` | `""` | no | | [instance\_type](#input\_instance\_type) | The virtual machine type (sku) to deploy. | `string` | `"Standard_L8s_v3"` | no | +| [key\_vault\_purge\_protection\_enabled](#input\_key\_vault\_purge\_protection\_enabled) | Enable purge protection for the key vault. | `bool` | `false` | no | | [log\_analytics\_workspace\_id](#input\_log\_analytics\_workspace\_id) | The Log Analytics workspace id. | `string` | `""` | no | | [logic\_app\_identity\_name](#input\_logic\_app\_identity\_name) | The user assigned identity name for the logic app (if empty - new one is created). | `string` | `""` | no | | [logic\_app\_subnet\_delegation\_cidr](#input\_logic\_app\_subnet\_delegation\_cidr) | Subnet delegation enables you to designate a specific subnet for an Azure PaaS service. | `string` | `"10.0.3.0/25"` | no | | [logic\_app\_subnet\_delegation\_id](#input\_logic\_app\_subnet\_delegation\_id) | Required to specify if subnet\_name were used to specify pre-defined subnets for weka. Logicapp subnet delegation requires an additional subnet, and in the case of pre-defined networking this one also should be pre-defined | `string` | `""` | no | -| [nfs\_client\_group\_name](#input\_nfs\_client\_group\_name) | Client access group name. | `string` | `"weka-cg"` | no | | [nfs\_deployment\_container\_name](#input\_nfs\_deployment\_container\_name) | Name of exising protocol deployment container | `string` | `""` | no | | [nfs\_interface\_group\_name](#input\_nfs\_interface\_group\_name) | Interface group name. | `string` | `"weka-ig"` | no | | [nfs\_protocol\_gateway\_disk\_size](#input\_nfs\_protocol\_gateway\_disk\_size) | The protocol gateways' default disk size. | `number` | `48` | no | @@ -452,9 +486,11 @@ proxy_url = VALUE | [prefix](#input\_prefix) | Prefix for all resources | `string` | `"weka"` | no | | [private\_dns\_rg\_name](#input\_private\_dns\_rg\_name) | The private DNS zone resource group name. Required when private\_dns\_zone\_name is set. | `string` | `""` | no | | [private\_dns\_zone\_name](#input\_private\_dns\_zone\_name) | The private DNS zone name. | `string` | `""` | no | +| [private\_dns\_zone\_use](#input\_private\_dns\_zone\_use) | Determines whether to use private DNS zone. Required for LB record creation. | `bool` | `true` | no | | [protection\_level](#input\_protection\_level) | Cluster data protection level. | `number` | `2` | no | | [protocol\_gateways\_identity\_name](#input\_protocol\_gateways\_identity\_name) | The user assigned identity name for the protocol gateways instances (if empty - new one is created). | `string` | `""` | no | | [proxy\_url](#input\_proxy\_url) | Weka home proxy url | `string` | `""` | no | +| [read\_function\_zip\_from\_storage\_account](#input\_read\_function\_zip\_from\_storage\_account) | Read function app zip from storage account (is read from public distribution storage account by default). | `bool` | `false` | no | | [rg\_name](#input\_rg\_name) | A predefined resource group in the Azure subscription. | `string` | n/a | yes | | [s3\_protocol\_gateway\_disk\_size](#input\_s3\_protocol\_gateway\_disk\_size) | The protocol gateways' default disk size. | `number` | `48` | no | | [s3\_protocol\_gateway\_fe\_cores\_num](#input\_s3\_protocol\_gateway\_fe\_cores\_num) | The number of frontend cores on single protocol gateway machine. | `number` | `1` | no | @@ -480,6 +516,9 @@ proxy_url = VALUE | [smbw\_enabled](#input\_smbw\_enabled) | Enable SMBW protocol. This option should be provided before cluster is created to leave extra capacity for SMBW setup. | `bool` | `true` | no | | [source\_image\_id](#input\_source\_image\_id) | Use weka custom image, ubuntu 20.04 with kernel 5.4 and ofed 5.8-1.1.2.1 | `string` | `"/communityGalleries/WekaIO-d7d3f308-d5a1-4c45-8e8a-818aed57375a/images/ubuntu20.04/versions/latest"` | no | | [ssh\_public\_key](#input\_ssh\_public\_key) | Ssh public key to pass to vms. | `string` | `null` | no | +| [storage\_account\_allowed\_ips](#input\_storage\_account\_allowed\_ips) | IP ranges to allow access from the internet or your on-premises networks to storage accounts. | `list(string)` | `[]` | no | +| [storage\_account\_public\_network\_access](#input\_storage\_account\_public\_network\_access) | Public network access to the storage accounts. | `string` | `"Enabled"` | no | +| [storage\_blob\_private\_dns\_zone\_name](#input\_storage\_blob\_private\_dns\_zone\_name) | The private DNS zone name for the storage account (blob). | `string` | `"privatelink.blob.core.windows.net"` | no | | [stripe\_width](#input\_stripe\_width) | Stripe width = cluster\_size - protection\_level - 1 (by default). | `number` | `-1` | no | | [subnet\_name](#input\_subnet\_name) | The subnet name. | `string` | `""` | no | | [subnet\_prefix](#input\_subnet\_prefix) | Address prefixes to use for the subnet | `string` | `"10.0.2.0/24"` | no | @@ -512,6 +551,8 @@ proxy_url = VALUE | [backend\_ips](#output\_backend\_ips) | If 'assign\_public\_ip' is set to true, it will output the public ips, If no it will output the private ips | | [backend\_lb\_private\_ip](#output\_backend\_lb\_private\_ip) | Backend load balancer ip address | | [client\_ips](#output\_client\_ips) | If 'private\_network' is set to false, it will output clients public ips, otherwise private ips. | +| [client\_vmss\_ips](#output\_client\_vmss\_ips) | If 'private\_network' is set to false, it will output clients public ips, otherwise private ips. | +| [clients\_vmss\_name](#output\_clients\_vmss\_name) | n/a | | [cluster\_helper\_commands](#output\_cluster\_helper\_commands) | Useful commands and script to interact with weka cluster | | [function\_app\_name](#output\_function\_app\_name) | Function app name | | [function\_key\_name](#output\_function\_key\_name) | Function app key name | @@ -528,4 +569,5 @@ proxy_url = VALUE | [vmss\_name](#output\_vmss\_name) | n/a | | [vnet\_name](#output\_vnet\_name) | Virtual network name | | [vnet\_rg\_name](#output\_vnet\_rg\_name) | Virtual network resource group name | +| [weka\_cluster\_admin\_password\_secret\_name](#output\_weka\_cluster\_admin\_password\_secret\_name) | Weka cluster admin password secret name | diff --git a/blob.tf b/blob.tf index 65c124f7..abf293ea 100644 --- a/blob.tf +++ b/blob.tf @@ -1,15 +1,22 @@ locals { - clusterization_target = var.clusterization_target != null ? var.clusterization_target : min(var.cluster_size, max(20, ceil(var.cluster_size * 0.8))) - # fields that depend on LB creation - vmss_health_probe_id = var.create_lb ? azurerm_lb_probe.backend_lb_probe[0].id : null - lb_backend_pool_ids = var.create_lb ? [azurerm_lb_backend_address_pool.lb_backend_pool[0].id] : [] + deployment_storage_account_id = var.deployment_storage_account_name == "" ? azurerm_storage_account.deployment_sa[0].id : data.azurerm_storage_account.deployment_blob[0].id + deployment_storage_account_name = var.deployment_storage_account_name == "" ? azurerm_storage_account.deployment_sa[0].name : var.deployment_storage_account_name + deployment_sa_connection_string = var.deployment_storage_account_name == "" ? azurerm_storage_account.deployment_sa[0].primary_connection_string : data.azurerm_storage_account.deployment_blob[0].primary_connection_string + deployment_container_name = var.deployment_container_name == "" ? "${local.alphanumeric_prefix_name}${local.alphanumeric_cluster_name}-deployment" : var.deployment_container_name + deployment_file_share_name = var.deployment_file_share_name == "" ? "${local.deployment_storage_account_name}-share" : var.deployment_file_share_name + deployment_sa_access_key = var.deployment_storage_account_name == "" ? azurerm_storage_account.deployment_sa[0].primary_access_key : data.azurerm_storage_account.deployment_blob[0].primary_access_key + + sa_allowed_ips_provided = length(var.storage_account_allowed_ips) > 0 + sa_public_access_enabled = var.storage_account_public_network_access == "Enabled" + sa_public_access_for_vnet = var.storage_account_public_network_access == "EnabledForVnet" + sa_public_access_disabled = var.storage_account_public_network_access == "Disabled" + create_sa_resources = local.sa_public_access_enabled || local.sa_public_access_for_vnet && local.sa_allowed_ips_provided } - resource "azurerm_storage_account" "deployment_sa" { - count = var.deployment_storage_account_name == "" ? 1 : 0 + count = var.deployment_storage_account_name == "" && local.create_sa_resources ? 1 : 0 name = substr("${local.alphanumeric_prefix_name}${local.alphanumeric_cluster_name}deployment", 0, 24) - location = data.azurerm_resource_group.rg.location + location = local.location resource_group_name = var.rg_name account_kind = "StorageV2" account_tier = "Standard" @@ -18,10 +25,20 @@ resource "azurerm_storage_account" "deployment_sa" { lifecycle { ignore_changes = [tags] } + + dynamic "network_rules" { + for_each = local.sa_public_access_for_vnet ? [1] : [] + content { + default_action = "Deny" + bypass = ["AzureServices"] + ip_rules = var.storage_account_allowed_ips + virtual_network_subnet_ids = [data.azurerm_subnet.subnet.id, local.function_app_subnet_delegation_id] + } + } } resource "azurerm_storage_container" "deployment" { - count = var.deployment_container_name == "" ? 1 : 0 + count = var.deployment_container_name == "" && local.create_sa_resources ? 1 : 0 name = "${local.alphanumeric_prefix_name}${local.alphanumeric_cluster_name}-deployment" storage_account_name = local.deployment_storage_account_name container_access_type = "private" @@ -29,6 +46,7 @@ resource "azurerm_storage_container" "deployment" { } resource "azurerm_storage_blob" "state" { + count = local.create_sa_resources ? 1 : 0 name = "state" storage_account_name = local.deployment_storage_account_name storage_container_name = local.deployment_container_name @@ -41,97 +59,18 @@ resource "azurerm_storage_blob" "state" { } } -data "azurerm_storage_account" "deployment_blob" { - count = var.deployment_storage_account_name != "" ? 1 : 0 - name = var.deployment_storage_account_name - resource_group_name = var.rg_name +resource "azurerm_storage_share" "function_app_share" { + count = var.deployment_file_share_name == "" && local.sa_public_access_for_vnet && local.sa_allowed_ips_provided ? 1 : 0 + name = local.deployment_file_share_name + storage_account_name = local.deployment_storage_account_name + quota = 100 + depends_on = [azurerm_storage_account.deployment_sa] } -resource "azurerm_storage_blob" "vmss_config" { - name = "vmss-config" - storage_account_name = local.deployment_storage_account_name - storage_container_name = local.deployment_container_name - type = "Block" - - source_content = jsonencode({ - name = "${var.prefix}-${var.cluster_name}-vmss" - location = data.azurerm_resource_group.rg.location - zones = var.zone != null ? [var.zone] : [] - resource_group_name = var.rg_name - sku = var.instance_type - upgrade_mode = "Manual" - health_probe_id = local.vmss_health_probe_id - admin_username = var.vm_username - ssh_public_key = local.public_ssh_key - computer_name_prefix = "${var.prefix}-${var.cluster_name}-backend" - custom_data = base64encode(local.custom_data_script) - disable_password_authentication = true - proximity_placement_group_id = local.placement_group_id - single_placement_group = var.vmss_single_placement_group - source_image_id = var.source_image_id - overprovision = false - orchestration_mode = "Uniform" - tags = merge(var.tags_map, { - "weka_cluster" : var.cluster_name, - "user_id" : data.azurerm_client_config.current.object_id, - }) - - os_disk = { - caching = "ReadWrite" - storage_account_type = "Premium_LRS" - } - - data_disk = { - lun = 0 - caching = "None" - create_option = "Empty" - disk_size_gb = local.disk_size - storage_account_type = "Premium_LRS" - } - - identity = { - type = "UserAssigned" - identity_ids = [local.vmss_identity_id] - } - - primary_nic = { - name = "${var.prefix}-${var.cluster_name}-backend-nic-0" - network_security_group_id = local.sg_id - enable_accelerated_networking = var.install_cluster_dpdk - - ip_configurations = [{ - primary = true - subnet_id = data.azurerm_subnet.subnet.id - load_balancer_backend_address_pool_ids = local.lb_backend_pool_ids - public_ip_address = { - assign = local.assign_public_ip - name = "${var.prefix}-${var.cluster_name}-public-ip" - domain_name_label = "${var.prefix}-${var.cluster_name}-backend" - } - }] - } - - secondary_nics = { - number = local.nics_numbers - 1 - name_prefix = "${var.prefix}-${var.cluster_name}-backend-nic" - network_security_group_id = local.sg_id - enable_accelerated_networking = var.install_cluster_dpdk - ip_configurations = [{ - primary = true - subnet_id = data.azurerm_subnet.subnet.id - load_balancer_backend_address_pool_ids = local.lb_backend_pool_ids - }] - } - }) - depends_on = [ - azurerm_storage_container.deployment, azurerm_lb_backend_address_pool.lb_backend_pool, azurerm_lb_probe.backend_lb_probe, - azurerm_proximity_placement_group.ppg, azurerm_lb_rule.backend_lb_rule, azurerm_lb_rule.ui_lb_rule - ] -} # state for protocols resource "azurerm_storage_container" "nfs_deployment" { - count = var.nfs_deployment_container_name == "" ? 1 : 0 + count = var.nfs_deployment_container_name == "" && local.create_sa_resources ? 1 : 0 name = "${local.alphanumeric_prefix_name}${local.alphanumeric_cluster_name}-protocol-deployment" storage_account_name = local.deployment_storage_account_name container_access_type = "private" @@ -139,7 +78,7 @@ resource "azurerm_storage_container" "nfs_deployment" { } resource "azurerm_storage_blob" "nfs_state" { - count = var.nfs_protocol_gateways_number > 0 ? 1 : 0 + count = var.nfs_protocol_gateways_number > 0 && local.create_sa_resources ? 1 : 0 name = "nfs_state" storage_account_name = local.deployment_storage_account_name storage_container_name = local.nfs_deployment_container_name @@ -157,3 +96,155 @@ resource "azurerm_storage_blob" "nfs_state" { ignore_changes = all } } + +resource "azurerm_storage_account" "logicapp" { + count = local.create_sa_resources ? 1 : 0 + name = substr("${local.alphanumeric_prefix_name}${local.alphanumeric_cluster_name}logicappsa", 0, 24) + resource_group_name = var.rg_name + location = local.location + account_tier = "Standard" + account_replication_type = "LRS" + + dynamic "network_rules" { + for_each = local.sa_public_access_for_vnet ? [1] : [] + content { + default_action = "Deny" + bypass = ["AzureServices"] + ip_rules = var.storage_account_allowed_ips + virtual_network_subnet_ids = [data.azurerm_subnet.subnet.id, var.logic_app_subnet_delegation_id == "" ? module.logic_app_subnet_delegation[0].id : var.logic_app_subnet_delegation_id] + } + } +} + +data "azurerm_storage_account" "deployment_blob" { + count = var.deployment_storage_account_name != "" ? 1 : 0 + name = var.deployment_storage_account_name + resource_group_name = local.resource_group_name +} + +resource "azurerm_private_dns_zone" "blob" { + count = var.create_storage_account_private_links ? 1 : 0 + name = "privatelink.blob.core.windows.net" + resource_group_name = local.resource_group_name +} + +data "azurerm_private_dns_zone" "blob" { + count = !var.create_storage_account_private_links && local.sa_public_access_disabled ? 1 : 0 + name = var.storage_blob_private_dns_zone_name +} + +resource "azurerm_private_dns_zone" "file" { + count = var.create_storage_account_private_links ? 1 : 0 + name = "privatelink.file.core.windows.net" + resource_group_name = local.resource_group_name +} + +resource "azurerm_private_dns_zone_virtual_network_link" "blob_privatelink" { + count = var.create_storage_account_private_links ? 1 : 0 + name = "${var.prefix}-${var.cluster_name}-blob-privatelink" + resource_group_name = local.resource_group_name + private_dns_zone_name = azurerm_private_dns_zone.blob[0].name + virtual_network_id = data.azurerm_virtual_network.vnet.id +} + +resource "azurerm_private_dns_zone_virtual_network_link" "file_privatelink" { + count = var.create_storage_account_private_links ? 1 : 0 + name = "${var.prefix}-${var.cluster_name}-file-privatelink" + resource_group_name = local.resource_group_name + private_dns_zone_name = azurerm_private_dns_zone.file[0].name + virtual_network_id = data.azurerm_virtual_network.vnet.id +} + +resource "azurerm_private_endpoint" "file_endpoint" { + count = var.create_storage_account_private_links ? 1 : 0 + name = "${var.prefix}-${var.cluster_name}-file-endpoint" + location = data.azurerm_resource_group.rg.location + resource_group_name = data.azurerm_resource_group.rg.name + subnet_id = data.azurerm_subnet.subnet.id + tags = merge(var.tags_map, { "weka_cluster" : var.cluster_name }) + + private_dns_zone_group { + name = "${var.prefix}-${var.cluster_name}-dns-zone-group-file" + private_dns_zone_ids = [azurerm_private_dns_zone.file[0].id] + } + + private_service_connection { + name = "${var.prefix}-${var.cluster_name}-privateFileSvcCon" + is_manual_connection = false + private_connection_resource_id = local.deployment_storage_account_id + subresource_names = ["file"] + } +} + +resource "azurerm_private_endpoint" "blob_endpoint" { + count = var.create_storage_account_private_links ? 1 : 0 + name = "${var.prefix}-${var.cluster_name}-blob-endpoint" + location = data.azurerm_resource_group.rg.location + resource_group_name = data.azurerm_resource_group.rg.name + subnet_id = data.azurerm_subnet.subnet.id + tags = merge(var.tags_map, { "weka_cluster" : var.cluster_name }) + + private_dns_zone_group { + name = "${var.prefix}-${var.cluster_name}-dns-zone-group-blob" + private_dns_zone_ids = [azurerm_private_dns_zone.blob[0].id] + } + private_service_connection { + name = "${var.prefix}-${var.cluster_name}-privateBlobSvcCon" + is_manual_connection = false + private_connection_resource_id = local.deployment_storage_account_id + subresource_names = ["blob"] + } +} + +data "azurerm_storage_account" "weka_obs" { + count = var.tiering_obs_name != "" ? 1 : 0 + name = var.tiering_obs_name + resource_group_name = var.rg_name +} + +resource "azurerm_private_endpoint" "weka_obs_blob_endpoint" { + count = var.create_storage_account_private_links && var.tiering_blob_obs_access_key != "" ? 1 : 0 + name = "${var.prefix}-${var.cluster_name}-obs-blob-endpoint" + location = data.azurerm_resource_group.rg.location + resource_group_name = data.azurerm_resource_group.rg.name + subnet_id = data.azurerm_subnet.subnet.id + tags = merge(var.tags_map, { "weka_cluster" : var.cluster_name }) + + private_dns_zone_group { + name = "${var.prefix}-${var.cluster_name}-dns-zone-group-obs-blob" + private_dns_zone_ids = [azurerm_private_dns_zone.blob[0].id] + } + private_service_connection { + name = "${var.prefix}-${var.cluster_name}-private-obs-BlobSvcCon" + is_manual_connection = false + private_connection_resource_id = data.azurerm_storage_account.weka_obs[0].id + subresource_names = ["blob"] + } + + lifecycle { + precondition { + condition = var.tiering_obs_name != "" + error_message = "Tiering OBS is not provided" + } + precondition { + condition = var.tiering_obs_container_name != "" + error_message = "Tiering OBS container name is not provided" + } + } +} + +data "azurerm_storage_account_blob_container_sas" "function_app_code_sas" { + count = local.sa_public_access_enabled || local.sa_public_access_for_vnet && local.sa_allowed_ips_provided ? 0 : 1 + connection_string = local.deployment_sa_connection_string + container_name = local.deployment_container_name + start = timestamp() + expiry = formatdate("YYYY-MM-DD'T'hh:mm:ssZ", timeadd(timestamp(), "1h")) + permissions { + read = true + add = false + create = false + write = false + delete = false + list = false + } +} diff --git a/ci/deployment/README.md b/ci/deployment/README.md index 0b5e9338..0280ca42 100644 --- a/ci/deployment/README.md +++ b/ci/deployment/README.md @@ -4,7 +4,7 @@ | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.4.6 | -| [azurerm](#requirement\_azurerm) | ~>3.75.0 | +| [azurerm](#requirement\_azurerm) | ~>3.114.0 | ## Providers diff --git a/ci/deployment/versions.tf b/ci/deployment/versions.tf index 281dc878..202d92cc 100644 --- a/ci/deployment/versions.tf +++ b/ci/deployment/versions.tf @@ -3,7 +3,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "~>3.75.0" + version = "~>3.114.0" } } } diff --git a/ci/service_account/README.md b/ci/service_account/README.md index 250d4ef7..372f177a 100644 --- a/ci/service_account/README.md +++ b/ci/service_account/README.md @@ -5,14 +5,14 @@ |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.7 | | [azuread](#requirement\_azuread) | >= 2.33.0 | -| [azurerm](#requirement\_azurerm) | ~> 3.75.0 | +| [azurerm](#requirement\_azurerm) | ~> 3.114.0 | ## Providers | Name | Version | |------|---------| | [azuread](#provider\_azuread) | >= 2.33.0 | -| [azurerm](#provider\_azurerm) | ~> 3.75.0 | +| [azurerm](#provider\_azurerm) | ~> 3.114.0 | ## Modules diff --git a/ci/service_account/versions.tf b/ci/service_account/versions.tf index 1ed01f47..faa76b46 100644 --- a/ci/service_account/versions.tf +++ b/ci/service_account/versions.tf @@ -6,7 +6,7 @@ terraform { } azurerm = { source = "hashicorp/azurerm" - version = "~> 3.75.0" + version = "~> 3.114.0" } } required_version = ">= 1.3.7" diff --git a/clients.tf b/clients.tf index c493eaf7..189898b6 100644 --- a/clients.tf +++ b/clients.tf @@ -20,6 +20,7 @@ module "clients" { sg_id = local.sg_id tags_map = var.tags_map custom_data = var.clients_custom_data + use_vmss = var.clients_use_vmss vmss_name = "${var.prefix}-${var.cluster_name}-vmss" depends_on = [azurerm_proximity_placement_group.ppg, module.network] arch = var.client_arch diff --git a/examples/existing_private_network/README.md b/examples/existing_private_network/README.md index 74ea14c0..df9cff51 100644 --- a/examples/existing_private_network/README.md +++ b/examples/existing_private_network/README.md @@ -4,7 +4,7 @@ | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.7 | -| [azurerm](#requirement\_azurerm) | ~> 3.75.0 | +| [azurerm](#requirement\_azurerm) | ~> 3.114.0 | ## Providers diff --git a/examples/existing_private_network/versions.tf b/examples/existing_private_network/versions.tf index 0cf72143..0e54932c 100644 --- a/examples/existing_private_network/versions.tf +++ b/examples/existing_private_network/versions.tf @@ -2,7 +2,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "~> 3.75.0" + version = "~> 3.114.0" } } required_version = ">= 1.3.7" diff --git a/examples/existing_private_network_with_peering/README.md b/examples/existing_private_network_with_peering/README.md index 74ea14c0..df9cff51 100644 --- a/examples/existing_private_network_with_peering/README.md +++ b/examples/existing_private_network_with_peering/README.md @@ -4,7 +4,7 @@ | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.7 | -| [azurerm](#requirement\_azurerm) | ~> 3.75.0 | +| [azurerm](#requirement\_azurerm) | ~> 3.114.0 | ## Providers diff --git a/examples/existing_private_network_with_peering/versions.tf b/examples/existing_private_network_with_peering/versions.tf index 0cf72143..0e54932c 100644 --- a/examples/existing_private_network_with_peering/versions.tf +++ b/examples/existing_private_network_with_peering/versions.tf @@ -2,7 +2,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "~> 3.75.0" + version = "~> 3.114.0" } } required_version = ">= 1.3.7" diff --git a/examples/public_network/README.md b/examples/public_network/README.md index 0cac3724..2c904ad2 100644 --- a/examples/public_network/README.md +++ b/examples/public_network/README.md @@ -4,7 +4,7 @@ | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.7 | -| [azurerm](#requirement\_azurerm) | ~> 3.75.0 | +| [azurerm](#requirement\_azurerm) | ~> 3.114.0 | ## Providers diff --git a/examples/public_network/versions.tf b/examples/public_network/versions.tf index 0cf72143..0e54932c 100644 --- a/examples/public_network/versions.tf +++ b/examples/public_network/versions.tf @@ -2,7 +2,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "~> 3.75.0" + version = "~> 3.114.0" } } required_version = ">= 1.3.7" diff --git a/examples/public_network_with_existing_obs/README.md b/examples/public_network_with_existing_obs/README.md index b14f0791..763f3ba3 100644 --- a/examples/public_network_with_existing_obs/README.md +++ b/examples/public_network_with_existing_obs/README.md @@ -4,7 +4,7 @@ | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.7 | -| [azurerm](#requirement\_azurerm) | ~> 3.75.0 | +| [azurerm](#requirement\_azurerm) | ~> 3.114.0 | ## Providers diff --git a/examples/public_network_with_existing_obs/versions.tf b/examples/public_network_with_existing_obs/versions.tf index 0cf72143..0e54932c 100644 --- a/examples/public_network_with_existing_obs/versions.tf +++ b/examples/public_network_with_existing_obs/versions.tf @@ -2,7 +2,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "~> 3.75.0" + version = "~> 3.114.0" } } required_version = ">= 1.3.7" diff --git a/function-app/code/common/common.go b/function-app/code/common/common.go index 091e3d5e..409fc49b 100644 --- a/function-app/code/common/common.go +++ b/function-app/code/common/common.go @@ -21,6 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease" "github.com/google/uuid" @@ -31,7 +32,10 @@ import ( ) const ( - WekaAdminUsername = "admin" + WekaAdminUsername = "admin" + WekaAdminPasswordKey = "weka-password" + WekaDeploymentUsername = "weka-deployment" + WekaDeploymentPasswordKey = "weka-deployment-password" // NFS VMs tag NfsInterfaceGroupPortKey = "nfs_interface_group_port" NfsInterfaceGroupPortValue = "ready" @@ -58,6 +62,16 @@ type BlobObjParams struct { BlobName string } +type AzureObsParams struct { + Name string + ContainerName string + AccessKey string + TieringSsdPercent string + NetworkAccess string + AllowedSubnets []string + AllowedPublicIps []string +} + const FindDrivesScript = ` import json import sys @@ -231,6 +245,120 @@ func ReadBlobObject(ctx context.Context, bl BlobObjParams) (state []byte, err er } +func containerExists(ctx context.Context, containerClient *container.Client, storageName, containerName string) (bool, error) { + _, err := containerClient.GetProperties(ctx, nil) + if err != nil { + var responseErr *azcore.ResponseError + if errors.As(err, &responseErr) && responseErr.ErrorCode == "ContainerNotFound" { + return false, nil + } + err = fmt.Errorf("failed to get container properties: %w", err) + return false, err + } + return true, nil +} + +func ensureStorageContainer(ctx context.Context, storageAccountName, containerName string) (err error) { + logger := logging.LoggerFromCtx(ctx) + + credential, err := getCredential(ctx) + if err != nil { + return + } + + containerUrl := getContainerUrl(storageAccountName, containerName) + containerClient, err := container.NewClient(containerUrl, credential, nil) + if err != nil { + err = fmt.Errorf("failed to create container client: %v", err) + return err + } + + exists, err := containerExists(ctx, containerClient, storageAccountName, containerName) + if err != nil { + logger.Error().Err(err).Send() + return + } + + if exists { + logger.Info().Str("container", containerName).Msg("container already exists") + return + } + + logger.Info().Str("container", containerName).Msg("container does not exist, creating new container") + + _, err = containerClient.Create(ctx, nil) + if err != nil { + err = fmt.Errorf("failed to create container: %v", err) + logger.Error().Err(err).Send() + } + return +} + +func blobExists(ctx context.Context, blobClient *blob.Client) (bool, error) { + _, err := blobClient.GetProperties(ctx, nil) + if err != nil { + var responseErr *azcore.ResponseError + if errors.As(err, &responseErr) && responseErr.ErrorCode == "BlobNotFound" { + return false, nil + } + err = fmt.Errorf("failed to get blob properties: %w", err) + return false, err + } + return true, nil +} + +func EnsureStateIsCreated(ctx context.Context, p BlobObjParams, initialState protocol.ClusterState) (exists bool, err error) { + logger := logging.LoggerFromCtx(ctx) + + err = ensureStorageContainer(ctx, p.StorageName, p.ContainerName) + if err != nil { + return + } + + credential, err := getCredential(ctx) + if err != nil { + return + } + + url := getBlobFileUrl(p.StorageName, p.ContainerName, p.BlobName) + blobClient, err := blob.NewClient(url, credential, nil) + if err != nil { + logger.Error().Err(err).Msg("failed to create blob client") + return + } + + exists, err = blobExists(ctx, blobClient) + if err != nil { + logger.Error().Err(err).Send() + return + } + + if exists { + logger.Info().Msg("state already exists") + return + } + + logger.Info().Msg("state does not exist, creating new state") + + err = WriteState(ctx, p, initialState) + if err != nil { + logger.Error().Err(err).Msg("failed to write initial state") + } + return +} + +func ReadStateOrCreateNew(ctx context.Context, p BlobObjParams, initialState protocol.ClusterState) (state protocol.ClusterState, err error) { + exists, err := EnsureStateIsCreated(ctx, p, initialState) + if err != nil { + return + } + + if exists { + return ReadState(ctx, p) + } + return initialState, nil +} + func ReadState(ctx context.Context, stateParams BlobObjParams) (state protocol.ClusterState, err error) { logger := logging.LoggerFromCtx(ctx) @@ -284,6 +412,10 @@ func getBlobUrl(storageName string) string { return fmt.Sprintf("https://%s.blob.core.windows.net/", storageName) } +func getBlobFileUrl(storageName, containerName, blobName string) string { + return fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s", storageName, containerName, blobName) +} + func getContainerUrl(storageName, containerName string) string { return fmt.Sprintf("https://%s.blob.core.windows.net/%s", storageName, containerName) } @@ -358,9 +490,118 @@ func UpdateClusterized(ctx context.Context, subscriptionId, resourceGroupName st return } -func CreateStorageAccount(ctx context.Context, subscriptionId, resourceGroupName, obsName, location string) (accessKey string, err error) { +func CreatePrivateDnsZoneGroup(ctx context.Context, subscriptionId, resourceGroupName, privateEndpointName, privateDNSZoneID string) (privateDNSZoneGroup *armnetwork.PrivateDNSZoneGroup, err error) { logger := logging.LoggerFromCtx(ctx) - logger.Info().Msgf("creating storage account: %s", obsName) + + credential, err := getCredential(ctx) + if err != nil { + return + } + + privateDNSZoneGroupName := fmt.Sprintf("%s-dns-group", privateEndpointName) + + parameters := armnetwork.PrivateDNSZoneGroup{ + Name: &privateDNSZoneGroupName, + Properties: &armnetwork.PrivateDNSZoneGroupPropertiesFormat{ + PrivateDNSZoneConfigs: []*armnetwork.PrivateDNSZoneConfig{ + { + Name: &privateDNSZoneGroupName, + Properties: &armnetwork.PrivateDNSZonePropertiesFormat{ + PrivateDNSZoneID: &privateDNSZoneID, + }, + }, + }, + }, + } + + client, err := armnetwork.NewPrivateDNSZoneGroupsClient(subscriptionId, credential, nil) + if err != nil { + err = fmt.Errorf("failed to create PrivateDNSZoneGroupsClient: %w", err) + return + } + + poller, err := client.BeginCreateOrUpdate(ctx, resourceGroupName, privateEndpointName, privateDNSZoneGroupName, parameters, nil) + if err != nil { + logger.Error().Err(err).Send() + return + } + + res, err := poller.PollUntilDone(ctx, nil) + if err != nil { + logger.Error().Err(err).Send() + return + } + + privateDNSZoneGroup = &res.PrivateDNSZoneGroup + return +} + +func CreateStorageAccountBlobPrivateEndpoint(ctx context.Context, subscriptionId, resourceGroupName, location, storageAccountName, privateEndpointName, subnetId, privateDnsZoneId string) (err error) { + logger := logging.LoggerFromCtx(ctx) + + credential, err := getCredential(ctx) + if err != nil { + return + } + + client, err := armnetwork.NewPrivateEndpointsClient(subscriptionId, credential, nil) + if err != nil { + logger.Error().Err(err).Send() + return + } + + storageAccountId := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Storage/storageAccounts/%s", subscriptionId, resourceGroupName, storageAccountName) + connectionName := fmt.Sprintf("%s-conn", privateEndpointName) + subresourceName := "blob" + + privateServiceConnection := &armnetwork.PrivateLinkServiceConnection{ + Name: &connectionName, + Properties: &armnetwork.PrivateLinkServiceConnectionProperties{ + PrivateLinkServiceID: &storageAccountId, + GroupIDs: []*string{&subresourceName}, + }, + } + + privateEndpoint := armnetwork.PrivateEndpoint{ + Location: &location, + Properties: &armnetwork.PrivateEndpointProperties{ + Subnet: &armnetwork.Subnet{ + ID: &subnetId, + }, + PrivateLinkServiceConnections: []*armnetwork.PrivateLinkServiceConnection{ + privateServiceConnection, + }, + }, + } + + poller, err := client.BeginCreateOrUpdate(ctx, resourceGroupName, privateEndpointName, privateEndpoint, nil) + if err != nil { + logger.Error().Err(err).Send() + return + } + + res, err := poller.PollUntilDone(ctx, nil) + if err != nil { + logger.Error().Err(err).Send() + return + } + + logger.Info().Msgf("private endpoint %s created", *res.PrivateEndpoint.ID) + + dnsZoneGroup, err := CreatePrivateDnsZoneGroup(ctx, subscriptionId, resourceGroupName, privateEndpointName, privateDnsZoneId) + if err != nil { + logger.Error().Err(err).Send() + return + } + + logger.Info().Msgf("private dns zone group %s created", *dnsZoneGroup.ID) + + return +} + +func CreateStorageAccount(ctx context.Context, subscriptionId, resourceGroupName, location string, obsParams AzureObsParams) (accessKey string, err error) { + logger := logging.LoggerFromCtx(ctx) + logger.Info().Msgf("creating storage account: %v", obsParams) credential, err := getCredential(ctx) if err != nil { @@ -374,18 +615,45 @@ func CreateStorageAccount(ctx context.Context, subscriptionId, resourceGroupName } skuName := armstorage.SKUNameStandardZRS kind := armstorage.KindStorageV2 - _, err = client.BeginCreate(ctx, resourceGroupName, obsName, armstorage.AccountCreateParameters{ + publicAccess := armstorage.PublicNetworkAccessEnabled + if obsParams.NetworkAccess == "Disabled" { + publicAccess = armstorage.PublicNetworkAccessDisabled + } + + var networkRuleSet *armstorage.NetworkRuleSet + if len(obsParams.AllowedSubnets) > 0 || len(obsParams.AllowedPublicIps) > 0 { + action := armstorage.DefaultActionDeny + bypass := armstorage.BypassAzureServices + networkRuleSet = &armstorage.NetworkRuleSet{ + DefaultAction: &action, + Bypass: &bypass, + } + for i := 0; i < len(obsParams.AllowedSubnets); i++ { + rule := &armstorage.VirtualNetworkRule{VirtualNetworkResourceID: &obsParams.AllowedSubnets[i]} + networkRuleSet.VirtualNetworkRules = append(networkRuleSet.VirtualNetworkRules, rule) + } + for i := 0; i < len(obsParams.AllowedPublicIps); i++ { + rule := &armstorage.IPRule{IPAddressOrRange: &obsParams.AllowedPublicIps[i]} + networkRuleSet.IPRules = append(networkRuleSet.IPRules, rule) + } + } + + _, err = client.BeginCreate(ctx, resourceGroupName, obsParams.Name, armstorage.AccountCreateParameters{ Kind: &kind, Location: &location, SKU: &armstorage.SKU{ Name: &skuName, }, + Properties: &armstorage.AccountPropertiesCreateParameters{ + PublicNetworkAccess: &publicAccess, + NetworkRuleSet: networkRuleSet, + }, }, nil) if err != nil { if azerr, ok := err.(*azcore.ResponseError); ok { if azerr.ErrorCode == "StorageAccountAlreadyExists" { - logger.Debug().Msgf("storage account %s already exists", obsName) + logger.Debug().Msgf("storage account %s already exists", obsParams.Name) err = nil } else { logger.Error().Msgf("storage creation failed: %s", err) @@ -398,7 +666,7 @@ func CreateStorageAccount(ctx context.Context, subscriptionId, resourceGroupName } for i := 0; i < 10; i++ { - accessKey, err = getStorageAccountAccessKey(ctx, subscriptionId, resourceGroupName, obsName) + accessKey, err = getStorageAccountAccessKey(ctx, subscriptionId, resourceGroupName, obsParams.Name) if err != nil { if azerr, ok := err.(*azcore.ResponseError); ok { @@ -414,7 +682,7 @@ func CreateStorageAccount(ctx context.Context, subscriptionId, resourceGroupName return } } else { - logger.Debug().Msgf("storage account '%s' is ready for use", obsName) + logger.Debug().Msgf("storage account '%s' is ready for use", obsParams.Name) break } } @@ -489,7 +757,7 @@ func GetKeyVaultValue(ctx context.Context, keyVaultUri, secretName string) (secr } resp, err := client.GetSecret(ctx, secretName, "", nil) if err != nil { - logger.Error().Err(err).Send() + logger.Info().Err(err).Send() return } @@ -498,6 +766,32 @@ func GetKeyVaultValue(ctx context.Context, keyVaultUri, secretName string) (secr return } +func SetKeyVaultValue(ctx context.Context, keyVaultUri, secretName, secretValue string) (err error) { + logger := logging.LoggerFromCtx(ctx) + logger.Info().Msgf("setting key vault secret: %s", secretName) + + credential, err := getCredential(ctx) + if err != nil { + return + } + + client, err := azsecrets.NewClient(keyVaultUri, credential, nil) + if err != nil { + logger.Error().Err(err).Send() + return + } + + params := azsecrets.SetSecretParameters{ + Value: &secretValue, + } + + _, err = client.SetSecret(ctx, secretName, params, nil) + if err != nil { + logger.Error().Err(err).Send() + } + return +} + // Gets all network interfaces in a VM scale set (Uniform) // see https://learn.microsoft.com/en-us/rest/api/virtualnetwork/network-interface-in-vm-ss/list-virtual-machine-scale-set-network-interfaces func getUniformScaleSetVmsNetworkInterfaces(ctx context.Context, subscriptionId, resourceGroupName, vmScaleSetName string) (networkInterfaces []*armnetwork.Interface, err error) { @@ -855,15 +1149,6 @@ func AssignStorageBlobDataContributorRoleToScaleSet( return &res.RoleAssignment, nil } -type ScaleSetInfo struct { - Id string - Name string - AdminUsername string - AdminPassword string - Capacity int - VMSize string -} - // Gets scale set // see https://learn.microsoft.com/en-us/rest/api/compute/virtual-machine-scale-sets/get func getScaleSet(ctx context.Context, subscriptionId, resourceGroupName, vmScaleSetName string) (*armcompute.VirtualMachineScaleSet, error) { @@ -904,33 +1189,6 @@ func GetScaleSetOrNil(ctx context.Context, subscriptionId, resourceGroupName, vm return scaleSet, nil } -// Gets single scale set info -func GetScaleSetInfo(ctx context.Context, subscriptionId, resourceGroupName, vmScaleSetName, keyVaultUri string) (*ScaleSetInfo, error) { - logger := logging.LoggerFromCtx(ctx) - - scaleSet, err := getScaleSet(ctx, subscriptionId, resourceGroupName, vmScaleSetName) - if err != nil { - logger.Error().Err(err).Send() - return nil, err - } - - wekaPassword, err := GetWekaClusterPassword(ctx, keyVaultUri) - if err != nil { - logger.Error().Err(err).Send() - return nil, err - } - - scaleSetInfo := ScaleSetInfo{ - Id: *scaleSet.ID, - Name: *scaleSet.Name, - AdminUsername: WekaAdminUsername, - AdminPassword: wekaPassword, - Capacity: int(*scaleSet.SKU.Capacity), - VMSize: *scaleSet.SKU.Name, - } - return &scaleSetInfo, err -} - func GetScaleSetInstances(ctx context.Context, vmssParams *ScaleSetParams) (vms []*VMInfoSummary, err error) { if vmssParams.Flexible { vms, err = GetFlexibleScaleSetInstances(ctx, vmssParams.SubscriptionId, vmssParams.ResourceGroupName, vmssParams.ScaleSetName, nil) @@ -1209,8 +1467,38 @@ func ReportMsg(ctx context.Context, hostName string, stateParams BlobObjParams, _ = UpdateStateReporting(ctx, stateParams, reportObj) } -func GetWekaClusterPassword(ctx context.Context, keyVaultUri string) (password string, err error) { - return GetKeyVaultValue(ctx, keyVaultUri, "weka-password") +func GetWekaAdminPassword(ctx context.Context, keyVaultUri string) (password string, err error) { + return GetKeyVaultValue(ctx, keyVaultUri, WekaAdminPasswordKey) +} + +func GetWekaDeploymentPassword(ctx context.Context, keyVaultUri string) (password string, err error) { + return GetKeyVaultValue(ctx, keyVaultUri, WekaDeploymentPasswordKey) +} + +// Get Weka deployment password if exists, otherwise get admin password +func GetWekaClusterCredentials(ctx context.Context, keyVaultUri string) (protocol.ClusterCreds, error) { + usename := WekaDeploymentUsername + password, err := GetWekaDeploymentPassword(ctx, keyVaultUri) + + var responseErr *azcore.ResponseError + if err != nil && errors.As(err, &responseErr) && responseErr.ErrorCode == "SecretNotFound" || err == nil && password == "" { + usename = WekaAdminUsername + password, err = GetWekaAdminPassword(ctx, keyVaultUri) + } + + credentials := protocol.ClusterCreds{ + Username: usename, + Password: password, + } + return credentials, err +} + +func SetWekaDeploymentPassword(ctx context.Context, keyVaultUri, password string) (err error) { + return SetKeyVaultValue(ctx, keyVaultUri, WekaDeploymentPasswordKey, password) +} + +func SetWekaAdminPassword(ctx context.Context, keyVaultUri, password string) (err error) { + return SetKeyVaultValue(ctx, keyVaultUri, WekaAdminPasswordKey, password) } func GetVmScaleSetName(prefix, clusterName string) string { @@ -1432,16 +1720,19 @@ func GetAzureInstanceNameCmd() string { return "curl -s -H Metadata:true --noproxy * http://169.254.169.254/metadata/instance?api-version=2021-02-01 | jq '.compute.name' | cut -c2- | rev | cut -c2- | rev" } -func ReadVmssConfig(ctx context.Context, storageName, containerName string) (vmssConfig VMSSConfig, err error) { +func ReadVmssConfig(ctx context.Context, vmssConfigStr string) (vmssConfig VMSSConfig, err error) { logger := logging.LoggerFromCtx(ctx) - params := BlobObjParams{ - StorageName: storageName, - ContainerName: containerName, - BlobName: "vmss-config", + if vmssConfigStr == "" { + err = fmt.Errorf("vmss config is not set") + logger.Error().Err(err).Msg("cannot read vmss config") + return } - asByteArray, err := ReadBlobObject(ctx, params) + + asByteArray := []byte(vmssConfigStr) + err = json.Unmarshal(asByteArray, &vmssConfig) if err != nil { + logger.Error().Err(err).Msg("cannot unmarshal vmss config") return } @@ -1449,12 +1740,6 @@ func ReadVmssConfig(ctx context.Context, storageName, containerName string) (vms hash := sha256.Sum256(asByteArray) hashStr := fmt.Sprintf("%x", hash) - err = json.Unmarshal(asByteArray, &vmssConfig) - if err != nil { - logger.Error().Err(err).Send() - return - } - // take first 16 characters of the hash vmssConfig.ConfigHash = hashStr[:16] return diff --git a/function-app/code/functions/clusterize/clusterize.go b/function-app/code/functions/clusterize/clusterize.go index e2e21144..7b7fe33d 100644 --- a/function-app/code/functions/clusterize/clusterize.go +++ b/function-app/code/functions/clusterize/clusterize.go @@ -11,8 +11,8 @@ import ( "weka-deployment/common" "weka-deployment/functions/azure_functions_def" - "github.com/rs/zerolog/log" "github.com/weka/go-cloud-lib/join" + "github.com/weka/go-cloud-lib/utils" "github.com/lithammer/dedent" @@ -23,14 +23,7 @@ import ( "github.com/weka/go-cloud-lib/protocol" ) -type AzureObsParams struct { - Name string - ContainerName string - AccessKey string - TieringSsdPercent string -} - -func GetObsScript(obsParams AzureObsParams) string { +func GetObsScript(obsParams common.AzureObsParams) string { template := ` TIERING_SSD_PERCENT=%s OBS_NAME=%s @@ -53,6 +46,10 @@ type ClusterizationParams struct { Location string Prefix string KeyVaultUri string + SubnetId string + PrivateDNSZoneId string + // if network access is disabled and private endpoints do not exist, create them with obs + CreateBlobPrivateEndpoint bool StateParams common.BlobObjParams InstallDpdk bool @@ -61,7 +58,7 @@ type ClusterizationParams struct { Cluster clusterize.ClusterParams NFSParams protocol.NFSParams NFSStateParams common.BlobObjParams - Obs AzureObsParams + Obs common.AzureObsParams FunctionAppName string } @@ -84,35 +81,88 @@ func GetShutdownScript() string { return dedent.Dedent(s) } -func HandleLastClusterVm(ctx context.Context, state protocol.ClusterState, p ClusterizationParams, funcDef functions_def.FunctionDef) (clusterizeScript string, err error) { +func PrepareWekaObs(ctx context.Context, p *ClusterizationParams) (err error) { logger := logging.LoggerFromCtx(ctx) - logger.Info().Msg("This is the last instance in the cluster, creating obs and clusterization script") - vmScaleSetName := common.GetVmScaleSetName(p.Prefix, p.Cluster.ClusterName) + noExistingObs := p.Obs.AccessKey == "" - if p.Cluster.SetObs { - if p.Obs.AccessKey == "" { - p.Obs.AccessKey, err = common.CreateStorageAccount( - ctx, p.SubscriptionId, p.ResourceGroupName, p.Obs.Name, p.Location, - ) - if err != nil { - err = fmt.Errorf("failed to create storage account: %w", err) - logger.Error().Err(err).Send() - return - } + if p.Obs.NetworkAccess == "Disabled" && noExistingObs && !p.CreateBlobPrivateEndpoint { + ignoredErr := fmt.Errorf("private endpoint creation is required for obs when public access is disabled") + logger.Error().Err(ignoredErr).Send() + + common.ReportMsg(ctx, p.Vm.Name, p.StateParams, "error", ignoredErr.Error()) + p.Cluster.SetObs = false + return nil + } + + if p.Obs.NetworkAccess == "Disabled" && p.CreateBlobPrivateEndpoint && p.PrivateDNSZoneId == "" { + ignoredErr := fmt.Errorf("private dns zone id is required for private endpoint creation when public access is disabled") + logger.Error().Err(ignoredErr).Send() + + common.ReportMsg(ctx, p.Vm.Name, p.StateParams, "error", ignoredErr.Error()) + p.Cluster.SetObs = false + return nil + } - err = common.CreateContainer(ctx, p.Obs.Name, p.Obs.ContainerName) + if noExistingObs { + p.Obs.AccessKey, err = common.CreateStorageAccount( + ctx, p.SubscriptionId, p.ResourceGroupName, p.Location, p.Obs, + ) + if err != nil { + err = fmt.Errorf("failed to create storage account: %w", err) + logger.Error().Err(err).Send() + return + } + + if p.Obs.NetworkAccess == "Disabled" && p.CreateBlobPrivateEndpoint { + endpointName := fmt.Sprintf("%s-pe", p.Obs.Name) + logger.Info().Msgf("public access is disabled for the storage account, creating private endpoint %s", endpointName) + + err = common.CreateStorageAccountBlobPrivateEndpoint(ctx, p.SubscriptionId, p.ResourceGroupName, p.Location, p.Obs.Name, endpointName, p.SubnetId, p.PrivateDNSZoneId) if err != nil { - err = fmt.Errorf("failed to create container: %w", err) + err = fmt.Errorf("failed to create private endpoint: %w", err) logger.Error().Err(err).Send() return } } } + // create container (if it doesn't exist) + err = common.CreateContainer(ctx, p.Obs.Name, p.Obs.ContainerName) + if err != nil { + err = fmt.Errorf("failed to create container: %w", err) + logger.Error().Err(err).Send() + return + } + return +} - wekaPassword, err := common.GetWekaClusterPassword(ctx, p.KeyVaultUri) +func HandleLastClusterVm(ctx context.Context, state protocol.ClusterState, p ClusterizationParams, funcDef functions_def.FunctionDef) (clusterizeScript string, err error) { + logger := logging.LoggerFromCtx(ctx) + logger.Info().Msg("This is the last instance in the cluster, creating obs and clusterization script") + + vmScaleSetName := common.GetVmScaleSetName(p.Prefix, p.Cluster.ClusterName) + + if p.Cluster.SetObs { + err = PrepareWekaObs(ctx, &p) + if err != nil { + logger.Error().Err(err).Send() + return + } + } + + logger.Info().Msg("setting weka admin password in secrets manager") + adminPassword := utils.GeneratePassword(16, 1, 1, 1) + err = common.SetWekaAdminPassword(ctx, p.KeyVaultUri, adminPassword) + if err != nil { + logger.Error().Err(err).Send() + return + } + + logger.Info().Msg("setting weka deployment password in key vault") + wekaServicePassword := utils.GeneratePassword(16, 1, 1, 1) + err = common.SetWekaDeploymentPassword(ctx, p.KeyVaultUri, wekaServicePassword) if err != nil { - err = fmt.Errorf("failed to get weka cluster password: %w", err) + err = fmt.Errorf("failed to set weka service password: %w", err) logger.Error().Err(err).Send() return } @@ -145,8 +195,6 @@ func HandleLastClusterVm(ctx context.Context, state protocol.ClusterState, p Clu clusterParams.VMNames = vmNamesList clusterParams.IPs = ipsList clusterParams.ObsScript = GetObsScript(p.Obs) - clusterParams.WekaPassword = wekaPassword - clusterParams.WekaUsername = common.WekaAdminUsername clusterParams.InstallDpdk = p.InstallDpdk clusterParams.FindDrivesScript = common.FindDrivesScript clusterParams.ClusterizationTarget = state.ClusterizationTarget @@ -194,20 +242,21 @@ func Clusterize(ctx context.Context, p ClusterizationParams) (clusterizeScript s func doNFSClusterize(ctx context.Context, p ClusterizationParams, funcDef functions_def.FunctionDef) (clusterizeScript string, err error) { nfsInterfaceGroupName := os.Getenv("NFS_INTERFACE_GROUP_NAME") - nfsClientGroupName := os.Getenv("NFS_CLIENT_GROUP_NAME") nfsProtocolgwsNum, _ := strconv.Atoi(os.Getenv("NFS_PROTOCOL_GATEWAYS_NUM")) nfsSecondaryIpsNum, _ := strconv.Atoi(os.Getenv("NFS_SECONDARY_IPS_NUM")) nfsVmssName := os.Getenv("NFS_VMSS_NAME") backendLbIp := os.Getenv("BACKEND_LB_IP") + logger := logging.LoggerFromCtx(ctx) + state, err := common.AddInstanceToState(ctx, p.SubscriptionId, p.ResourceGroupName, p.NFSStateParams, p.Vm) if err != nil { - log.Error().Err(err).Send() + logger.Error().Err(err).Send() return } msg := fmt.Sprintf("This (%s) is nfs instance %d/%d that is ready for joining the interface group", p.Vm.Name, len(state.Instances), nfsProtocolgwsNum) - log.Info().Msgf(msg) + logger.Info().Msgf(msg) if len(state.Instances) != nfsProtocolgwsNum { clusterizeScript = cloudCommon.GetScriptWithReport(msg, funcDef.GetFunctionCmdDefinition(functions_def.Report), p.Vm.Protocol) return @@ -229,19 +278,18 @@ func doNFSClusterize(ctx context.Context, p ClusterizationParams, funcDef functi secondaryIps, err := common.GetScaleSetSecondaryIps(ctx, vmssParams) if err != nil { err = fmt.Errorf("failed to get scale set secondary ips: %w", err) - log.Error().Err(err).Send() + logger.Error().Err(err).Send() return } if len(secondaryIps) < nfsSecondaryIpsNum { err = fmt.Errorf("not enough secondary ips in vmss %s: %d/%d", nfsVmssName, len(secondaryIps), nfsSecondaryIpsNum) - log.Error().Err(err).Send() + logger.Error().Err(err).Send() return } nfsParams := protocol.NFSParams{ InterfaceGroupName: nfsInterfaceGroupName, - ClientGroupName: nfsClientGroupName, SecondaryIps: secondaryIps, ContainersUid: containersUid, NicNames: nicNames, @@ -256,7 +304,7 @@ func doNFSClusterize(ctx context.Context, p ClusterizationParams, funcDef functi } clusterizeScript = scriptGenerator.GetNFSSetupScript() - log.Info().Msg("Clusterization script for NFS generated") + logger.Info().Msg("Clusterization script for NFS generated") return } @@ -300,12 +348,6 @@ func doClusterize(ctx context.Context, p ClusterizationParams, funcDef functions clusterizeScript = cloudCommon.GetErrorScript(err, reportFunction, p.Vm.Protocol) } } else { - wekaPassword, err2 := common.GetWekaClusterPassword(ctx, p.KeyVaultUri) - if err2 != nil { - logger.Error().Err(err2).Send() - return - } - vmsPrivateIps, err2 := common.GetVmsPrivateIps(ctx, vmssParams) if err2 != nil { err = fmt.Errorf("failed to get vms private ips: %w", err) @@ -320,9 +362,7 @@ func doClusterize(ctx context.Context, p ClusterizationParams, funcDef functions } joinParams := join.JoinParams{ - WekaUsername: common.WekaAdminUsername, - WekaPassword: wekaPassword, - IPs: ipsList, + IPs: ipsList, } joinScriptGenerator := join.JoinScriptGenerator{ @@ -349,12 +389,20 @@ func Handler(w http.ResponseWriter, r *http.Request) { obsName := os.Getenv("OBS_NAME") obsContainerName := os.Getenv("OBS_CONTAINER_NAME") obsAccessKey := os.Getenv("OBS_ACCESS_KEY") + obsNetworkAccess := os.Getenv("OBS_NETWORK_ACCESS") + obsAllowedSubnetsStr := os.Getenv("OBS_ALLOWED_SUBNETS") + obsAllowedSubnets := []string{} + obsAllowedPublicIpsStr := os.Getenv("OBS_ALLOWED_PUBLIC_IPS") + obsAllowedPublicIps := []string{} location := os.Getenv("LOCATION") tieringSsdPercent := os.Getenv("TIERING_SSD_PERCENT") tieringTargetSsdRetention, _ := strconv.Atoi(os.Getenv("TIERING_TARGET_SSD_RETENTION")) tieringStartDemote, _ := strconv.Atoi(os.Getenv("TIERING_START_DEMOTE")) prefix := os.Getenv("PREFIX") keyVaultUri := os.Getenv("KEY_VAULT_URI") + subnetId := os.Getenv("SUBNET_ID") + blobPrivateDnsZoneId := os.Getenv("BLOB_PRIVATE_DNS_ZONE_ID") + createblobPrivateEndpoint, _ := strconv.ParseBool(os.Getenv("CREATE_BLOB_PRIVATE_ENDPOINT")) // data protection-related vars stripeWidth, _ := strconv.Atoi(os.Getenv("STRIPE_WIDTH")) protectionLevel, _ := strconv.Atoi(os.Getenv("PROTECTION_LEVEL")) @@ -375,6 +423,13 @@ func Handler(w http.ResponseWriter, r *http.Request) { addFrontend = true } + if obsAllowedSubnetsStr != "" { + obsAllowedSubnets = strings.Split(obsAllowedSubnetsStr, ",") + } + if obsAllowedPublicIpsStr != "" { + obsAllowedPublicIps = strings.Split(obsAllowedPublicIpsStr, ",") + } + resData := make(map[string]interface{}) var invokeRequest common.InvokeRequest @@ -403,14 +458,17 @@ func Handler(w http.ResponseWriter, r *http.Request) { } params := ClusterizationParams{ - SubscriptionId: subscriptionId, - ResourceGroupName: resourceGroupName, - Location: location, - Prefix: prefix, - KeyVaultUri: keyVaultUri, - StateParams: common.BlobObjParams{StorageName: stateStorageName, ContainerName: stateContainerName, BlobName: stateBlobName}, - Vm: vm, - InstallDpdk: installDpdk, + SubscriptionId: subscriptionId, + ResourceGroupName: resourceGroupName, + Location: location, + Prefix: prefix, + KeyVaultUri: keyVaultUri, + SubnetId: subnetId, + PrivateDNSZoneId: blobPrivateDnsZoneId, + CreateBlobPrivateEndpoint: createblobPrivateEndpoint, + StateParams: common.BlobObjParams{StorageName: stateStorageName, ContainerName: stateContainerName, BlobName: stateBlobName}, + Vm: vm, + InstallDpdk: installDpdk, Cluster: clusterize.ClusterParams{ ClusterizationTarget: clusterizationTarget, ClusterName: clusterName, @@ -429,11 +487,14 @@ func Handler(w http.ResponseWriter, r *http.Request) { TieringTargetSSDRetention: tieringTargetSsdRetention, TieringStartDemote: tieringStartDemote, }, - Obs: AzureObsParams{ + Obs: common.AzureObsParams{ Name: obsName, ContainerName: obsContainerName, AccessKey: obsAccessKey, TieringSsdPercent: tieringSsdPercent, + NetworkAccess: obsNetworkAccess, + AllowedSubnets: obsAllowedSubnets, + AllowedPublicIps: obsAllowedPublicIps, }, NFSStateParams: common.BlobObjParams{StorageName: stateStorageName, ContainerName: nfsStateContainerName, BlobName: nfsStateBlobName}, FunctionAppName: functionAppName, diff --git a/function-app/code/functions/debug/debug.go b/function-app/code/functions/debug/debug.go index 8b634adf..718eb152 100644 --- a/function-app/code/functions/debug/debug.go +++ b/function-app/code/functions/debug/debug.go @@ -116,7 +116,7 @@ func Handler(w http.ResponseWriter, r *http.Request) { }, SetObs: setObs, }, - Obs: clusterizeFunc.AzureObsParams{ + Obs: common.AzureObsParams{ Name: obsName, ContainerName: obsContainerName, AccessKey: obsAccessKey, diff --git a/function-app/code/functions/deploy/deploy.go b/function-app/code/functions/deploy/deploy.go index 1807fd6f..8a62a4f1 100644 --- a/function-app/code/functions/deploy/deploy.go +++ b/function-app/code/functions/deploy/deploy.go @@ -86,12 +86,6 @@ func GetNfsDeployScript(ctx context.Context, funcDef functions_def.FunctionDef, return } - wekaPassword, err := common.GetWekaClusterPassword(ctx, p.KeyVaultUri) - if err != nil { - logger.Error().Err(err).Send() - return - } - var token string token, err = getWekaIoToken(ctx, p.KeyVaultUri) if err != nil { @@ -107,10 +101,7 @@ func GetNfsDeployScript(ctx context.Context, funcDef functions_def.FunctionDef, ProxyUrl: p.ProxyUrl, Gateways: p.Gateways, Protocol: protocol.NFS, - WekaUsername: common.WekaAdminUsername, - WekaPassword: wekaPassword, NFSInterfaceGroupName: p.NFSInterfaceGroupName, - NFSClientGroupName: p.NFSClientGroupName, NFSSecondaryIpsNum: p.NFSSecondaryIpsNum, ProtocolGatewayFeCoresNum: p.NFSGatewayFeCoresNum, LoadBalancerIP: p.BackendLbIp, @@ -226,12 +217,6 @@ func GetDeployScript(ctx context.Context, funcDef functions_def.FunctionDef, p A } bashScript = deployScriptGenerator.GetDeployScript() } else { - wekaPassword, err := common.GetWekaClusterPassword(ctx, p.KeyVaultUri) - if err != nil { - logger.Error().Err(err).Send() - return "", err - } - vmScaleSetName := common.GetVmScaleSetName(p.Prefix, p.ClusterName) vmssParams := &common.ScaleSetParams{ SubscriptionId: p.SubscriptionId, @@ -262,8 +247,6 @@ func GetDeployScript(ctx context.Context, funcDef functions_def.FunctionDef, p A } joinParams := join.JoinParams{ - WekaUsername: common.WekaAdminUsername, - WekaPassword: wekaPassword, IPs: ips, InstallDpdk: p.InstallDpdk, InstanceParams: instanceParams, @@ -331,7 +314,6 @@ func Handler(w http.ResponseWriter, r *http.Request) { diskSize, _ := strconv.Atoi(os.Getenv("DISK_SIZE")) // nfs params nfsInterfaceGroupName := os.Getenv("NFS_INTERFACE_GROUP_NAME") - nfsClientGroupName := os.Getenv("NFS_CLIENT_GROUP_NAME") nfsProtocolgwsNum, _ := strconv.Atoi(os.Getenv("NFS_PROTOCOL_GATEWAYS_NUM")) nfsStateContainerName := os.Getenv("NFS_STATE_CONTAINER_NAME") nfsStateBlobName := os.Getenv("NFS_STATE_BLOB_NAME") @@ -401,7 +383,6 @@ func Handler(w http.ResponseWriter, r *http.Request) { FunctionAppName: functionAppName, Gateways: getGateways(subnet, nicsNumInt), NFSInterfaceGroupName: nfsInterfaceGroupName, - NFSClientGroupName: nfsClientGroupName, NFSProtocolGWsNum: nfsProtocolgwsNum, NFSStateParams: common.BlobObjParams{StorageName: stateStorageName, ContainerName: nfsStateContainerName, BlobName: nfsStateBlobName}, NFSSecondaryIpsNum: nfsSecondaryIpsNum, diff --git a/function-app/code/functions/fetch/fetch.go b/function-app/code/functions/fetch/fetch.go index 99e48e26..ff1d2247 100644 --- a/function-app/code/functions/fetch/fetch.go +++ b/function-app/code/functions/fetch/fetch.go @@ -2,6 +2,7 @@ package fetch import ( "context" + "encoding/json" "fmt" "net/http" "os" @@ -38,6 +39,13 @@ func Handler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() logger := logging.LoggerFromCtx(ctx) + fetchRequest, err := parseFetchRequest(r) + if err != nil { + logger.Error().Err(err).Send() + common.WriteErrorResponse(w, err) + return + } + backendsStateParams := common.BlobObjParams{ StorageName: stateStorageName, ContainerName: stateContainerName, @@ -58,12 +66,36 @@ func Handler(w http.ResponseWriter, r *http.Request) { return } - wekaAdminPassword, err := common.GetWekaClusterPassword(ctx, keyVaultUri) - if err != nil { - err = fmt.Errorf("cannot get weka admin password: %v", err) - logger.Error().Err(err).Send() - common.WriteErrorResponse(w, err) - return + var wekaPassword string + var adminPassword string + username := common.WekaDeploymentUsername + + if fetchRequest.ShowAdminPassword { + adminPassword, err = common.GetWekaAdminPassword(ctx, keyVaultUri) + if err != nil { + err = fmt.Errorf("cannot get weka admin password: %w", err) + logger.Error().Err(err).Send() + common.WriteErrorResponse(w, err) + return + } + + wekaPassword, err = common.GetWekaDeploymentPassword(ctx, keyVaultUri) + if err != nil { + err = fmt.Errorf("cannot get weka deployment password: %w", err) + logger.Error().Err(err).Send() + common.WriteErrorResponse(w, err) + return + } + } else { + credentials, err := common.GetWekaClusterCredentials(ctx, keyVaultUri) + if err != nil { + err = fmt.Errorf("cannot get weka cluster password: %v", err) + logger.Error().Err(err).Send() + common.WriteErrorResponse(w, err) + return + } + wekaPassword = credentials.Password + username = credentials.Username } desiredCapacity, err := getCapacity(ctx, backendsStateParams) @@ -74,8 +106,9 @@ func Handler(w http.ResponseWriter, r *http.Request) { } response := protocol.HostGroupInfoResponse{ - Username: common.WekaAdminUsername, - Password: wekaAdminPassword, + Username: username, + Password: wekaPassword, + AdminPassword: adminPassword, WekaBackendsDesiredCapacity: desiredCapacity, WekaBackendInstances: instances, DownBackendsRemovalTimeout: downBackendsRemovalTimeout, @@ -162,3 +195,31 @@ func getCapacity(ctx context.Context, stateParams common.BlobObjParams) (desired desired = state.DesiredSize return } + +func parseFetchRequest(r *http.Request) (fetchRequest protocol.FetchRequest, err error) { + var invokeRequest common.InvokeRequest + + if err = json.NewDecoder(r.Body).Decode(&invokeRequest); err != nil { + err = fmt.Errorf("cannot decode the request: %w", err) + return + } + + var reqData map[string]interface{} + err = json.Unmarshal(invokeRequest.Data["req"], &reqData) + if err != nil { + err = fmt.Errorf("cannot unmarshal the request data: %w", err) + return + } + + if reqData["Body"] == nil { + return + } + + err = json.Unmarshal([]byte(reqData["Body"].(string)), &fetchRequest) + if err != nil { + err = fmt.Errorf("cannot unmarshal the request body: %w", err) + return + } + + return +} diff --git a/init-script.sh b/function-app/code/functions/scale_up/init_script.go old mode 100755 new mode 100644 similarity index 67% rename from init-script.sh rename to function-app/code/functions/scale_up/init_script.go index 43b977e6..f02368a5 --- a/init-script.sh +++ b/function-app/code/functions/scale_up/init_script.go @@ -1,38 +1,40 @@ -function retry { - local retry_max=$1 - local retry_sleep=$2 - shift 2 - local count=$retry_max - while [ $count -gt 0 ]; do - "$@" && break - count=$(($count - 1)) - sleep $retry_sleep - done - [ $count -eq 0 ] && { - echo "Retry failed [$retry_max]" - shutdown -h now - return 1 - } - return 0 -} +package scale_up + +import "fmt" + +var ( + initScript = `#!/bin/bash +set -ex + +# user data +%s -# retry for 2 minutes -# NOTE: in some cases it takes time for all access policies to be applied -retry 12 10 curl --fail ${report_url}?code="${function_app_default_key}" -H "Content-Type:application/json" -d "{\"hostname\": \"$HOSTNAME\", \"type\": \"progress\", \"message\": \"Running init script\"}" +DISK_SIZE=%d +NICS_NUM=%d +SUBNET_RANGE="%s" +APT_REPO_SERVER="%s" + +# report function definition +%s + +# deploy function definition +%s + +report "{\"hostname\": \"$HOSTNAME\", \"type\": \"progress\", \"message\": \"Running init script\"}" while fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock >/dev/null 2>&1; do sleep 2 done # set apt private repo -if [[ "${apt_repo_server}" ]]; then +if [[ "$APT_REPO_SERVER" ]]; then mv /etc/apt/sources.list /etc/apt/sources.list.bak - echo "deb ${apt_repo_server} focal main restricted universe" > /etc/apt/sources.list - echo "deb ${apt_repo_server} focal-updates main restricted" >> /etc/apt/sources.list + echo "deb $APT_REPO_SERVER focal main restricted universe" > /etc/apt/sources.list + echo "deb $APT_REPO_SERVER focal-updates main restricted" >> /etc/apt/sources.list apt update -y fi -for(( i=0; i<${nics_num}; i++ )); do +for(( i=0; i<$NICS_NUM; i++ )); do cat <<-EOF | sed -i "/ eth$i/r /dev/stdin" /etc/netplan/50-cloud-init.yaml mtu: 3900 EOF @@ -47,7 +49,7 @@ gateway=$(ip r | grep default | awk '{print $3}') eth=$(ifconfig | grep eth0 -C2 | grep 'inet ' | awk '{print $2}') cat <<-EOF | sed -i "/ set-name: eth0/r /dev/stdin" /etc/netplan/50-cloud-init.yaml routes: - - to: ${subnet_range} + - to: $SUBNET_RANGE via: $gateway metric: 200 table: 200 @@ -63,9 +65,9 @@ EOF netplan apply -if [[ ${nics_num} -gt 1 ]]; then +if [[ $NICS_NUM -gt 1 ]]; then are_routes_ready='ip route | grep eth1' - for(( i=2; i<${nics_num}; i++ )); do + for(( i=2; i<$NICS_NUM; i++ )); do are_routes_ready=$are_routes_ready' && ip route | grep eth'"$i" done cat >>/usr/sbin/remove-routes.sh < /tmp/deploy.sh 2>/tmp/deploy_err.log || [ ! -s /tmp/deploy.sh ]; do +while ! deploy "{\"name\": \"$compute_name:$HOSTNAME\"}" > /tmp/deploy.sh 2>/tmp/deploy_err.log || [ ! -s /tmp/deploy.sh ]; do echo "Retry $retry: waiting for deploy script generation success" cat /tmp/deploy_err.log retry=$((retry + 1)) @@ -155,10 +158,16 @@ mv /root/weka-prepackaged $weka_dir if [ $retry -gt 0 ]; then msg="Deploy script generation retried $retry times" echo "$msg" - curl -i "${report_url}?code=${function_app_default_key}" -H "Content-Type:application/json" -d "{\"hostname\": \"$HOSTNAME\", \"type\": \"debug\", \"message\": \"$msg\"}" + report "{\"hostname\": \"$HOSTNAME\", \"type\": \"debug\", \"message\": \"$msg\"}" fi echo "$(date -u): running deploy script" chmod +x /tmp/deploy.sh /tmp/deploy.sh 2>&1 | tee /tmp/weka_deploy.log +` +) + +func getInitScript(userData string, diskSize int, nicsNum int, subnetRange string, aptRepoServer string, reportFuncDef string, deployFuncDef string) string { + return fmt.Sprintf(initScript, userData, diskSize, nicsNum, subnetRange, aptRepoServer, reportFuncDef, deployFuncDef) +} diff --git a/function-app/code/functions/scale_up/scale_up.go b/function-app/code/functions/scale_up/scale_up.go index 6c597faa..5e21db8e 100644 --- a/function-app/code/functions/scale_up/scale_up.go +++ b/function-app/code/functions/scale_up/scale_up.go @@ -2,12 +2,16 @@ package scale_up import ( "context" + "encoding/base64" "fmt" "net/http" "os" + "strconv" "time" "weka-deployment/common" + "weka-deployment/functions/azure_functions_def" + "github.com/weka/go-cloud-lib/functions_def" "github.com/weka/go-cloud-lib/logging" "github.com/weka/go-cloud-lib/protocol" ) @@ -23,8 +27,51 @@ var ( nfsContainerName = os.Getenv("NFS_STATE_CONTAINER_NAME") nfsStateBlobName = os.Getenv("NFS_STATE_BLOB_NAME") nfsScaleSetName = os.Getenv("NFS_VMSS_NAME") + vmssConfigStr = os.Getenv("VMSS_CONFIG") + // initial state of the cluster + initialClusterSize, _ = strconv.Atoi(os.Getenv("INITIAL_CLUSTER_SIZE")) + clusterizeTarget, _ = strconv.Atoi(os.Getenv("CLUSTERIZATION_TARGET")) + initialNfsSize, _ = strconv.Atoi(os.Getenv("NFS_PROTOCOL_GATEWAYS_NUM")) + clusterInitialState = protocol.ClusterState{ + InitialSize: initialClusterSize, + DesiredSize: initialClusterSize, + ClusterizationTarget: clusterizeTarget, + } + nfsInitialState = protocol.ClusterState{ + InitialSize: initialNfsSize, + DesiredSize: initialNfsSize, + ClusterizationTarget: initialNfsSize, + } ) +func getBackendCustomDataScript(ctx context.Context) (customData string, err error) { + functionAppName := os.Getenv("FUNCTION_APP_NAME") + keyVaultUri := os.Getenv("KEY_VAULT_URI") + diskSize, _ := strconv.Atoi(os.Getenv("DISK_SIZE")) + nicsNum, _ := strconv.Atoi(os.Getenv("NICS_NUM")) + subnet := os.Getenv("SUBNET") + aptRepo := os.Getenv("APT_REPO_SERVER") + userData := os.Getenv("USER_DATA") + + logger := logging.LoggerFromCtx(ctx) + + functionAppKey, err := common.GetKeyVaultValue(ctx, keyVaultUri, "function-app-default-key") + if err != nil { + logger.Error().Err(err).Msg("cannot get function app key") + return + } + + baseFunctionUrl := fmt.Sprintf("https://%s.azurewebsites.net/api/", functionAppName) + funcDef := azure_functions_def.NewFuncDef(baseFunctionUrl, functionAppKey) + reportFunction := funcDef.GetFunctionCmdDefinition(functions_def.Report) + deployFunction := funcDef.GetFunctionCmdDefinition(functions_def.Deploy) + + customDataStr := getInitScript(userData, diskSize, nicsNum, subnet, aptRepo, reportFunction, deployFunction) + // base64 encode the custom data + customData = base64.StdEncoding.EncodeToString([]byte(customDataStr)) + return +} + func Handler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() logger := logging.LoggerFromCtx(ctx) @@ -36,7 +83,7 @@ func Handler(w http.ResponseWriter, r *http.Request) { BlobName: stateBlobName, } - state, err := common.ReadState(ctx, stateParams) + state, err := common.ReadStateOrCreateNew(ctx, stateParams, clusterInitialState) if err != nil { logger.Error().Err(err).Msg("cannot read state") common.WriteErrorResponse(w, err) @@ -58,7 +105,7 @@ func Handler(w http.ResponseWriter, r *http.Request) { } // get expected vmss config - vmssConfig, err := common.ReadVmssConfig(ctx, stateStorageName, stateContainerName) + vmssConfig, err := common.ReadVmssConfig(ctx, vmssConfigStr) if err != nil { logger.Error().Err(err).Msgf("cannot read vmss config") common.WriteErrorResponse(w, err) @@ -138,7 +185,7 @@ func handleNFSScaleUp(ctx context.Context) (message string, err error) { ContainerName: nfsContainerName, BlobName: nfsStateBlobName, } - nfsState, err := common.ReadState(ctx, nfsStateParams) + nfsState, err := common.ReadStateOrCreateNew(ctx, nfsStateParams, nfsInitialState) if err != nil { logger.Error().Err(err).Msg("cannot read NFS state") return @@ -167,12 +214,18 @@ func handleNFSScaleUp(ctx context.Context) (message string, err error) { return } -func createVmss(ctx context.Context, vmssConfig *common.VMSSConfig, vmssName string, vmssSize int) error { +func createVmss(ctx context.Context, vmssConfig *common.VMSSConfig, vmssName string, vmssSize int) (err error) { logger := logging.LoggerFromCtx(ctx) vmssConfigHash := vmssConfig.ConfigHash + vmssConfig.CustomData, err = getBackendCustomDataScript(ctx) + if err != nil { + logger.Error().Err(err).Msg("cannot get custom data script") + return err + } + logger.Info().Msgf("creating new vmss %s of size %d", vmssName, vmssSize) - _, err := common.CreateOrUpdateVmss(ctx, subscriptionId, resourceGroupName, vmssName, vmssConfigHash, *vmssConfig, vmssSize) + _, err = common.CreateOrUpdateVmss(ctx, subscriptionId, resourceGroupName, vmssName, vmssConfigHash, *vmssConfig, vmssSize) if err != nil { return err } @@ -180,7 +233,7 @@ func createVmss(ctx context.Context, vmssConfig *common.VMSSConfig, vmssName str return nil } -func handleVmssUpdate(ctx context.Context, currentConfig, newConfig *common.VMSSConfig, stateParams common.BlobObjParams, desiredSize int) error { +func handleVmssUpdate(ctx context.Context, currentConfig, newConfig *common.VMSSConfig, stateParams common.BlobObjParams, desiredSize int) (err error) { logger := logging.LoggerFromCtx(ctx) newConfigHash := newConfig.ConfigHash @@ -200,7 +253,13 @@ func handleVmssUpdate(ctx context.Context, currentConfig, newConfig *common.VMSS return common.AddClusterUpdate(ctx, stateParams, update) } - _, err := common.CreateOrUpdateVmss(ctx, subscriptionId, resourceGroupName, currentConfig.Name, newConfigHash, *newConfig, desiredSize) + newConfig.CustomData, err = getBackendCustomDataScript(ctx) + if err != nil { + logger.Error().Err(err).Msg("cannot get custom data script") + return err + } + + _, err = common.CreateOrUpdateVmss(ctx, subscriptionId, resourceGroupName, currentConfig.Name, newConfigHash, *newConfig, desiredSize) if err != nil { logger.Error().Err(err).Msgf("cannot update vmss %s", currentConfig.Name) errStr := err.Error() diff --git a/function-app/code/functions/status/status.go b/function-app/code/functions/status/status.go index 3d08006c..fc66c053 100644 --- a/function-app/code/functions/status/status.go +++ b/function-app/code/functions/status/status.go @@ -136,13 +136,13 @@ func GetClusterStatus(ctx context.Context, vmssParams *common.ScaleSetParams, st return } - wekaPassword, err := common.GetWekaClusterPassword(ctx, keyVaultUri) + credentials, err := common.GetWekaClusterCredentials(ctx, keyVaultUri) if err != nil { return } jrpcBuilder := func(ip string) *jrpc.BaseClient { - return connectors.NewJrpcClient(ctx, ip, weka.ManagementJrpcPort, common.WekaAdminUsername, wekaPassword) + return connectors.NewJrpcClient(ctx, ip, weka.ManagementJrpcPort, credentials.Username, credentials.Password) } vmIps, err := common.GetVmsPrivateIps(ctx, vmssParams) @@ -180,8 +180,8 @@ func GetClusterStatus(ctx context.Context, vmssParams *common.ScaleSetParams, st return } -func GetRefreshStatus(ctx context.Context, vmssParams *common.ScaleSetParams, stateParams common.BlobObjParams, extended bool) (*common.VMSSStateVerbose, error) { - vmssConfig, err := common.ReadVmssConfig(ctx, stateParams.StorageName, stateParams.ContainerName) +func GetRefreshStatus(ctx context.Context, vmssParams *common.ScaleSetParams, stateParams common.BlobObjParams, vmssConfigStr string, extended bool) (*common.VMSSStateVerbose, error) { + vmssConfig, err := common.ReadVmssConfig(ctx, vmssConfigStr) if err != nil { return nil, err } @@ -235,6 +235,7 @@ func Handler(w http.ResponseWriter, r *http.Request) { nfsStateContainerName := os.Getenv("NFS_STATE_CONTAINER_NAME") nfsStateBlobName := os.Getenv("NFS_STATE_BLOB_NAME") nfsScaleSetName := os.Getenv("NFS_VMSS_NAME") + vmssConfigStr := os.Getenv("VMSS_CONFIG") ctx := r.Context() logger := logging.LoggerFromCtx(ctx) @@ -298,9 +299,9 @@ func Handler(w http.ResponseWriter, r *http.Request) { } else if requestBody.Type == "progress" { result, err = GetReports(ctx, stateParams, vmssParams) } else if requestBody.Type == "vmss" { - result, err = GetRefreshStatus(ctx, vmssParams, stateParams, false) + result, err = GetRefreshStatus(ctx, vmssParams, stateParams, vmssConfigStr, false) } else if requestBody.Type == "vmss-extended" { - result, err = GetRefreshStatus(ctx, vmssParams, stateParams, true) + result, err = GetRefreshStatus(ctx, vmssParams, stateParams, vmssConfigStr, true) } else { result = "Invalid status type" } diff --git a/function-app/code/go.mod b/function-app/code/go.mod index 38a7ba2a..6ab9b82e 100644 --- a/function-app/code/go.mod +++ b/function-app/code/go.mod @@ -14,8 +14,7 @@ require ( github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 github.com/lithammer/dedent v1.1.0 - github.com/rs/zerolog v1.29.0 - github.com/weka/go-cloud-lib v0.0.0-20240711150309-5e177acf14bf + github.com/weka/go-cloud-lib v0.0.0-20240730090621-daac1ef4a039 ) require ( @@ -30,6 +29,7 @@ require ( github.com/mattn/go-isatty v0.0.14 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/rs/xid v1.4.0 // indirect + github.com/rs/zerolog v1.29.0 // indirect golang.org/x/crypto v0.24.0 // indirect golang.org/x/net v0.26.0 // indirect golang.org/x/oauth2 v0.6.0 // indirect diff --git a/function-app/code/go.sum b/function-app/code/go.sum index ea0f30c1..bf02728f 100644 --- a/function-app/code/go.sum +++ b/function-app/code/go.sum @@ -55,8 +55,8 @@ github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w= github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/weka/go-cloud-lib v0.0.0-20240711150309-5e177acf14bf h1:o0s3CopCNc8D97E10n9yU0mHg0Jq+HhMNm7HWqhjUzM= -github.com/weka/go-cloud-lib v0.0.0-20240711150309-5e177acf14bf/go.mod h1:FCQuk2bLvtDHe2Kjsu0oInJP1VOVsuxqPGHGMmVIPMg= +github.com/weka/go-cloud-lib v0.0.0-20240730090621-daac1ef4a039 h1:PVACFcS9r4sgtm6Z9UiN7/LMoDySTPP0Ghsxj+MSOtQ= +github.com/weka/go-cloud-lib v0.0.0-20240730090621-daac1ef4a039/go.mod h1:FCQuk2bLvtDHe2Kjsu0oInJP1VOVsuxqPGHGMmVIPMg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= diff --git a/function-app/distribution/README.md b/function-app/distribution/README.md index 0ecb8403..d231c971 100644 --- a/function-app/distribution/README.md +++ b/function-app/distribution/README.md @@ -4,13 +4,13 @@ | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.4.6 | -| [azurerm](#requirement\_azurerm) | ~>3.75.0 | +| [azurerm](#requirement\_azurerm) | ~>3.114.0 | ## Providers | Name | Version | |------|---------| -| [azurerm](#provider\_azurerm) | ~>3.75.0 | +| [azurerm](#provider\_azurerm) | ~>3.114.0 | ## Modules diff --git a/function-app/distribution/create_sa/README.md b/function-app/distribution/create_sa/README.md index 450427f1..d597152a 100644 --- a/function-app/distribution/create_sa/README.md +++ b/function-app/distribution/create_sa/README.md @@ -4,13 +4,13 @@ | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.4.6 | -| [azurerm](#requirement\_azurerm) | ~>3.75.0 | +| [azurerm](#requirement\_azurerm) | ~>3.114.0 | ## Providers | Name | Version | |------|---------| -| [azurerm](#provider\_azurerm) | ~>3.75.0 | +| [azurerm](#provider\_azurerm) | ~>3.114.0 | ## Modules diff --git a/function-app/distribution/create_sa/versions.tf b/function-app/distribution/create_sa/versions.tf index 281dc878..202d92cc 100644 --- a/function-app/distribution/create_sa/versions.tf +++ b/function-app/distribution/create_sa/versions.tf @@ -3,7 +3,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "~>3.75.0" + version = "~>3.114.0" } } } diff --git a/function-app/distribution/state.tf b/function-app/distribution/state.tf index 674c0acc..e56ef43b 100644 --- a/function-app/distribution/state.tf +++ b/function-app/distribution/state.tf @@ -3,7 +3,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "~>3.75.0" + version = "~>3.114.0" } } # https://learn.microsoft.com/en-us/azure/developer/terraform/store-state-in-azure-storage?tabs=azure-cli diff --git a/functions.tf b/functions.tf index 2db99ee9..8a89bda7 100644 --- a/functions.tf +++ b/functions.tf @@ -1,28 +1,196 @@ locals { - create_private_function = var.function_access_restriction_enabled ? 1 : 0 - stripe_width_calculated = var.cluster_size - var.protection_level - 1 - stripe_width = local.stripe_width_calculated < 16 ? local.stripe_width_calculated : 16 - location = data.azurerm_resource_group.rg.location - function_app_zip_name = "${var.function_app_dist}/${var.function_app_version}.zip" - weka_sa = "${var.function_app_storage_account_prefix}${local.location}" - weka_sa_container = "${var.function_app_storage_account_container_prefix}${local.location}" - function_code_path = "${path.module}/function-app/code" - function_app_code_hash = md5(join("", [for f in fileset(local.function_code_path, "**") : filemd5("${local.function_code_path}/${f}")])) - get_compute_memory_index = var.set_dedicated_fe_container ? 1 : 0 - deployment_storage_account_id = var.deployment_storage_account_name == "" ? azurerm_storage_account.deployment_sa[0].id : data.azurerm_storage_account.deployment_blob[0].id - deployment_storage_account_name = var.deployment_storage_account_name == "" ? azurerm_storage_account.deployment_sa[0].name : var.deployment_storage_account_name - deployment_container_name = var.deployment_container_name == "" ? azurerm_storage_container.deployment[0].name : var.deployment_container_name - obs_storage_account_name = var.tiering_obs_name == "" ? "${substr("${local.alphanumeric_prefix_name}${local.alphanumeric_cluster_name}", 0, 21)}obs" : var.tiering_obs_name - obs_container_name = var.tiering_obs_container_name == "" ? "${var.prefix}-${var.cluster_name}-obs" : var.tiering_obs_container_name - function_app_name = "${local.alphanumeric_prefix_name}-${local.alphanumeric_cluster_name}-function-app" - install_weka_url = var.install_weka_url != "" ? var.install_weka_url : "https://$TOKEN@get.weka.io/dist/v1/install/${var.weka_version}/${var.weka_version}" - supported_regions = split("\n", replace(chomp(file("${path.module}/supported_regions/${var.function_app_dist}.txt")), "\r", "")) + create_private_function = var.function_access_restriction_enabled ? 1 : 0 + stripe_width_calculated = var.cluster_size - var.protection_level - 1 + stripe_width = local.stripe_width_calculated < 16 ? local.stripe_width_calculated : 16 + location = data.azurerm_resource_group.rg.location + read_remote_function_zip = !var.read_function_zip_from_storage_account + function_app_zip_name = local.read_remote_function_zip ? "${var.function_app_dist}/${var.function_app_version}.zip" : var.deployment_function_app_code_blob + weka_sa = local.read_remote_function_zip ? "${var.function_app_storage_account_prefix}${local.location}" : var.deployment_storage_account_name + weka_sa_container = local.read_remote_function_zip ? "${var.function_app_storage_account_container_prefix}${local.location}" : var.deployment_container_name + function_app_blob_sas = local.read_remote_function_zip ? "" : data.azurerm_storage_account_blob_container_sas.function_app_code_sas[0].sas + function_code_path = "${path.module}/function-app/code" + function_app_code_hash = md5(join("", [for f in fileset(local.function_code_path, "**") : filemd5("${local.function_code_path}/${f}")])) + get_compute_memory_index = var.set_dedicated_fe_container ? 1 : 0 + obs_storage_account_name = var.tiering_obs_name == "" ? "${substr("${local.alphanumeric_prefix_name}${local.alphanumeric_cluster_name}", 0, 21)}obs" : var.tiering_obs_name + obs_container_name = var.tiering_obs_container_name == "" ? "${var.prefix}-${var.cluster_name}-obs" : var.tiering_obs_container_name + function_app_name = "${local.alphanumeric_prefix_name}-${local.alphanumeric_cluster_name}-function-app" + install_weka_url = var.install_weka_url != "" ? var.install_weka_url : "https://$TOKEN@get.weka.io/dist/v1/install/${var.weka_version}/${var.weka_version}" + supported_regions = split("\n", replace(chomp(file("${path.module}/supported_regions/${var.function_app_dist}.txt")), "\r", "")) # log analytics for function app log_analytics_workspace_id = var.enable_application_insights ? var.log_analytics_workspace_id == "" ? azurerm_log_analytics_workspace.la_workspace[0].id : var.log_analytics_workspace_id : "" application_insights_id = var.enable_application_insights ? var.application_insights_name == "" ? azurerm_application_insights.application_insights[0].id : data.azurerm_application_insights.application_insights[0].id : "" insights_instrumenation_key = var.enable_application_insights ? var.application_insights_name == "" ? azurerm_application_insights.application_insights[0].instrumentation_key : data.azurerm_application_insights.application_insights[0].instrumentation_key : "" # nfs autoscaling - nfs_deployment_container_name = var.nfs_deployment_container_name == "" ? azurerm_storage_container.nfs_deployment[0].name : var.nfs_deployment_container_name + nfs_deployment_container_name = var.nfs_deployment_container_name == "" ? "${local.alphanumeric_prefix_name}${local.alphanumeric_cluster_name}-protocol-deployment" : var.nfs_deployment_container_name + + clusterization_target = var.clusterization_target != null ? var.clusterization_target : min(var.cluster_size, max(20, ceil(var.cluster_size * 0.8))) + # fields that depend on LB creation + vmss_health_probe_id = var.create_lb ? azurerm_lb_probe.backend_lb_probe[0].id : null + lb_backend_pool_ids = var.create_lb ? [azurerm_lb_backend_address_pool.lb_backend_pool[0].id] : [] + + vmss_config = jsonencode({ + name = "${var.prefix}-${var.cluster_name}-vmss" + location = data.azurerm_resource_group.rg.location + zones = var.zone != null ? [var.zone] : [] + resource_group_name = var.rg_name + sku = var.instance_type + upgrade_mode = "Manual" + health_probe_id = local.vmss_health_probe_id + admin_username = var.vm_username + ssh_public_key = local.public_ssh_key + computer_name_prefix = "${var.prefix}-${var.cluster_name}-backend" + custom_data = "" + disable_password_authentication = true + proximity_placement_group_id = local.placement_group_id + single_placement_group = var.vmss_single_placement_group + source_image_id = var.source_image_id + overprovision = false + orchestration_mode = "Uniform" + tags = merge(var.tags_map, { + "weka_cluster" : var.cluster_name, + "user_id" : data.azurerm_client_config.current.object_id, + }) + + os_disk = { + caching = "ReadWrite" + storage_account_type = "Premium_LRS" + } + + data_disk = { + lun = 0 + caching = "None" + create_option = "Empty" + disk_size_gb = local.disk_size + storage_account_type = "Premium_LRS" + } + + identity = { + type = "UserAssigned" + identity_ids = [local.vmss_identity_id] + } + + primary_nic = { + name = "${var.prefix}-${var.cluster_name}-backend-nic-0" + network_security_group_id = local.sg_id + enable_accelerated_networking = var.install_cluster_dpdk + + ip_configurations = [{ + primary = true + subnet_id = data.azurerm_subnet.subnet.id + load_balancer_backend_address_pool_ids = local.lb_backend_pool_ids + public_ip_address = { + assign = local.assign_public_ip + name = "${var.prefix}-${var.cluster_name}-public-ip" + domain_name_label = "${var.prefix}-${var.cluster_name}-backend" + } + }] + } + + secondary_nics = { + number = local.nics_numbers - 1 + name_prefix = "${var.prefix}-${var.cluster_name}-backend-nic" + network_security_group_id = local.sg_id + enable_accelerated_networking = var.install_cluster_dpdk + ip_configurations = [{ + primary = true + subnet_id = data.azurerm_subnet.subnet.id + load_balancer_backend_address_pool_ids = local.lb_backend_pool_ids + }] + } + }) + + # function app settings + initial_app_settings = { + "USER_ASSIGNED_CLIENT_ID" = local.function_app_identity_client_id + "STATE_STORAGE_NAME" = local.deployment_storage_account_name + "STATE_CONTAINER_NAME" = local.deployment_container_name + "STATE_BLOB_NAME" = "state" + "HOSTS_NUM" = var.cluster_size + "CLUSTER_NAME" = var.cluster_name + "PROTECTION_LEVEL" = var.protection_level + "STRIPE_WIDTH" = var.stripe_width != -1 ? var.stripe_width : local.stripe_width + "HOTSPARE" = var.hotspare + "VM_USERNAME" = var.vm_username + "SUBSCRIPTION_ID" = var.subscription_id + "RESOURCE_GROUP_NAME" = data.azurerm_resource_group.rg.name + "LOCATION" = data.azurerm_resource_group.rg.location + "SET_OBS" = var.tiering_enable_obs_integration + "CREATE_CONFIG_FS" = (var.smbw_enabled && var.smb_setup_protocol) || var.s3_setup_protocol + "OBS_NAME" = local.obs_storage_account_name + "OBS_CONTAINER_NAME" = local.obs_container_name + "OBS_ACCESS_KEY" = var.tiering_blob_obs_access_key + "OBS_NETWORK_ACCESS" = var.storage_account_public_network_access + "OBS_ALLOWED_SUBNETS" = join(",", local.sa_public_access_for_vnet ? [data.azurerm_subnet.subnet.id, local.function_app_subnet_delegation_id] : []) + "OBS_ALLOWED_PUBLIC_IPS" = join(",", var.storage_account_allowed_ips) + DRIVE_CONTAINER_CORES_NUM = var.containers_config_map[var.instance_type].drive + COMPUTE_CONTAINER_CORES_NUM = var.set_dedicated_fe_container == false ? var.containers_config_map[var.instance_type].compute + 1 : var.containers_config_map[var.instance_type].compute + FRONTEND_CONTAINER_CORES_NUM = var.set_dedicated_fe_container == false ? 0 : var.containers_config_map[var.instance_type].frontend + COMPUTE_MEMORY = var.containers_config_map[var.instance_type].memory[local.get_compute_memory_index] + DISK_SIZE = local.disk_size + "NVMES_NUM" = var.containers_config_map[var.instance_type].nvme + "TIERING_SSD_PERCENT" = var.tiering_enable_ssd_percent + "TIERING_TARGET_SSD_RETENTION" = var.tiering_obs_target_ssd_retention + "TIERING_START_DEMOTE" = var.tiering_obs_start_demote + "PREFIX" = var.prefix + "KEY_VAULT_URI" = azurerm_key_vault.key_vault.vault_uri + "INSTALL_DPDK" = var.install_cluster_dpdk + "NICS_NUM" = var.containers_config_map[var.instance_type].nics + "INSTALL_URL" = local.install_weka_url + "LOG_LEVEL" = var.function_app_log_level + "SUBNET" = local.subnet_range + "SUBNET_ID" = data.azurerm_subnet.subnet.id + "BLOB_PRIVATE_DNS_ZONE_ID" = var.create_storage_account_private_links ? azurerm_private_dns_zone.blob[0].id : local.sa_public_access_disabled ? data.azurerm_private_dns_zone.blob[0].id : "" + "CREATE_BLOB_PRIVATE_ENDPOINT" = var.create_storage_account_private_links && local.sa_public_access_disabled + FUNCTION_APP_NAME = local.function_app_name + PROXY_URL = var.proxy_url + WEKA_HOME_URL = var.weka_home_url + POST_CLUSTER_CREATION_SCRIPT = var.script_post_cluster_creation + PRE_START_IO_SCRIPT = var.script_pre_start_io + DOWN_BACKENDS_REMOVAL_TIMEOUT = var.debug_down_backends_removal_timeout + + https_only = true + FUNCTION_APP_EDIT_MODE = "readonly" + HASH = var.function_app_version + WEBSITE_RUN_FROM_PACKAGE = "https://${local.weka_sa}.blob.core.windows.net/${local.weka_sa_container}/${local.function_app_zip_name}${local.function_app_blob_sas}" + + NFS_STATE_CONTAINER_NAME = local.nfs_deployment_container_name + NFS_STATE_BLOB_NAME = "nfs_state" + NFS_INTERFACE_GROUP_NAME = var.nfs_interface_group_name + NFS_SECONDARY_IPS_NUM = var.nfs_protocol_gateway_secondary_ips_per_nic + NFS_PROTOCOL_GATEWAY_FE_CORES_NUM = var.nfs_protocol_gateway_fe_cores_num + NFS_PROTOCOL_GATEWAYS_NUM = var.nfs_protocol_gateways_number + NFS_VMSS_NAME = var.nfs_protocol_gateways_number > 0 ? "${var.prefix}-${var.cluster_name}-nfs-protocol-gateway-vmss" : "" + NFS_DISK_SIZE = var.nfs_protocol_gateway_disk_size + SMB_DISK_SIZE = var.smb_protocol_gateway_disk_size + S3_DISK_SIZE = var.s3_protocol_gateway_disk_size + SMB_PROTOCOL_GATEWAY_FE_CORES_NUM = var.smb_protocol_gateway_fe_cores_num + S3_PROTOCOL_GATEWAY_FE_CORES_NUM = var.s3_protocol_gateway_fe_cores_num + TRACES_PER_FRONTEND = var.traces_per_ionode + + BACKEND_LB_IP = var.create_lb ? azurerm_lb.backend_lb[0].private_ip_address : "" + # state + INITIAL_CLUSTER_SIZE = var.cluster_size + CLUSTERIZATION_TARGET = local.clusterization_target + VMSS_CONFIG = local.vmss_config + # init script inputs + APT_REPO_SERVER = var.apt_repo_server + USER_DATA = var.user_data + } + + secured_storage_account_app_settings = { + # "AzureWebJobsStorage" and "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING" are not needed as we set storage_account_access_key + "WEBSITE_CONTENTSHARE" = local.deployment_file_share_name + "WEBSITE_CONTENTOVERVNET" = 1 + "WEBSITE_VNET_ROUTE_ALL" = 1 + } + dns_storgage_account_app_settings = { + "WEBSITE_DNS_SERVER" = "168.63.129.16" + } + merged_secured_storage_account_app_settings = var.create_storage_account_private_links ? merge(local.secured_storage_account_app_settings, local.dns_storgage_account_app_settings) : local.secured_storage_account_app_settings + + app_settings = local.sa_public_access_enabled ? local.initial_app_settings : merge(local.initial_app_settings, local.merged_secured_storage_account_app_settings) + + function_app_subnet_delegation_id = var.function_app_subnet_delegation_id == "" ? module.function_app_subnet_delegation[0].id : var.function_app_subnet_delegation_id } resource "azurerm_log_analytics_workspace" "la_workspace" { @@ -96,33 +264,22 @@ resource "azurerm_service_plan" "app_service_plan" { } } -resource "azurerm_subnet" "subnet_delegation" { - count = var.function_app_subnet_delegation_id == "" ? 1 : 0 - name = "${var.prefix}-${var.cluster_name}-subnet-delegation" - resource_group_name = local.vnet_rg_name - virtual_network_name = local.vnet_name - address_prefixes = [var.function_app_subnet_delegation_cidr] - service_endpoints = ["Microsoft.Storage", "Microsoft.KeyVault", "Microsoft.Web"] - delegation { - name = "subnet-delegation" - service_delegation { - name = "Microsoft.Web/serverFarms" - actions = ["Microsoft.Network/virtualNetworks/subnets/action"] - } - } -} - resource "azurerm_linux_function_app" "function_app" { name = local.function_app_name resource_group_name = data.azurerm_resource_group.rg.name location = data.azurerm_resource_group.rg.location service_plan_id = azurerm_service_plan.app_service_plan.id storage_account_name = local.deployment_storage_account_name - storage_account_access_key = var.deployment_storage_account_access_key == "" ? azurerm_storage_account.deployment_sa[0].primary_access_key : var.deployment_storage_account_access_key + storage_account_access_key = local.deployment_sa_access_key https_only = true - virtual_network_subnet_id = var.function_app_subnet_delegation_id == "" ? azurerm_subnet.subnet_delegation[0].id : var.function_app_subnet_delegation_id + virtual_network_subnet_id = local.function_app_subnet_delegation_id + site_config { - vnet_route_all_enabled = true + vnet_route_all_enabled = true + application_insights_key = local.insights_instrumenation_key + application_stack { + use_custom_runtime = true + } dynamic "ip_restriction" { for_each = range(local.create_private_function) content { @@ -133,9 +290,9 @@ resource "azurerm_linux_function_app" "function_app" { } } dynamic "ip_restriction" { - for_each = range(local.create_private_function) + for_each = range(local.sa_public_access_enabled ? local.create_private_function : 0) content { - virtual_network_subnet_id = var.logic_app_subnet_delegation_id == "" ? azurerm_subnet.logicapp_subnet_delegation[0].id : var.logic_app_subnet_delegation_id + virtual_network_subnet_id = var.logic_app_subnet_delegation_id == "" ? module.logic_app_subnet_delegation[0].id : var.logic_app_subnet_delegation_id action = "Allow" priority = 301 name = "VirtualNetwork" @@ -143,74 +300,7 @@ resource "azurerm_linux_function_app" "function_app" { } } - app_settings = { - "USER_ASSIGNED_CLIENT_ID" = local.function_app_identity_client_id - "APPINSIGHTS_INSTRUMENTATIONKEY" = local.insights_instrumenation_key - "STATE_STORAGE_NAME" = local.deployment_storage_account_name - "STATE_CONTAINER_NAME" = local.deployment_container_name - "STATE_BLOB_NAME" = "state" - "HOSTS_NUM" = var.cluster_size - "CLUSTER_NAME" = var.cluster_name - "PROTECTION_LEVEL" = var.protection_level - "STRIPE_WIDTH" = var.stripe_width != -1 ? var.stripe_width : local.stripe_width - "HOTSPARE" = var.hotspare - "VM_USERNAME" = var.vm_username - "SUBSCRIPTION_ID" = var.subscription_id - "RESOURCE_GROUP_NAME" = data.azurerm_resource_group.rg.name - "LOCATION" = data.azurerm_resource_group.rg.location - "SET_OBS" = var.tiering_enable_obs_integration - "CREATE_CONFIG_FS" = (var.smbw_enabled && var.smb_setup_protocol) || var.s3_setup_protocol - "OBS_NAME" = local.obs_storage_account_name - "OBS_CONTAINER_NAME" = local.obs_container_name - "OBS_ACCESS_KEY" = var.tiering_blob_obs_access_key - DRIVE_CONTAINER_CORES_NUM = var.containers_config_map[var.instance_type].drive - COMPUTE_CONTAINER_CORES_NUM = var.set_dedicated_fe_container == false ? var.containers_config_map[var.instance_type].compute + 1 : var.containers_config_map[var.instance_type].compute - FRONTEND_CONTAINER_CORES_NUM = var.set_dedicated_fe_container == false ? 0 : var.containers_config_map[var.instance_type].frontend - COMPUTE_MEMORY = var.containers_config_map[var.instance_type].memory[local.get_compute_memory_index] - DISK_SIZE = local.disk_size - "NVMES_NUM" = var.containers_config_map[var.instance_type].nvme - "TIERING_SSD_PERCENT" = var.tiering_enable_ssd_percent - "TIERING_TARGET_SSD_RETENTION" = var.tiering_obs_target_ssd_retention - "TIERING_START_DEMOTE" = var.tiering_obs_start_demote - "PREFIX" = var.prefix - "KEY_VAULT_URI" = azurerm_key_vault.key_vault.vault_uri - "INSTALL_DPDK" = var.install_cluster_dpdk - "NICS_NUM" = var.containers_config_map[var.instance_type].nics - "INSTALL_URL" = local.install_weka_url - "LOG_LEVEL" = var.function_app_log_level - "SUBNET" = data.azurerm_subnet.subnet.address_prefix - FUNCTION_APP_NAME = local.function_app_name - PROXY_URL = var.proxy_url - WEKA_HOME_URL = var.weka_home_url - POST_CLUSTER_CREATION_SCRIPT = var.script_post_cluster_creation - PRE_START_IO_SCRIPT = var.script_pre_start_io - DOWN_BACKENDS_REMOVAL_TIMEOUT = var.debug_down_backends_removal_timeout - - https_only = true - FUNCTIONS_EXTENSION_VERSION = "~4" - FUNCTIONS_WORKER_RUNTIME = "custom" - FUNCTION_APP_EDIT_MODE = "readonly" - HASH = var.function_app_version - WEBSITE_RUN_FROM_PACKAGE = "https://${local.weka_sa}.blob.core.windows.net/${local.weka_sa_container}/${local.function_app_zip_name}" - WEBSITE_VNET_ROUTE_ALL = true - - NFS_STATE_CONTAINER_NAME = local.nfs_deployment_container_name - NFS_STATE_BLOB_NAME = "nfs_state" - NFS_INTERFACE_GROUP_NAME = var.nfs_interface_group_name - NFS_CLIENT_GROUP_NAME = var.nfs_client_group_name - NFS_SECONDARY_IPS_NUM = var.nfs_protocol_gateway_secondary_ips_per_nic - NFS_PROTOCOL_GATEWAY_FE_CORES_NUM = var.nfs_protocol_gateway_fe_cores_num - NFS_PROTOCOL_GATEWAYS_NUM = var.nfs_protocol_gateways_number - NFS_VMSS_NAME = var.nfs_protocol_gateways_number > 0 ? "${var.prefix}-${var.cluster_name}-nfs-protocol-gateway-vmss" : "" - NFS_DISK_SIZE = var.nfs_protocol_gateway_disk_size - SMB_DISK_SIZE = var.smb_protocol_gateway_disk_size - S3_DISK_SIZE = var.s3_protocol_gateway_disk_size - SMB_PROTOCOL_GATEWAY_FE_CORES_NUM = var.smb_protocol_gateway_fe_cores_num - S3_PROTOCOL_GATEWAY_FE_CORES_NUM = var.s3_protocol_gateway_fe_cores_num - TRACES_PER_FRONTEND = var.traces_per_ionode - - BACKEND_LB_IP = var.create_lb ? azurerm_lb.backend_lb[0].private_ip_address : "" - } + app_settings = local.app_settings identity { type = "UserAssigned" @@ -228,7 +318,17 @@ resource "azurerm_linux_function_app" "function_app" { condition = contains(local.supported_regions, data.azurerm_resource_group.rg.location) error_message = "The region '${data.azurerm_resource_group.rg.location}' is not supported for the function_app_dist '${var.function_app_dist}'. Supported regions: ${join(", ", local.supported_regions)}" } + + precondition { + condition = local.sa_public_access_enabled || local.sa_public_access_for_vnet && local.sa_allowed_ips_provided || (local.sa_public_access_disabled || local.sa_public_access_for_vnet && !local.sa_allowed_ips_provided) && var.deployment_storage_account_name != "" + error_message = "You shoud pick one of 3 options: 1. Public access enabled, 2. Public access enabled for VNET + public IPs whitelisted, 3. Public access disabled (or enabled for VNET without IPs whitelisted) and deployment_storage_account_name provided" + } + + precondition { + condition = local.read_remote_function_zip || var.deployment_function_app_code_blob != "" + error_message = "You should provide value for 'deployment_function_app_code_blob' or 'read_function_zip_from_storage_account' should be false" + } } - depends_on = [module.network, azurerm_storage_account.deployment_sa, azurerm_subnet.logicapp_subnet_delegation] + depends_on = [module.network, module.iam, azurerm_storage_account.deployment_sa, azurerm_private_endpoint.file_endpoint, azurerm_private_endpoint.blob_endpoint] } diff --git a/key-vault.tf b/key-vault.tf index b6840799..79c1f6f3 100644 --- a/key-vault.tf +++ b/key-vault.tf @@ -6,7 +6,7 @@ resource "azurerm_key_vault" "key_vault" { resource_group_name = var.rg_name enabled_for_deployment = true tenant_id = data.azurerm_client_config.current.tenant_id - purge_protection_enabled = false + purge_protection_enabled = var.key_vault_purge_protection_enabled sku_name = "standard" tags = merge(var.tags_map, { "weka_cluster" : var.cluster_name }) lifecycle { @@ -14,13 +14,13 @@ resource "azurerm_key_vault" "key_vault" { } } -resource "azurerm_key_vault_access_policy" "function_app_get_secret_permission" { +resource "azurerm_key_vault_access_policy" "function_app_secret_permissions" { key_vault_id = azurerm_key_vault.key_vault.id tenant_id = data.azurerm_client_config.current.tenant_id object_id = local.function_app_identity_principal secret_permissions = [ - "Get", + "Get", "Set" ] depends_on = [azurerm_key_vault.key_vault] @@ -96,23 +96,25 @@ resource "azurerm_key_vault_secret" "get_weka_io_token" { } } -resource "random_password" "weka_password" { - length = 16 - lower = true - min_lower = 1 - upper = true - min_upper = 1 - numeric = true - min_numeric = 1 -} - resource "azurerm_key_vault_secret" "weka_password_secret" { name = "weka-password" - value = random_password.weka_password.result + value = "" + key_vault_id = azurerm_key_vault.key_vault.id + tags = merge(var.tags_map, { "weka_cluster" : var.cluster_name }) + lifecycle { + ignore_changes = [value, tags] + } + depends_on = [azurerm_key_vault.key_vault, azurerm_key_vault_access_policy.key_vault_access_policy] +} + + +resource "azurerm_key_vault_secret" "weka_deployment_password" { + name = "weka-deployment-password" + value = "" key_vault_id = azurerm_key_vault.key_vault.id tags = merge(var.tags_map, { "weka_cluster" : var.cluster_name }) lifecycle { ignore_changes = [value, tags] } - depends_on = [azurerm_key_vault.key_vault, random_password.weka_password, azurerm_key_vault_access_policy.key_vault_access_policy] + depends_on = [azurerm_key_vault.key_vault, azurerm_key_vault_access_policy.key_vault_access_policy] } diff --git a/lb.tf b/lb.tf index 8209381c..98ecdca9 100644 --- a/lb.tf +++ b/lb.tf @@ -150,7 +150,7 @@ resource "azurerm_lb_rule" "backend_lb_rule" { } resource "azurerm_private_dns_a_record" "dns_a_record_backend_lb" { - count = var.create_lb ? 1 : 0 + count = var.create_lb && var.private_dns_zone_use ? 1 : 0 name = lower("${var.cluster_name}-backend") zone_name = local.private_dns_zone_name resource_group_name = local.private_dns_rg_name diff --git a/main.tf b/main.tf index 95682ab5..a10507ec 100644 --- a/main.tf +++ b/main.tf @@ -7,23 +7,9 @@ locals { disk_size = var.default_disk_size + var.traces_per_ionode * (var.containers_config_map[var.instance_type].compute + var.containers_config_map[var.instance_type].drive + var.containers_config_map[var.instance_type].frontend) alphanumeric_cluster_name = lower(replace(var.cluster_name, "/\\W|_|\\s/", "")) alphanumeric_prefix_name = lower(replace(var.prefix, "/\\W|_|\\s/", "")) - subnet_range = data.azurerm_subnet.subnet.address_prefix + subnet_range = data.azurerm_subnet.subnet.address_prefixes[0] nics_numbers = var.install_cluster_dpdk ? var.containers_config_map[var.instance_type].nics : 1 - init_script = templatefile("${path.module}/init-script.sh", { - apt_repo_server = var.apt_repo_server - user = var.vm_username - subnet_range = local.subnet_range - nics_num = local.nics_numbers - deploy_url = "https://${azurerm_linux_function_app.function_app.name}.azurewebsites.net/api/deploy" - report_url = "https://${azurerm_linux_function_app.function_app.name}.azurewebsites.net/api/report" - function_app_default_key = data.azurerm_function_app_host_keys.function_keys.default_function_key - disk_size = local.disk_size - }) - custom_data_script = templatefile("${path.module}/user-data.sh", { - user_data = var.user_data - init_script = local.init_script - }) - placement_group_id = var.placement_group_id != "" ? var.placement_group_id : var.vmss_single_placement_group ? azurerm_proximity_placement_group.ppg[0].id : null + placement_group_id = var.placement_group_id != "" ? var.placement_group_id : var.vmss_single_placement_group ? azurerm_proximity_placement_group.ppg[0].id : null } # ===================== SSH key ++++++++++++++++++++++++= # diff --git a/modules/clients/README.md b/modules/clients/README.md index f750557f..90f3defe 100644 --- a/modules/clients/README.md +++ b/modules/clients/README.md @@ -4,13 +4,13 @@ | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.4.6 | -| [azurerm](#requirement\_azurerm) | ~>3.75.0 | +| [azurerm](#requirement\_azurerm) | ~>3.114.0 | ## Providers | Name | Version | |------|---------| -| [azurerm](#provider\_azurerm) | ~>3.75.0 | +| [azurerm](#provider\_azurerm) | ~>3.114.0 | ## Modules @@ -21,6 +21,7 @@ No modules. | Name | Type | |------|------| | [azurerm_linux_virtual_machine.this](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/linux_virtual_machine) | resource | +| [azurerm_linux_virtual_machine_scale_set.this](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/linux_virtual_machine_scale_set) | resource | | [azurerm_network_interface.private_first_nic](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_interface) | resource | | [azurerm_network_interface.private_nics](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_interface) | resource | | [azurerm_network_interface.public_first_nic](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_interface) | resource | @@ -56,6 +57,7 @@ No modules. | [ssh\_public\_key](#input\_ssh\_public\_key) | Ssh public key to pass to vms. | `string` | n/a | yes | | [subnet\_name](#input\_subnet\_name) | The subnet names. | `string` | n/a | yes | | [tags\_map](#input\_tags\_map) | A map of tags to assign the same metadata to all resources in the environment. Format: key:value. | `map(string)` | `{}` | no | +| [use\_vmss](#input\_use\_vmss) | Use VMSS | `bool` | `false` | no | | [vm\_identity\_name](#input\_vm\_identity\_name) | The name of the user assigned identity for the client VMs. | `string` | `""` | no | | [vm\_username](#input\_vm\_username) | The user name for logging in to the virtual machines. | `string` | `"weka"` | no | | [vmss\_name](#input\_vmss\_name) | The name of the backends virtual machine scale set. | `string` | n/a | yes | @@ -68,4 +70,5 @@ No modules. |------|-------------| | [client\_ips](#output\_client\_ips) | n/a | | [client\_names](#output\_client\_names) | n/a | +| [vmss\_name](#output\_vmss\_name) | n/a | diff --git a/modules/clients/main.tf b/modules/clients/main.tf index 4d7367b4..01f8cd6d 100644 --- a/modules/clients/main.tf +++ b/modules/clients/main.tf @@ -9,16 +9,17 @@ data "azurerm_subnet" "subnet" { } locals { - first_nic_ids = var.assign_public_ip ? azurerm_network_interface.public_first_nic[*].id : azurerm_network_interface.private_first_nic[*].id - nics_num = var.frontend_container_cores_num + 1 + first_nic_ids = var.assign_public_ip ? azurerm_network_interface.public_first_nic[*].id : azurerm_network_interface.private_first_nic[*].id + private_nic_first_index = var.assign_public_ip ? 1 : 0 + nics_num = var.frontend_container_cores_num + 1 preparation_script = templatefile("${path.module}/init.sh", { apt_repo_server = var.apt_repo_server nics_num = local.nics_num - subnet_range = data.azurerm_subnet.subnet.address_prefix + subnet_range = data.azurerm_subnet.subnet.address_prefixes[0] }) mount_wekafs_script = templatefile("${path.module}/mount_wekafs.sh", { - all_gateways = cidrhost(data.azurerm_subnet.subnet.address_prefix, 1) + all_gateways = cidrhost(data.azurerm_subnet.subnet.address_prefixes[0], 1) frontend_container_cores_num = var.frontend_container_cores_num backend_lb_ip = var.backend_lb_ip clients_use_dpdk = var.clients_use_dpdk @@ -47,7 +48,7 @@ locals { } resource "azurerm_public_ip" "public_ip" { - count = var.assign_public_ip ? var.clients_number : 0 + count = var.use_vmss ? 0 : var.assign_public_ip ? var.clients_number : 0 name = "${var.clients_name}-public-ip-${count.index}" resource_group_name = var.rg_name location = data.azurerm_resource_group.rg.location @@ -57,12 +58,12 @@ resource "azurerm_public_ip" "public_ip" { } resource "azurerm_network_interface" "public_first_nic" { - count = var.assign_public_ip ? var.clients_number : 0 - name = "${var.clients_name}-backend-nic-${count.index}" - enable_accelerated_networking = var.clients_use_dpdk - resource_group_name = var.rg_name - location = data.azurerm_resource_group.rg.location - tags = var.tags_map + count = var.use_vmss ? 0 : var.assign_public_ip ? var.clients_number : 0 + name = "${var.clients_name}-nic-${count.index}" + accelerated_networking_enabled = var.clients_use_dpdk + resource_group_name = var.rg_name + location = data.azurerm_resource_group.rg.location + tags = var.tags_map ip_configuration { name = "ipconfig0" subnet_id = data.azurerm_subnet.subnet.id @@ -73,18 +74,18 @@ resource "azurerm_network_interface" "public_first_nic" { } resource "azurerm_network_interface_security_group_association" "public_first" { - count = var.assign_public_ip ? var.clients_number : 0 + count = var.use_vmss ? 0 : var.assign_public_ip ? var.clients_number : 0 network_interface_id = azurerm_network_interface.public_first_nic[count.index].id network_security_group_id = var.sg_id } resource "azurerm_network_interface" "private_first_nic" { - count = var.assign_public_ip ? 0 : var.clients_number - name = "${var.clients_name}-backend-nic-${count.index}" - enable_accelerated_networking = var.clients_use_dpdk - resource_group_name = var.rg_name - location = data.azurerm_resource_group.rg.location - tags = var.tags_map + count = var.use_vmss ? 0 : var.assign_public_ip ? 0 : var.clients_number + name = "${var.clients_name}-nic-${count.index}" + accelerated_networking_enabled = var.clients_use_dpdk + resource_group_name = var.rg_name + location = data.azurerm_resource_group.rg.location + tags = var.tags_map ip_configuration { name = "ipconfig0" subnet_id = data.azurerm_subnet.subnet.id @@ -94,18 +95,18 @@ resource "azurerm_network_interface" "private_first_nic" { } resource "azurerm_network_interface_security_group_association" "private_first" { - count = var.assign_public_ip ? 0 : var.clients_number + count = var.use_vmss ? 0 : var.assign_public_ip ? 0 : var.clients_number network_interface_id = azurerm_network_interface.private_first_nic[count.index].id network_security_group_id = var.sg_id } resource "azurerm_network_interface" "private_nics" { - count = (local.nics_num - 1) * var.clients_number - name = "${var.clients_name}-backend-nic-${count.index + var.clients_number}" - enable_accelerated_networking = var.clients_use_dpdk - resource_group_name = var.rg_name - location = data.azurerm_resource_group.rg.location - tags = var.tags_map + count = var.use_vmss ? 0 : (local.nics_num - 1) * var.clients_number + name = "${var.clients_name}-nic-${count.index + var.clients_number}" + accelerated_networking_enabled = var.clients_use_dpdk + resource_group_name = var.rg_name + location = data.azurerm_resource_group.rg.location + tags = var.tags_map ip_configuration { name = "ipconfig${count.index + var.clients_number}" subnet_id = data.azurerm_subnet.subnet.id @@ -114,13 +115,13 @@ resource "azurerm_network_interface" "private_nics" { } resource "azurerm_network_interface_security_group_association" "private" { - count = (local.nics_num - 1) * var.clients_number + count = var.use_vmss ? 0 : (local.nics_num - 1) * var.clients_number network_interface_id = azurerm_network_interface.private_nics[count.index].id network_security_group_id = var.sg_id } resource "azurerm_linux_virtual_machine" "this" { - count = var.clients_number + count = var.use_vmss ? 0 : var.clients_number name = "${var.clients_name}-vm-${count.index}" location = data.azurerm_resource_group.rg.location resource_group_name = var.rg_name @@ -156,3 +157,85 @@ resource "azurerm_linux_virtual_machine" "this" { } depends_on = [azurerm_network_interface.private_first_nic, azurerm_network_interface.private_nics, azurerm_network_interface.public_first_nic] } + +resource "azurerm_linux_virtual_machine_scale_set" "this" { + count = var.use_vmss ? 1 : 0 + instances = var.clients_number + name = var.clients_name + location = data.azurerm_resource_group.rg.location + resource_group_name = var.rg_name + admin_username = var.vm_username + tags = merge({ "weka_cluster_client" : var.clients_name }, var.tags_map) + custom_data = local.vms_custom_data + source_image_id = local.source_image_id + sku = local.instance_type + + dynamic "network_interface" { + for_each = range(local.private_nic_first_index) + content { + name = "${var.clients_name}-clients-nic-0" + network_security_group_id = var.sg_id + primary = true + enable_accelerated_networking = var.clients_use_dpdk + ip_configuration { + primary = true + name = "ipconfig0" + subnet_id = data.azurerm_subnet.subnet.id + public_ip_address { + name = "${var.clients_name}-public-ip" + domain_name_label = var.clients_name + } + } + } + } + dynamic "network_interface" { + for_each = range(local.private_nic_first_index, 1) + content { + name = "${var.clients_name}-nic-0" + network_security_group_id = var.sg_id + primary = true + enable_accelerated_networking = var.clients_use_dpdk + ip_configuration { + primary = true + name = "ipconfig0" + subnet_id = data.azurerm_subnet.subnet.id + } + } + } + dynamic "network_interface" { + for_each = range(1, local.nics_num) + content { + name = "${var.clients_name}-nic-${network_interface.value}" + network_security_group_id = var.sg_id + primary = false + enable_accelerated_networking = var.clients_use_dpdk + ip_configuration { + primary = false + name = "ipconfig${network_interface.value}" + subnet_id = data.azurerm_subnet.subnet.id + } + } + } + + proximity_placement_group_id = var.ppg_id + disable_password_authentication = true + + identity { + type = "UserAssigned" + identity_ids = [local.client_identity_id] + } + + os_disk { + caching = "ReadWrite" + storage_account_type = "StandardSSD_LRS" + } + + admin_ssh_key { + public_key = var.ssh_public_key + username = var.vm_username + } + lifecycle { + ignore_changes = [tags, custom_data] + } + depends_on = [azurerm_network_interface.private_first_nic, azurerm_network_interface.private_nics, azurerm_network_interface.public_first_nic] +} diff --git a/modules/clients/outputs.tf b/modules/clients/outputs.tf index d25a94e7..dbbc1c3e 100644 --- a/modules/clients/outputs.tf +++ b/modules/clients/outputs.tf @@ -2,6 +2,10 @@ output "client_ips" { value = var.assign_public_ip ? azurerm_linux_virtual_machine.this[*].public_ip_address : azurerm_linux_virtual_machine.this[*].private_ip_address } +output "vmss_name" { + value = var.clients_name +} + output "client_names" { value = azurerm_linux_virtual_machine.this[*].name } diff --git a/modules/clients/variables.tf b/modules/clients/variables.tf index 7b6a0536..cccab788 100644 --- a/modules/clients/variables.tf +++ b/modules/clients/variables.tf @@ -119,3 +119,9 @@ variable "source_image_id" { description = "Use weka custom image, ubuntu 20.04 with kernel 5.4 and ofed 5.8-1.1.2.1" default = "" } + +variable "use_vmss" { + type = bool + description = "Use VMSS" + default = false +} diff --git a/modules/clients/versions.tf b/modules/clients/versions.tf index 281dc878..202d92cc 100644 --- a/modules/clients/versions.tf +++ b/modules/clients/versions.tf @@ -3,7 +3,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "~>3.75.0" + version = "~>3.114.0" } } } diff --git a/modules/iam/README.md b/modules/iam/README.md index 812bde25..032623c2 100644 --- a/modules/iam/README.md +++ b/modules/iam/README.md @@ -4,13 +4,13 @@ | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.4.6 | -| [azurerm](#requirement\_azurerm) | ~>3.75.0 | +| [azurerm](#requirement\_azurerm) | ~>3.114.0 | ## Providers | Name | Version | |------|---------| -| [azurerm](#provider\_azurerm) | ~>3.75.0 | +| [azurerm](#provider\_azurerm) | ~>3.114.0 | ## Modules @@ -23,6 +23,9 @@ No modules. | [azurerm_role_assignment.function_app_key_vault_secrets_user](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | | [azurerm_role_assignment.function_app_reader](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | | [azurerm_role_assignment.function_app_scale_set_machine_owner](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.join_sg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.join_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.key_vault_set_secret](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | | [azurerm_role_assignment.logic_app_standard_reader](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | | [azurerm_role_assignment.logic_app_standard_reader_secret](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | | [azurerm_role_assignment.logic_app_standard_reader_smb_data](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | @@ -31,15 +34,22 @@ No modules. | [azurerm_role_assignment.nfs_storage_blob_data_contributor](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | | [azurerm_role_assignment.obs_data_contributor](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | | [azurerm_role_assignment.obs_storage_blob_data_contributor](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.private_endpoint](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | | [azurerm_role_assignment.reader](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | | [azurerm_role_assignment.storage_account_contributor](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | | [azurerm_role_assignment.storage_blob_data_contributor](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | | [azurerm_role_assignment.weka_tar_data_reader](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | +| [azurerm_role_definition.join_sg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_definition) | resource | +| [azurerm_role_definition.join_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_definition) | resource | +| [azurerm_role_definition.key_vault_set_secret](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_definition) | resource | +| [azurerm_role_definition.private_endpoint](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_definition) | resource | | [azurerm_user_assigned_identity.function_app](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/user_assigned_identity) | resource | | [azurerm_user_assigned_identity.logic_app](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/user_assigned_identity) | resource | | [azurerm_user_assigned_identity.vmss](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/user_assigned_identity) | resource | | [azurerm_resource_group.rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | +| [azurerm_resource_group.vnet_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | | [azurerm_storage_account.obs_sa](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/storage_account) | data source | +| [azurerm_subnet.subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subnet) | data source | | [azurerm_user_assigned_identity.function_app](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/user_assigned_identity) | data source | | [azurerm_user_assigned_identity.logic_app](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/user_assigned_identity) | data source | | [azurerm_user_assigned_identity.vmss](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/user_assigned_identity) | data source | @@ -58,11 +68,17 @@ No modules. | [nfs\_deployment\_container\_name](#input\_nfs\_deployment\_container\_name) | The name of the container for the NFS deployment. | `string` | n/a | yes | | [nfs\_protocol\_gateways\_number](#input\_nfs\_protocol\_gateways\_number) | The number of NFS protocol gateways. | `number` | n/a | yes | | [obs\_container\_name](#input\_obs\_container\_name) | The name of the container for the OBS. | `string` | n/a | yes | +| [obs\_create\_private\_endpoint](#input\_obs\_create\_private\_endpoint) | Create private endpoint for OBS. | `bool` | `false` | no | | [prefix](#input\_prefix) | Prefix for all resources | `string` | n/a | yes | | [rg\_name](#input\_rg\_name) | A predefined resource group in the Azure subscription. | `string` | n/a | yes | +| [sg\_id](#input\_sg\_id) | Security group id | `string` | `""` | no | +| [subnet\_name](#input\_subnet\_name) | The name of the subnet. | `string` | n/a | yes | +| [support\_logic\_app](#input\_support\_logic\_app) | Enable support for logic app. | `bool` | `true` | no | | [tiering\_enable\_obs\_integration](#input\_tiering\_enable\_obs\_integration) | Enable OBS integration for tiering. | `bool` | n/a | yes | | [tiering\_obs\_name](#input\_tiering\_obs\_name) | Name of existing obs storage account. | `string` | `""` | no | | [vmss\_identity\_name](#input\_vmss\_identity\_name) | The user assigned identity name for the vmss instances (if empty - new one is created). | `string` | `""` | no | +| [vnet\_name](#input\_vnet\_name) | The name of the virtual network. | `string` | n/a | yes | +| [vnet\_rg\_name](#input\_vnet\_rg\_name) | The name of the resource group for the VNet. | `string` | n/a | yes | | [weka\_tar\_storage\_account\_id](#input\_weka\_tar\_storage\_account\_id) | n/a | `string` | `""` | no | ## Outputs diff --git a/modules/iam/function_app.tf b/modules/iam/function_app.tf index 95abacd2..c17de988 100644 --- a/modules/iam/function_app.tf +++ b/modules/iam/function_app.tf @@ -42,11 +42,35 @@ resource "azurerm_role_assignment" "obs_storage_blob_data_contributor" { resource "azurerm_role_assignment" "function_app_key_vault_secrets_user" { count = var.function_app_identity_name == "" ? 1 : 0 - scope = data.azurerm_resource_group.rg.id + scope = var.key_vault_id role_definition_name = "Key Vault Secrets User" principal_id = azurerm_user_assigned_identity.function_app[0].principal_id } +resource "azurerm_role_definition" "key_vault_set_secret" { + count = var.function_app_identity_name == "" ? 1 : 0 + name = "${var.prefix}-${var.cluster_name}-key-vault-new-secret-writer" + scope = var.key_vault_id + description = "Can create new secrets in the key vault" + + permissions { + actions = [ + # See: https://learn.microsoft.com/en-us/azure/role-based-access-control/permissions/security#microsoftkeyvault + "Microsoft.KeyVault/vaults/secrets/write", + ] + not_actions = [] + } + + assignable_scopes = [var.key_vault_id] +} + +resource "azurerm_role_assignment" "key_vault_set_secret" { + count = var.function_app_identity_name == "" ? 1 : 0 + scope = var.key_vault_id + role_definition_id = azurerm_role_definition.key_vault_set_secret[0].role_definition_resource_id + principal_id = azurerm_user_assigned_identity.function_app[0].principal_id +} + resource "azurerm_role_assignment" "function_app_reader" { count = var.function_app_identity_name == "" ? 1 : 0 scope = data.azurerm_resource_group.rg.id @@ -67,3 +91,81 @@ resource "azurerm_role_assignment" "managed_identity_operator" { role_definition_name = "Managed Identity Operator" principal_id = azurerm_user_assigned_identity.function_app[0].principal_id } + +data "azurerm_subnet" "subnet" { + name = var.subnet_name + virtual_network_name = var.vnet_name + resource_group_name = var.vnet_rg_name +} + +resource "azurerm_role_definition" "join_subnet" { + count = var.function_app_identity_name == "" && var.rg_name != var.vnet_rg_name ? 1 : 0 + name = "${var.prefix}-${var.cluster_name}-join-subnet" + scope = data.azurerm_subnet.subnet.id + description = "Can join subnet" + + permissions { + actions = [ + "Microsoft.Network/virtualNetworks/subnets/join/action", # for VMSS creation from function app + "Microsoft.Network/virtualNetworks/subnets/joinViaServiceEndpoint/action", # for using SA service endpoint (when network is in different RG) - for weka obs + ] + not_actions = [] + } + + assignable_scopes = [data.azurerm_subnet.subnet.id] +} + +resource "azurerm_role_assignment" "join_subnet" { + count = var.function_app_identity_name == "" && var.rg_name != var.vnet_rg_name ? 1 : 0 + scope = data.azurerm_subnet.subnet.id + role_definition_id = azurerm_role_definition.join_subnet[0].role_definition_resource_id + principal_id = azurerm_user_assigned_identity.function_app[0].principal_id +} + +resource "azurerm_role_definition" "join_sg" { + count = var.function_app_identity_name == "" && var.rg_name != var.vnet_rg_name && var.sg_id != "" ? 1 : 0 + name = "${var.prefix}-${var.cluster_name}-join-sg" + scope = var.sg_id + description = "Can join security group" + + permissions { + actions = [ + "Microsoft.Network/networkSecurityGroups/join/action", + ] + not_actions = [] + } + + assignable_scopes = [var.sg_id] +} + +resource "azurerm_role_assignment" "join_sg" { + count = var.function_app_identity_name == "" && var.rg_name != var.vnet_rg_name && var.sg_id != "" ? 1 : 0 + scope = var.sg_id + role_definition_id = azurerm_role_definition.join_sg[0].role_definition_resource_id + principal_id = azurerm_user_assigned_identity.function_app[0].principal_id +} + + +resource "azurerm_role_definition" "private_endpoint" { + count = var.function_app_identity_name == "" && var.obs_create_private_endpoint ? 1 : 0 + name = "${var.prefix}-${var.cluster_name}-private-endpoint" + scope = data.azurerm_resource_group.rg.id + description = "Can create private endpoint" + + permissions { + actions = [ + "Microsoft.Network/privateEndpoints/write", + "Microsoft.Network/privateEndpoints/privateDnsZoneGroups/write", + "Microsoft.Network/privateDnsZones/join/action", + ] + not_actions = [] + } + assignable_scopes = [data.azurerm_resource_group.rg.id] +} + +resource "azurerm_role_assignment" "private_endpoint" { + count = var.function_app_identity_name == "" && var.obs_create_private_endpoint ? 1 : 0 + scope = data.azurerm_resource_group.rg.id + role_definition_id = azurerm_role_definition.private_endpoint[0].role_definition_resource_id + principal_id = azurerm_user_assigned_identity.function_app[0].principal_id +} diff --git a/modules/iam/logic_app.tf b/modules/iam/logic_app.tf index 46986aa5..226617aa 100644 --- a/modules/iam/logic_app.tf +++ b/modules/iam/logic_app.tf @@ -1,32 +1,32 @@ data "azurerm_user_assigned_identity" "logic_app" { - count = var.logic_app_identity_name != "" ? 1 : 0 + count = var.logic_app_identity_name != "" && var.support_logic_app ? 1 : 0 name = var.logic_app_identity_name resource_group_name = data.azurerm_resource_group.rg.name } resource "azurerm_user_assigned_identity" "logic_app" { - count = var.logic_app_identity_name == "" ? 1 : 0 + count = var.logic_app_identity_name == "" && var.support_logic_app ? 1 : 0 location = data.azurerm_resource_group.rg.location name = "${var.prefix}-${var.cluster_name}-logic-app-identity" resource_group_name = data.azurerm_resource_group.rg.name } resource "azurerm_role_assignment" "logic_app_standard_reader" { - count = var.logic_app_identity_name == "" ? 1 : 0 + count = var.logic_app_identity_name == "" && var.support_logic_app ? 1 : 0 scope = data.azurerm_resource_group.rg.id role_definition_name = "Reader" principal_id = azurerm_user_assigned_identity.logic_app[0].principal_id } resource "azurerm_role_assignment" "logic_app_standard_reader_secret" { - count = var.logic_app_identity_name == "" ? 1 : 0 + count = var.logic_app_identity_name == "" && var.support_logic_app ? 1 : 0 scope = var.key_vault_id role_definition_name = "Key Vault Secrets User" principal_id = azurerm_user_assigned_identity.logic_app[0].principal_id } resource "azurerm_role_assignment" "logic_app_standard_reader_smb_data" { - count = var.logic_app_identity_name == "" ? 1 : 0 + count = var.logic_app_identity_name == "" && var.support_logic_app ? 1 : 0 scope = var.logic_app_storage_account_id role_definition_name = "Storage File Data SMB Share Contributor" principal_id = azurerm_user_assigned_identity.logic_app[0].principal_id diff --git a/modules/iam/main.tf b/modules/iam/main.tf index 58a4215f..dee59959 100644 --- a/modules/iam/main.tf +++ b/modules/iam/main.tf @@ -2,6 +2,10 @@ data "azurerm_resource_group" "rg" { name = var.rg_name } +data "azurerm_resource_group" "vnet_rg" { + name = var.vnet_rg_name +} + locals { obs_scope = var.tiering_obs_name != "" ? "${data.azurerm_storage_account.obs_sa[0].id}/blobServices/default/containers/${var.obs_container_name}" : "" deployment_storage_account_scope = "${var.deployment_storage_account_id}/blobServices/default/containers/${var.deployment_container_name}" diff --git a/modules/iam/outputs.tf b/modules/iam/outputs.tf index a5ea994a..fd3075db 100644 --- a/modules/iam/outputs.tf +++ b/modules/iam/outputs.tf @@ -1,10 +1,10 @@ output "logic_app_identity_id" { - value = var.logic_app_identity_name == "" ? azurerm_user_assigned_identity.logic_app[0].id : data.azurerm_user_assigned_identity.logic_app[0].id + value = var.support_logic_app ? var.logic_app_identity_name == "" ? azurerm_user_assigned_identity.logic_app[0].id : data.azurerm_user_assigned_identity.logic_app[0].id : null description = "The ID of the managed identity for the logic app" } output "logic_app_identity_principal_id" { - value = var.logic_app_identity_name == "" ? azurerm_user_assigned_identity.logic_app[0].principal_id : data.azurerm_user_assigned_identity.logic_app[0].principal_id + value = var.support_logic_app ? var.logic_app_identity_name == "" ? azurerm_user_assigned_identity.logic_app[0].principal_id : data.azurerm_user_assigned_identity.logic_app[0].principal_id : null description = "The principal ID of the managed identity for the logic app" } diff --git a/modules/iam/variables.tf b/modules/iam/variables.tf index 3c750c3d..8fa713a5 100644 --- a/modules/iam/variables.tf +++ b/modules/iam/variables.tf @@ -3,6 +3,27 @@ variable "rg_name" { description = "A predefined resource group in the Azure subscription." } +variable "vnet_rg_name" { + type = string + description = "The name of the resource group for the VNet." +} + +variable "vnet_name" { + type = string + description = "The name of the virtual network." +} + +variable "subnet_name" { + type = string + description = "The name of the subnet." +} + +variable "sg_id" { + type = string + description = "Security group id" + default = "" +} + variable "prefix" { type = string description = "Prefix for all resources" @@ -33,6 +54,12 @@ variable "function_app_identity_name" { default = "" } +variable "support_logic_app" { + type = bool + description = "Enable support for logic app." + default = true +} + variable "logic_app_identity_name" { type = string description = "The user assigned identity name for the logic app (if empty - new one is created)." @@ -80,6 +107,12 @@ variable "obs_container_name" { description = "The name of the container for the OBS." } +variable "obs_create_private_endpoint" { + type = bool + description = "Create private endpoint for OBS." + default = false +} + variable "nfs_protocol_gateways_number" { type = number description = "The number of NFS protocol gateways." diff --git a/modules/iam/versions.tf b/modules/iam/versions.tf index 281dc878..202d92cc 100644 --- a/modules/iam/versions.tf +++ b/modules/iam/versions.tf @@ -3,7 +3,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "~>3.75.0" + version = "~>3.114.0" } } } diff --git a/modules/iam/vmss.tf b/modules/iam/vmss.tf index b8ca29a6..6983fafb 100644 --- a/modules/iam/vmss.tf +++ b/modules/iam/vmss.tf @@ -20,7 +20,7 @@ resource "azurerm_role_assignment" "reader" { resource "azurerm_role_assignment" "network_contributor" { count = var.vmss_identity_name == "" ? 1 : 0 - scope = data.azurerm_resource_group.rg.id + scope = data.azurerm_resource_group.vnet_rg.id role_definition_name = "Network Contributor" principal_id = azurerm_user_assigned_identity.vmss[0].principal_id } diff --git a/modules/logic_app/README.md b/modules/logic_app/README.md new file mode 100644 index 00000000..f29acb3b --- /dev/null +++ b/modules/logic_app/README.md @@ -0,0 +1,68 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.6 | +| [azurerm](#requirement\_azurerm) | ~>3.114.0 | +| [local](#requirement\_local) | ~>2.4.0 | +| [null](#requirement\_null) | ~>3.2.0 | + +## Providers + +| Name | Version | +|------|---------| +| [azurerm](#provider\_azurerm) | ~>3.114.0 | +| [local](#provider\_local) | ~>2.4.0 | +| [null](#provider\_null) | ~>3.2.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [azurerm_key_vault_access_policy.standard_logic_app_get_secret_permission](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | +| [azurerm_logic_app_standard.logic_app_standard](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/logic_app_standard) | resource | +| [azurerm_service_plan.logicapp_service_plan](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/service_plan) | resource | +| [azurerm_storage_share.storage_share](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_share) | resource | +| [azurerm_storage_share_directory.share_directory_scale_down](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_share_directory) | resource | +| [azurerm_storage_share_directory.share_directory_scale_up](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_share_directory) | resource | +| [azurerm_storage_share_file.connections_share_file](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_share_file) | resource | +| [azurerm_storage_share_file.scale_down_share_file](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_share_file) | resource | +| [azurerm_storage_share_file.scale_up_share_file](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_share_file) | resource | +| [local_file.connections_workflow_file](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.scale_down_workflow_file](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.scale_up_workflow_file](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [null_resource.wait_for_logic_app](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source | +| [azurerm_storage_account.logicapp](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/storage_account) | data source | +| [azurerm_storage_share.storage_share](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/storage_share) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cluster\_name](#input\_cluster\_name) | Cluster name | `string` | n/a | yes | +| [function\_app\_id](#input\_function\_app\_id) | The ID of the function app. | `string` | n/a | yes | +| [function\_app\_key](#input\_function\_app\_key) | The key of the function app. | `string` | n/a | yes | +| [function\_app\_name](#input\_function\_app\_name) | The name of the function app. | `string` | n/a | yes | +| [key\_vault\_id](#input\_key\_vault\_id) | The id of the Azure Key Vault. | `string` | n/a | yes | +| [key\_vault\_uri](#input\_key\_vault\_uri) | The URI of the Azure Key Vault. | `string` | n/a | yes | +| [location](#input\_location) | The Azure region to deploy all resources to. | `string` | n/a | yes | +| [logic\_app\_identity\_id](#input\_logic\_app\_identity\_id) | The ID of the managed identity for the logic app | `string` | n/a | yes | +| [logic\_app\_identity\_principal](#input\_logic\_app\_identity\_principal) | The principal ID of the managed identity for the logic app | `string` | n/a | yes | +| [logic\_app\_subnet\_delegation\_id](#input\_logic\_app\_subnet\_delegation\_id) | Required to specify if subnet\_name were used to specify pre-defined subnets for weka. Logicapp subnet delegation requires an additional subnet, and in the case of pre-defined networking this one also should be pre-defined | `string` | n/a | yes | +| [prefix](#input\_prefix) | Prefix for all resources | `string` | n/a | yes | +| [restricted\_inbound\_access](#input\_restricted\_inbound\_access) | Restrict inbound access to internal VNet | `bool` | n/a | yes | +| [rg\_name](#input\_rg\_name) | A predefined resource group in the Azure subscription. | `string` | n/a | yes | +| [storage\_account\_name](#input\_storage\_account\_name) | The name of the storage account. | `string` | n/a | yes | +| [subnet\_id](#input\_subnet\_id) | The ID of the cluster subnet. | `string` | n/a | yes | +| [use\_secured\_storage\_account](#input\_use\_secured\_storage\_account) | Use secured storage account with logic app. | `bool` | `false` | no | + +## Outputs + +No outputs. + diff --git a/logic_app/connections.json b/modules/logic_app/connections.json similarity index 100% rename from logic_app/connections.json rename to modules/logic_app/connections.json diff --git a/logic_app.tf b/modules/logic_app/main.tf similarity index 54% rename from logic_app.tf rename to modules/logic_app/main.tf index 80ab431a..667c47af 100644 --- a/logic_app.tf +++ b/modules/logic_app/main.tf @@ -1,30 +1,13 @@ -resource "azurerm_storage_account" "logicapp" { - name = substr("${local.alphanumeric_prefix_name}${local.alphanumeric_cluster_name}logicappsa", 0, 24) - resource_group_name = var.rg_name - location = local.location - account_tier = "Standard" - account_replication_type = "LRS" -} - -resource "azurerm_subnet" "logicapp_subnet_delegation" { - count = var.logic_app_subnet_delegation_id == "" ? 1 : 0 - name = "${var.prefix}-${var.cluster_name}-logicapp-delegation" - resource_group_name = local.vnet_rg_name - virtual_network_name = local.vnet_name - address_prefixes = [var.logic_app_subnet_delegation_cidr] - service_endpoints = ["Microsoft.KeyVault", "Microsoft.Web"] - delegation { - name = "logic-delegation" - service_delegation { - name = "Microsoft.Web/serverFarms" - actions = ["Microsoft.Network/virtualNetworks/subnets/action"] - } - } +data "azurerm_client_config" "current" {} + +data "azurerm_storage_account" "logicapp" { + name = var.storage_account_name + resource_group_name = var.rg_name } resource "azurerm_service_plan" "logicapp_service_plan" { name = "${var.prefix}-${var.cluster_name}-logic-app-service-plan" - location = local.location + location = var.location resource_group_name = var.rg_name os_type = "Windows" sku_name = "WS1" @@ -33,25 +16,44 @@ resource "azurerm_service_plan" "logicapp_service_plan" { } } +resource "azurerm_storage_share" "storage_share" { + count = var.use_secured_storage_account ? 1 : 0 + name = "${var.prefix}-${var.cluster_name}-logic-app-content" + storage_account_name = data.azurerm_storage_account.logicapp.name + quota = 100 +} + +data "azurerm_storage_share" "storage_share" { + count = var.use_secured_storage_account ? 0 : 1 + name = "${azurerm_logic_app_standard.logic_app_standard.name}-content" + storage_account_name = data.azurerm_storage_account.logicapp.name +} + +locals { + storage_share_id = var.use_secured_storage_account ? azurerm_storage_share.storage_share[0].id : data.azurerm_storage_share.storage_share[0].id +} + resource "azurerm_logic_app_standard" "logic_app_standard" { name = "${var.prefix}-${var.cluster_name}-logic-app" - location = local.location + location = var.location resource_group_name = var.rg_name app_service_plan_id = azurerm_service_plan.logicapp_service_plan.id - storage_account_name = azurerm_storage_account.logicapp.name - storage_account_access_key = azurerm_storage_account.logicapp.primary_access_key + storage_account_share_name = var.use_secured_storage_account ? azurerm_storage_share.storage_share[0].name : null + storage_account_name = data.azurerm_storage_account.logicapp.name + storage_account_access_key = data.azurerm_storage_account.logicapp.primary_access_key version = "~4" # sets FUNCTIONS_EXTENSION_VERSION (should be same as for function app) identity { type = "UserAssigned" - identity_ids = [local.logic_app_identity_id] + identity_ids = [var.logic_app_identity_id] } site_config { - vnet_route_all_enabled = true + public_network_access_enabled = false + vnet_route_all_enabled = true dynamic "ip_restriction" { - for_each = range(local.create_private_function) + for_each = range(var.restricted_inbound_access ? 1 : 0) content { - virtual_network_subnet_id = data.azurerm_subnet.subnet.id + virtual_network_subnet_id = var.subnet_id action = "Allow" priority = 300 name = "VirtualNetwork" @@ -59,57 +61,65 @@ resource "azurerm_logic_app_standard" "logic_app_standard" { } } app_settings = { + "WEBSITE_CONTENTOVERVNET" = var.use_secured_storage_account ? 1 : 0 "FUNCTIONS_WORKER_RUNTIME" = "node" "WEBSITE_NODE_DEFAULT_VERSION" = "~18" - "function_app_key" = data.azurerm_function_app_host_keys.function_keys.default_function_key - "keyVaultUri" = azurerm_key_vault.key_vault.vault_uri + "function_app_key" = var.function_app_key + "keyVaultUri" = var.key_vault_uri } - virtual_network_subnet_id = var.logic_app_subnet_delegation_id == "" ? azurerm_subnet.logicapp_subnet_delegation[0].id : var.logic_app_subnet_delegation_id - depends_on = [azurerm_service_plan.logicapp_service_plan, azurerm_subnet.logicapp_subnet_delegation, azurerm_storage_account.logicapp] + https_only = true + virtual_network_subnet_id = var.logic_app_subnet_delegation_id + + depends_on = [azurerm_service_plan.logicapp_service_plan] } resource "azurerm_key_vault_access_policy" "standard_logic_app_get_secret_permission" { - key_vault_id = azurerm_key_vault.key_vault.id + key_vault_id = var.key_vault_id tenant_id = data.azurerm_client_config.current.tenant_id - object_id = local.logic_app_identity_principal + object_id = var.logic_app_identity_principal secret_permissions = [ "Get", ] - depends_on = [azurerm_key_vault.key_vault] } +resource "null_resource" "wait_for_logic_app" { + count = var.use_secured_storage_account ? 1 : 0 + triggers = { + logic_app_id = azurerm_logic_app_standard.logic_app_standard.id + } + + provisioner "local-exec" { + # wait for "site/wwwroot" to be created in file share + command = "sleep 60" + } -resource "azurerm_storage_share_directory" "share_directory_scale_down" { - name = "site/wwwroot/scale-down" - share_name = "${azurerm_logic_app_standard.logic_app_standard.name}-content" - storage_account_name = azurerm_storage_account.logicapp.name - depends_on = [azurerm_storage_account.logicapp] + depends_on = [azurerm_logic_app_standard.logic_app_standard] } -resource "azurerm_storage_share_directory" "share_directory_scale_up" { - name = "site/wwwroot/scale-up" - share_name = "${azurerm_logic_app_standard.logic_app_standard.name}-content" - storage_account_name = azurerm_storage_account.logicapp.name - depends_on = [azurerm_storage_account.logicapp] +resource "azurerm_storage_share_directory" "share_directory_scale_down" { + name = "site/wwwroot/scale-down" + storage_share_id = local.storage_share_id + depends_on = [null_resource.wait_for_logic_app] } -data "azurerm_storage_share" "storage_share" { - name = "${azurerm_logic_app_standard.logic_app_standard.name}-content" - storage_account_name = azurerm_storage_account.logicapp.name +resource "azurerm_storage_share_directory" "share_directory_scale_up" { + name = "site/wwwroot/scale-up" + storage_share_id = local.storage_share_id + depends_on = [null_resource.wait_for_logic_app] } locals { - connections_workflow_path = "${path.module}/logic_app/connections.json" + connections_workflow_path = "${path.module}/connections.json" connections_workflow = templatefile(local.connections_workflow_path, { - function_name = azurerm_linux_function_app.function_app.name - function_id = azurerm_linux_function_app.function_app.id + function_name = var.function_app_name + function_id = var.function_app_id }) connections_workflow_hash = md5(join("", [for f in fileset(local.connections_workflow, "**") : filemd5("${local.connections_workflow}/${f}")])) connections_workflow_filename = "/tmp/${var.prefix}_${var.cluster_name}_connections_workflow_${local.connections_workflow_hash}" - scale_up_workflow_path = "${path.module}/logic_app/scale_up.json" + scale_up_workflow_path = "${path.module}/scale_up.json" scale_up_workflow_hash = md5(join("", [for f in fileset(local.scale_up_workflow_path, "**") : filemd5("${local.scale_up_workflow_path}/${f}")])) scale_up_workflow_filename = "/tmp/${var.prefix}_${var.cluster_name}_scale_up_workflow_${local.scale_up_workflow_hash}" - scale_down_workflow_path = "${path.module}/logic_app/scale_down.json" + scale_down_workflow_path = "${path.module}/scale_down.json" scale_down_workflow_hash = md5(join("", [for f in fileset(local.scale_down_workflow_path, "**") : filemd5("${local.scale_down_workflow_path}/${f}")])) scale_down_workflow_filename = "/tmp/${var.prefix}_${var.cluster_name}_scale_down_workflow_${local.scale_down_workflow_hash}" } @@ -132,23 +142,23 @@ resource "local_file" "scale_down_workflow_file" { resource "azurerm_storage_share_file" "scale_down_share_file" { name = "workflow.json" path = azurerm_storage_share_directory.share_directory_scale_down.name - storage_share_id = data.azurerm_storage_share.storage_share.id + storage_share_id = local.storage_share_id source = local_file.scale_down_workflow_file.filename - depends_on = [azurerm_storage_share_directory.share_directory_scale_down, data.azurerm_storage_share.storage_share, local_file.scale_down_workflow_file] + depends_on = [azurerm_storage_share_directory.share_directory_scale_down, azurerm_storage_share.storage_share, local_file.scale_down_workflow_file] } resource "azurerm_storage_share_file" "scale_up_share_file" { name = "workflow.json" path = azurerm_storage_share_directory.share_directory_scale_up.name - storage_share_id = data.azurerm_storage_share.storage_share.id + storage_share_id = local.storage_share_id source = local_file.scale_up_workflow_file.filename - depends_on = [azurerm_storage_share_directory.share_directory_scale_up, data.azurerm_storage_share.storage_share, local_file.scale_up_workflow_file] + depends_on = [azurerm_storage_share_directory.share_directory_scale_up, azurerm_storage_share.storage_share, local_file.scale_up_workflow_file] } resource "azurerm_storage_share_file" "connections_share_file" { name = "connections.json" path = "site/wwwroot" - storage_share_id = data.azurerm_storage_share.storage_share.id + storage_share_id = local.storage_share_id source = local_file.connections_workflow_file.filename - depends_on = [azurerm_storage_share_directory.share_directory_scale_down, data.azurerm_storage_share.storage_share, local_file.connections_workflow_file] + depends_on = [azurerm_storage_share_directory.share_directory_scale_down, azurerm_storage_share.storage_share, local_file.connections_workflow_file] } diff --git a/modules/logic_app/outputs.tf b/modules/logic_app/outputs.tf new file mode 100644 index 00000000..e69de29b diff --git a/logic_app/scale_down.json b/modules/logic_app/scale_down.json similarity index 100% rename from logic_app/scale_down.json rename to modules/logic_app/scale_down.json diff --git a/logic_app/scale_up.json b/modules/logic_app/scale_up.json similarity index 100% rename from logic_app/scale_up.json rename to modules/logic_app/scale_up.json diff --git a/modules/logic_app/variables.tf b/modules/logic_app/variables.tf new file mode 100644 index 00000000..76411ea8 --- /dev/null +++ b/modules/logic_app/variables.tf @@ -0,0 +1,81 @@ +variable "prefix" { + type = string + description = "Prefix for all resources" +} + +variable "cluster_name" { + type = string + description = "Cluster name" +} + +variable "rg_name" { + type = string + description = "A predefined resource group in the Azure subscription." +} + +variable "subnet_id" { + type = string + description = "The ID of the cluster subnet." +} + +variable "location" { + type = string + description = "The Azure region to deploy all resources to." +} + +variable "logic_app_subnet_delegation_id" { + type = string + description = "Required to specify if subnet_name were used to specify pre-defined subnets for weka. Logicapp subnet delegation requires an additional subnet, and in the case of pre-defined networking this one also should be pre-defined" +} + +variable "storage_account_name" { + type = string + description = "The name of the storage account." +} + +variable "logic_app_identity_id" { + type = string + description = "The ID of the managed identity for the logic app" +} + +variable "logic_app_identity_principal" { + type = string + description = "The principal ID of the managed identity for the logic app" + +} + +variable "restricted_inbound_access" { + type = bool + description = "Restrict inbound access to internal VNet" +} + +variable "function_app_name" { + type = string + description = "The name of the function app." +} + +variable "function_app_id" { + type = string + description = "The ID of the function app." +} + +variable "function_app_key" { + type = string + description = "The key of the function app." +} + +variable "key_vault_id" { + type = string + description = "The id of the Azure Key Vault." +} + +variable "key_vault_uri" { + type = string + description = "The URI of the Azure Key Vault." +} + +variable "use_secured_storage_account" { + type = bool + description = "Use secured storage account with logic app." + default = false +} diff --git a/modules/logic_app/versions.tf b/modules/logic_app/versions.tf new file mode 100644 index 00000000..8f93fa62 --- /dev/null +++ b/modules/logic_app/versions.tf @@ -0,0 +1,17 @@ +terraform { + required_version = ">= 1.4.6" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~>3.114.0" + } + local = { + source = "hashicorp/local" + version = "~>2.4.0" + } + null = { + source = "hashicorp/null" + version = "~>3.2.0" + } + } +} diff --git a/modules/network/README.md b/modules/network/README.md index e5ddaa26..6caddf3b 100644 --- a/modules/network/README.md +++ b/modules/network/README.md @@ -64,13 +64,13 @@ No modules. | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.4.6 | -| [azurerm](#requirement\_azurerm) | ~>3.75.0 | +| [azurerm](#requirement\_azurerm) | ~>3.114.0 | ## Providers | Name | Version | |------|---------| -| [azurerm](#provider\_azurerm) | ~>3.75.0 | +| [azurerm](#provider\_azurerm) | ~>3.114.0 | ## Modules @@ -108,6 +108,7 @@ No modules. | [prefix](#input\_prefix) | The prefix for all the resource names. For example, the prefix for your system name. | `string` | `"weka"` | no | | [private\_dns\_rg\_name](#input\_private\_dns\_rg\_name) | The private DNS zone resource group name. Required when private\_dns\_zone\_name is set. | `string` | `""` | no | | [private\_dns\_zone\_name](#input\_private\_dns\_zone\_name) | The private DNS zone name. | `string` | `""` | no | +| [private\_dns\_zone\_use](#input\_private\_dns\_zone\_use) | Determines whether to use private DNS zone. Required for LB dns name. | `bool` | `true` | no | | [rg\_name](#input\_rg\_name) | A predefined resource group in the Azure subscription. | `string` | n/a | yes | | [sg\_id](#input\_sg\_id) | The security group id. | `string` | `""` | no | | [subnet\_name](#input\_subnet\_name) | Subnet name, if exist. | `string` | `""` | no | diff --git a/modules/network/main.tf b/modules/network/main.tf index 167009fd..1e5548ea 100644 --- a/modules/network/main.tf +++ b/modules/network/main.tf @@ -142,7 +142,7 @@ resource "azurerm_subnet_network_security_group_association" "sg_association" { # ================== Private DNS ========================= # resource "azurerm_private_dns_zone" "dns" { - count = var.private_dns_zone_name == "" ? 1 : 0 + count = var.private_dns_zone_name == "" && var.private_dns_zone_use ? 1 : 0 name = "${var.prefix}.private.net" resource_group_name = local.private_dns_rg_name tags = merge(var.tags_map) @@ -152,7 +152,7 @@ resource "azurerm_private_dns_zone" "dns" { } resource "azurerm_private_dns_zone_virtual_network_link" "dns_vnet_link" { - count = var.private_dns_zone_name == "" ? 1 : 0 + count = var.private_dns_zone_name == "" && var.private_dns_zone_use ? 1 : 0 name = "${var.prefix}-private-network-link" resource_group_name = data.azurerm_resource_group.rg.name private_dns_zone_name = azurerm_private_dns_zone.dns[0].name diff --git a/modules/network/outputs.tf b/modules/network/outputs.tf index 3263ba16..3792121f 100644 --- a/modules/network/outputs.tf +++ b/modules/network/outputs.tf @@ -14,7 +14,7 @@ output "sg_id" { } output "private_dns_zone_name" { - value = var.private_dns_zone_name == "" ? azurerm_private_dns_zone.dns[0].name : var.private_dns_zone_name + value = var.private_dns_zone_use ? var.private_dns_zone_name == "" ? azurerm_private_dns_zone.dns[0].name : var.private_dns_zone_name : null description = "Displays the private DNS zone name." } diff --git a/modules/network/variables.tf b/modules/network/variables.tf index 70d39ae0..63b1034f 100644 --- a/modules/network/variables.tf +++ b/modules/network/variables.tf @@ -69,6 +69,12 @@ variable "private_dns_zone_name" { default = "" } +variable "private_dns_zone_use" { + type = bool + description = "Determines whether to use private DNS zone. Required for LB dns name." + default = true +} + variable "sg_id" { type = string description = "The security group id." diff --git a/modules/network/versions.tf b/modules/network/versions.tf index 281dc878..202d92cc 100644 --- a/modules/network/versions.tf +++ b/modules/network/versions.tf @@ -3,7 +3,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "~>3.75.0" + version = "~>3.114.0" } } } diff --git a/modules/peering_vnets/README.md b/modules/peering_vnets/README.md index b9aaec7f..46a4d696 100644 --- a/modules/peering_vnets/README.md +++ b/modules/peering_vnets/README.md @@ -42,13 +42,13 @@ No outputs. | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.4.6 | -| [azurerm](#requirement\_azurerm) | ~>3.75.0 | +| [azurerm](#requirement\_azurerm) | ~>3.114.0 | ## Providers | Name | Version | |------|---------| -| [azurerm](#provider\_azurerm) | ~>3.75.0 | +| [azurerm](#provider\_azurerm) | ~>3.114.0 | ## Modules diff --git a/modules/peering_vnets/versions.tf b/modules/peering_vnets/versions.tf index 281dc878..202d92cc 100644 --- a/modules/peering_vnets/versions.tf +++ b/modules/peering_vnets/versions.tf @@ -3,7 +3,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "~>3.75.0" + version = "~>3.114.0" } } } diff --git a/modules/protocol_gateways/README.md b/modules/protocol_gateways/README.md index 2da41a71..6ff1bc04 100644 --- a/modules/protocol_gateways/README.md +++ b/modules/protocol_gateways/README.md @@ -4,13 +4,13 @@ | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.4.6 | -| [azurerm](#requirement\_azurerm) | ~>3.75.0 | +| [azurerm](#requirement\_azurerm) | ~>3.114.0 | ## Providers | Name | Version | |------|---------| -| [azurerm](#provider\_azurerm) | ~>3.75.0 | +| [azurerm](#provider\_azurerm) | ~>3.114.0 | ## Modules diff --git a/modules/protocol_gateways/main.tf b/modules/protocol_gateways/main.tf index ba3bc402..faa4c4d5 100644 --- a/modules/protocol_gateways/main.tf +++ b/modules/protocol_gateways/main.tf @@ -19,11 +19,11 @@ resource "azurerm_public_ip" "this" { } resource "azurerm_network_interface" "primary_gateway_nic_public" { - count = var.assign_public_ip && var.protocol != "NFS" ? var.gateways_number : 0 - name = "${var.gateways_name}-primary-nic-${count.index}" - location = var.location - resource_group_name = var.rg_name - enable_accelerated_networking = true + count = var.assign_public_ip && var.protocol != "NFS" ? var.gateways_number : 0 + name = "${var.gateways_name}-primary-nic-${count.index}" + location = var.location + resource_group_name = var.rg_name + accelerated_networking_enabled = true ip_configuration { primary = true @@ -51,11 +51,11 @@ resource "azurerm_network_interface_security_group_association" "primary_gateway } resource "azurerm_network_interface" "primary_gateway_nic_private" { - count = var.assign_public_ip || var.protocol == "NFS" ? 0 : var.gateways_number - name = "${var.gateways_name}-primary-nic-${count.index}" - location = var.location - resource_group_name = var.rg_name - enable_accelerated_networking = true + count = var.assign_public_ip || var.protocol == "NFS" ? 0 : var.gateways_number + name = "${var.gateways_name}-primary-nic-${count.index}" + location = var.location + resource_group_name = var.rg_name + accelerated_networking_enabled = true ip_configuration { primary = true @@ -86,11 +86,11 @@ locals { } resource "azurerm_network_interface" "secondary_gateway_nic" { - count = var.protocol != "NFS" ? local.secondary_nics_num : 0 - name = "${var.gateways_name}-secondary-nic-${count.index + var.gateways_number}" - location = var.location - resource_group_name = var.rg_name - enable_accelerated_networking = true + count = var.protocol != "NFS" ? local.secondary_nics_num : 0 + name = "${var.gateways_name}-secondary-nic-${count.index + var.gateways_number}" + location = var.location + resource_group_name = var.rg_name + accelerated_networking_enabled = true ip_configuration { primary = true @@ -115,7 +115,7 @@ locals { init_script = templatefile("${path.module}/init.sh", { apt_repo_server = var.apt_repo_server nics_num = local.nics_numbers - subnet_range = data.azurerm_subnet.subnet.address_prefix + subnet_range = data.azurerm_subnet.subnet.address_prefixes[0] disk_size = local.disk_size deploy_url = var.deploy_function_url report_url = var.report_function_url diff --git a/modules/protocol_gateways/versions.tf b/modules/protocol_gateways/versions.tf index 281dc878..202d92cc 100644 --- a/modules/protocol_gateways/versions.tf +++ b/modules/protocol_gateways/versions.tf @@ -3,7 +3,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "~>3.75.0" + version = "~>3.114.0" } } } diff --git a/modules/subnet_delegation/README.md b/modules/subnet_delegation/README.md new file mode 100644 index 00000000..92b58038 --- /dev/null +++ b/modules/subnet_delegation/README.md @@ -0,0 +1,43 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.6 | +| [azurerm](#requirement\_azurerm) | ~>3.114.0 | + +## Providers + +| Name | Version | +|------|---------| +| [azurerm](#provider\_azurerm) | ~>3.114.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [azurerm_subnet.this](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cidr\_range](#input\_cidr\_range) | The address space that is used by the subnet. | `string` | n/a | yes | +| [cluster\_name](#input\_cluster\_name) | Cluster name | `string` | n/a | yes | +| [delegation\_name](#input\_delegation\_name) | The name of the subnet delegation. | `string` | `"subnet-delegation"` | no | +| [prefix](#input\_prefix) | Prefix for all resources | `string` | n/a | yes | +| [rg\_name](#input\_rg\_name) | A predefined resource group in the Azure subscription. | `string` | n/a | yes | +| [service\_endpoints](#input\_service\_endpoints) | The list of service endpoints. | `list(string)` | 
[
  "Microsoft.Storage",
  "Microsoft.KeyVault",
  "Microsoft.Web"
]
| no | +| [vnet\_name](#input\_vnet\_name) | The name of the virtual network. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [id](#output\_id) | n/a | +| [name](#output\_name) | n/a | + diff --git a/modules/subnet_delegation/main.tf b/modules/subnet_delegation/main.tf new file mode 100644 index 00000000..8018c542 --- /dev/null +++ b/modules/subnet_delegation/main.tf @@ -0,0 +1,14 @@ +resource "azurerm_subnet" "this" { + name = "${var.prefix}-${var.cluster_name}-${var.delegation_name}" + resource_group_name = var.rg_name + virtual_network_name = var.vnet_name + address_prefixes = [var.cidr_range] + service_endpoints = var.service_endpoints + delegation { + name = var.delegation_name + service_delegation { + name = "Microsoft.Web/serverFarms" + actions = ["Microsoft.Network/virtualNetworks/subnets/action"] + } + } +} diff --git a/modules/subnet_delegation/outputs.tf b/modules/subnet_delegation/outputs.tf new file mode 100644 index 00000000..a4468d1b --- /dev/null +++ b/modules/subnet_delegation/outputs.tf @@ -0,0 +1,7 @@ +output "name" { + value = azurerm_subnet.this.name +} + +output "id" { + value = azurerm_subnet.this.id +} diff --git a/modules/subnet_delegation/variables.tf b/modules/subnet_delegation/variables.tf new file mode 100644 index 00000000..4e0cbc34 --- /dev/null +++ b/modules/subnet_delegation/variables.tf @@ -0,0 +1,36 @@ +variable "prefix" { + type = string + description = "Prefix for all resources" +} + +variable "cluster_name" { + type = string + description = "Cluster name" +} + +variable "rg_name" { + type = string + description = "A predefined resource group in the Azure subscription." +} + +variable "vnet_name" { + type = string + description = "The name of the virtual network." +} + +variable "cidr_range" { + type = string + description = "The address space that is used by the subnet." +} + +variable "delegation_name" { + type = string + description = "The name of the subnet delegation." + default = "subnet-delegation" +} + +variable "service_endpoints" { + type = list(string) + description = "The list of service endpoints." + default = ["Microsoft.Storage", "Microsoft.KeyVault", "Microsoft.Web"] +} diff --git a/modules/subnet_delegation/versions.tf b/modules/subnet_delegation/versions.tf new file mode 100644 index 00000000..202d92cc --- /dev/null +++ b/modules/subnet_delegation/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.4.6" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~>3.114.0" + } + } +} diff --git a/outputs.tf b/outputs.tf index deb022f5..ee767d48 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,7 +1,9 @@ locals { vmss_name = "${var.prefix}-${var.cluster_name}-vmss" + clients_vmss_name = var.clients_number > 0 && var.clients_use_vmss ? module.clients[0].vmss_name : "" key_vault_name = azurerm_key_vault.key_vault.name vm_ips = local.assign_public_ip ? "az vmss list-instance-public-ips -g ${var.rg_name} --name ${local.vmss_name} --subscription ${var.subscription_id} --query \"[].ipAddress\" \n" : "az vmss nic list -g ${var.rg_name} --vmss-name ${local.vmss_name} --subscription ${var.subscription_id} --query \"[].ipConfigurations[]\" | jq -r '.[] | select(.name==\"ipconfig0\")'.privateIPAddress \n" + clients_vmss_ips = local.assign_public_ip ? "az vmss list-instance-public-ips -g ${var.rg_name} --name ${local.clients_vmss_name} --subscription ${var.subscription_id} --query \"[].ipAddress\" \n" : "az vmss nic list -g ${var.rg_name} --vmss-name ${local.clients_vmss_name} --subscription ${var.subscription_id} --query \"[].ipConfigurations[]\" | jq -r '.[] | select(.name==\"ipconfig0\")'.privateIPAddress \n" ssh_keys_commands = "########################################## Download ssh keys command from blob ###########################################################\n az keyvault secret download --file private.pem --encoding utf-8 --vault-name ${local.key_vault_name} --name private-key --query \"value\" \n az keyvault secret download --file public.pub --encoding utf-8 --vault-name ${local.key_vault_name} --name public-key --query \"value\"\n" blob_commands = var.ssh_public_key == null ? local.ssh_keys_commands : "" private_ssh_key_path = var.ssh_public_key == null ? local.ssh_private_key_path : null @@ -36,7 +38,6 @@ output "vmss_name" { value = local.vmss_name } - output "function_app_name" { value = local.function_app_name description = "Function app name" @@ -57,8 +58,21 @@ output "backend_ips" { description = "If 'assign_public_ip' is set to true, it will output the public ips, If no it will output the private ips" } +output "clients_vmss_name" { + value = local.clients_vmss_name != "" ? local.clients_vmss_name : null +} + output "client_ips" { - value = var.clients_number > 0 ? module.clients[0].client_ips : null + value = ( + var.clients_number > 0 && !var.clients_use_vmss ? module.clients[0].client_ips : + var.clients_number > 0 && var.clients_use_vmss ? local.clients_vmss_ips : + null + ) + description = "If 'private_network' is set to false, it will output clients public ips, otherwise private ips." +} + +output "client_vmss_ips" { + value = var.clients_number > 0 && var.clients_use_vmss ? local.clients_vmss_ips : null description = "If 'private_network' is set to false, it will output clients public ips, otherwise private ips." } @@ -112,6 +126,25 @@ output "ppg_id" { description = "Placement proximity group id" } +output "weka_cluster_admin_password_secret_name" { + value = azurerm_key_vault_secret.weka_password_secret.name + description = "Weka cluster admin password secret name" +} + + +locals { + resize_helper_command = !local.create_logic_app ? "" : < --resource-group ${var.rg_name} --force-deletion true --subscription ${var.subscription_id} EOT description = "Useful commands and script to interact with weka cluster" diff --git a/prerequisites.tf b/prerequisites.tf index e460ac1e..e72d2777 100644 --- a/prerequisites.tf +++ b/prerequisites.tf @@ -15,6 +15,7 @@ module "network" { allow_ssh_cidrs = var.allow_ssh_cidrs allow_weka_api_cidrs = var.allow_weka_api_cidrs private_dns_zone_name = var.private_dns_zone_name + private_dns_zone_use = var.private_dns_zone_use sg_id = var.sg_id create_nat_gateway = var.create_nat_gateway } @@ -24,10 +25,15 @@ module "iam" { rg_name = var.rg_name prefix = var.prefix cluster_name = var.cluster_name + vnet_rg_name = local.vnet_rg_name + vnet_name = local.vnet_name + subnet_name = local.subnet_name + sg_id = var.sg_id vmss_identity_name = var.vmss_identity_name function_app_identity_name = var.function_app_identity_name + support_logic_app = local.create_logic_app logic_app_identity_name = var.logic_app_identity_name - logic_app_storage_account_id = azurerm_storage_account.logicapp.id + logic_app_storage_account_id = local.create_logic_app ? azurerm_storage_account.logicapp[0].id : "" key_vault_id = azurerm_key_vault.key_vault.id weka_tar_storage_account_id = var.weka_tar_storage_account_id deployment_storage_account_id = local.deployment_storage_account_id @@ -37,9 +43,12 @@ module "iam" { tiering_enable_obs_integration = var.tiering_enable_obs_integration tiering_obs_name = var.tiering_obs_name obs_container_name = local.obs_container_name + obs_create_private_endpoint = var.create_storage_account_private_links && local.sa_public_access_disabled + depends_on = [module.network] } locals { + create_logic_app = local.create_sa_resources vnet_name = var.vnet_name == "" ? module.network.vnet_name : var.vnet_name vnet_rg_name = var.vnet_rg_name == "" ? module.network.vnet_rg_name : var.vnet_rg_name subnet_name = var.subnet_name == "" ? module.network.subnet_name : var.subnet_name @@ -65,6 +74,55 @@ module "peering" { depends_on = [module.network] } +module "logic_app_subnet_delegation" { + count = var.logic_app_subnet_delegation_id == "" && local.create_logic_app ? 1 : 0 + source = "./modules/subnet_delegation" + rg_name = local.vnet_rg_name + vnet_name = local.vnet_name + prefix = var.prefix + cluster_name = var.cluster_name + cidr_range = var.logic_app_subnet_delegation_cidr + delegation_name = "logic-app-delegation" + + depends_on = [module.network] +} + +module "function_app_subnet_delegation" { + count = var.function_app_subnet_delegation_id == "" ? 1 : 0 + source = "./modules/subnet_delegation" + rg_name = local.vnet_rg_name + vnet_name = local.vnet_name + prefix = var.prefix + cluster_name = var.cluster_name + cidr_range = var.function_app_subnet_delegation_cidr + delegation_name = "function-app-delegation" + + depends_on = [module.network] +} + +module "logicapp" { + count = local.create_logic_app ? 1 : 0 + source = "./modules/logic_app" + rg_name = var.rg_name + location = local.location + prefix = var.prefix + cluster_name = var.cluster_name + logic_app_subnet_delegation_id = var.logic_app_subnet_delegation_id == "" ? module.logic_app_subnet_delegation[0].id : var.logic_app_subnet_delegation_id + storage_account_name = azurerm_storage_account.logicapp[0].name + logic_app_identity_id = local.logic_app_identity_id + logic_app_identity_principal = local.logic_app_identity_principal + restricted_inbound_access = var.function_access_restriction_enabled + subnet_id = data.azurerm_subnet.subnet.id + function_app_id = azurerm_linux_function_app.function_app.id + function_app_name = azurerm_linux_function_app.function_app.name + function_app_key = data.azurerm_function_app_host_keys.function_keys.default_function_key + key_vault_id = azurerm_key_vault.key_vault.id + key_vault_uri = azurerm_key_vault.key_vault.vault_uri + use_secured_storage_account = local.sa_public_access_enabled ? false : true + + depends_on = [azurerm_storage_account.logicapp, module.logic_app_subnet_delegation, module.iam] +} + data "azurerm_subnet" "subnet" { resource_group_name = local.vnet_rg_name virtual_network_name = local.vnet_name diff --git a/user-data.sh b/user-data.sh deleted file mode 100644 index c2aa50c6..00000000 --- a/user-data.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -ex - -${user_data} - -${init_script} diff --git a/utils/apt-repo/main.tf b/utils/apt-repo/main.tf index 9efce0fb..54de1e31 100644 --- a/utils/apt-repo/main.tf +++ b/utils/apt-repo/main.tf @@ -105,10 +105,10 @@ resource "azurerm_virtual_machine" "apt_repo_vm_linux" { } resource "azurerm_network_interface" "vm_interface" { - name = "${var.prefix}-vm-nic" - location = azurerm_resource_group.rg.location - resource_group_name = azurerm_resource_group.rg.name - enable_accelerated_networking = false + name = "${var.prefix}-vm-nic" + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + accelerated_networking_enabled = false ip_configuration { name = "ipconfig" diff --git a/variables.tf b/variables.tf index 9c1b2093..1e4737ed 100644 --- a/variables.tf +++ b/variables.tf @@ -195,6 +195,12 @@ variable "private_dns_rg_name" { default = "" } +variable "private_dns_zone_use" { + type = bool + description = "Determines whether to use private DNS zone. Required for LB record creation." + default = true +} + variable "vnets_to_peer_to_deployment_vnet" { type = list(object({ vnet = string @@ -387,13 +393,13 @@ variable "function_app_storage_account_container_prefix" { variable "function_app_version" { type = string description = "Function app code version (hash)" - default = "5464597f9be93b3c954324b1811ace7a" + default = "70129b9f8d813e6f87aeed9be4764327" } variable "function_app_dist" { type = string description = "Function app code dist" - default = "release" + default = "dev" validation { condition = contains(["dev", "release"], var.function_app_dist) @@ -552,6 +558,12 @@ variable "clients_custom_data" { default = "" } +variable "clients_use_vmss" { + type = bool + default = false + description = "Use VMSS for clients" +} + variable "placement_group_id" { type = string default = "" @@ -576,10 +588,15 @@ variable "deployment_container_name" { description = "Name of exising deployment container" } -variable "deployment_storage_account_access_key" { +variable "deployment_file_share_name" { type = string - description = "The access key of the existing Blob object store container." - sensitive = true + default = "" + description = "Name of exising deployment file share. Will use '-share' name if not provided." +} + +variable "deployment_function_app_code_blob" { + type = string + description = "The path to the function app code blob file." default = "" } @@ -643,12 +660,6 @@ variable "nfs_setup_protocol" { default = false } -variable "nfs_client_group_name" { - type = string - description = "Client access group name." - default = "weka-cg" -} - variable "nfs_interface_group_name" { type = string description = "Interface group name." @@ -828,3 +839,44 @@ variable "debug_down_backends_removal_timeout" { default = "3h" description = "Don't change this value without consulting weka support team. Timeout for removing down backends. Valid time units are ns, us (or µs), ms, s, m, h." } + +variable "storage_account_public_network_access" { + type = string + description = "Public network access to the storage accounts." + default = "Enabled" + + validation { + condition = contains(["Enabled", "Disabled", "EnabledForVnet"], var.storage_account_public_network_access) + error_message = "Allowed values: [\"Enabled\", \"Disabled\", \"EnabledForVnet\"]." + } +} + +variable "storage_account_allowed_ips" { + type = list(string) + description = "IP ranges to allow access from the internet or your on-premises networks to storage accounts." + default = [] +} + +variable "create_storage_account_private_links" { + type = bool + default = false + description = "Create private links for storage accounts (needed in case if public network access for the storage account is disabled)." +} + +variable "storage_blob_private_dns_zone_name" { + type = string + description = "The private DNS zone name for the storage account (blob)." + default = "privatelink.blob.core.windows.net" +} + +variable "read_function_zip_from_storage_account" { + type = bool + default = false + description = "Read function app zip from storage account (is read from public distribution storage account by default)." +} + +variable "key_vault_purge_protection_enabled" { + type = bool + default = false + description = "Enable purge protection for the key vault." +} diff --git a/versions.tf b/versions.tf index 4e25ecb8..e5dc6e28 100644 --- a/versions.tf +++ b/versions.tf @@ -3,11 +3,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "~>3.75.0" - } - random = { - source = "hashicorp/random" - version = "~>3.5.1" + version = "~>3.114.0" } tls = { source = "hashicorp/tls" diff --git a/zip_function_app_creation/write_function_hash_to_variables.sh b/zip_function_app_creation/write_function_hash_to_variables.sh index 52fbb0e4..efd9e911 100755 --- a/zip_function_app_creation/write_function_hash_to_variables.sh +++ b/zip_function_app_creation/write_function_hash_to_variables.sh @@ -12,6 +12,8 @@ old_function_app_zip_version=$(awk '/Function app code version/{getline;print $N echo "Replacing '$old_function_app_zip_version' function_app_version to '$new_function_app_zip_version'" if [ $os_name == "darwin" ]; then sed -i '' "s/$old_function_app_zip_version/$new_function_app_zip_version/" variables.tf + sed -i '' "s/$old_function_app_zip_version/$new_function_app_zip_version/" README.md else sed -i "s/$old_function_app_zip_version/$new_function_app_zip_version/" variables.tf + sed -i "s/$old_function_app_zip_version/$new_function_app_zip_version/" README.md fi