From c8781fdc40b4bbd3e3f22ee05676da3785cf79f6 Mon Sep 17 00:00:00 2001 From: hezijie Date: Thu, 11 Jan 2024 16:26:20 +0800 Subject: [PATCH 1/6] add new cluster_name_random_suffix variable so the users could decide whether they'd like to add a random suffix a cluster's name. add create_before_destroy to var.nodes so the users could decide whether they'd like to have a random suffix in node pools' names. --- README.md | 7 +- extra_node_pool.tf | 168 +++++++++++++++++++++++++++++++++++++++++++-- locals.tf | 3 + main.tf | 13 +++- variables.tf | 13 +++- 5 files changed, 194 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 1a4fd3bf..ae682f2d 100644 --- a/README.md +++ b/README.md @@ -235,7 +235,8 @@ No modules. | [azapi_update_resource.aks_cluster_http_proxy_config_no_proxy](https://registry.terraform.io/providers/Azure/azapi/latest/docs/resources/update_resource) | resource | | [azapi_update_resource.aks_cluster_post_create](https://registry.terraform.io/providers/Azure/azapi/latest/docs/resources/update_resource) | resource | | [azurerm_kubernetes_cluster.main](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster) | resource | -| [azurerm_kubernetes_cluster_node_pool.node_pool](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster_node_pool) | resource | +| [azurerm_kubernetes_cluster_node_pool.node_pool_create_after_destroy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster_node_pool) | resource | +| [azurerm_kubernetes_cluster_node_pool.node_pool_create_before_destroy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster_node_pool) | resource | | [azurerm_log_analytics_solution.main](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/log_analytics_solution) | resource | | [azurerm_log_analytics_workspace.main](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/log_analytics_workspace) | resource | | [azurerm_role_assignment.acr](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | @@ -246,6 +247,7 @@ No modules. | [azurerm_role_assignment.network_contributor](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | | [azurerm_role_assignment.network_contributor_on_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | | [null_resource.http_proxy_config_no_proxy_keeper](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.kubernetes_cluster_name_keeper](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [null_resource.kubernetes_version_keeper](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [null_resource.pool_name_keeper](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [tls_private_key.ssh](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource | @@ -307,6 +309,7 @@ No modules. | [client\_secret](#input\_client\_secret) | (Optional) The Client Secret (password) for the Service Principal used for the AKS deployment | `string` | `""` | no | | [cluster\_log\_analytics\_workspace\_name](#input\_cluster\_log\_analytics\_workspace\_name) | (Optional) The name of the Analytics workspace | `string` | `null` | no | | [cluster\_name](#input\_cluster\_name) | (Optional) The name for the AKS resources created in the specified Azure Resource Group. This variable overwrites the 'prefix' var (The 'prefix' var will still be applied to the dns\_prefix if it is set) | `string` | `null` | no | +| [cluster\_name\_random\_suffix](#input\_cluster\_name\_random\_suffix) | Whether to add a random suffix on Aks cluster's name or not. `azurerm_kubernetes_cluster` resource defined in this module is `create_before_destroy = true` implicity now(described [here](https://github.com/Azure/terraform-azurerm-aks/issues/389)), without this random suffix we'll not be able to recreate this cluster directly due to the naming conflict. | `bool` | `false` | no | | [confidential\_computing](#input\_confidential\_computing) | (Optional) Enable Confidential Computing. | 
object({
    sgx_quote_helper_enabled = bool
  })
| `null` | no | | [create\_role\_assignment\_network\_contributor](#input\_create\_role\_assignment\_network\_contributor) | (Deprecated) Create a role assignment for the AKS Service Principal to be a Network Contributor on the subnets used for the AKS Cluster | `bool` | `false` | no | | [create\_role\_assignments\_for\_application\_gateway](#input\_create\_role\_assignments\_for\_application\_gateway) | (Optional) Whether to create the corresponding role assignments for application gateway or not. Defaults to `true`. | `bool` | `true` | no | @@ -358,7 +361,7 @@ No modules. | [network\_plugin\_mode](#input\_network\_plugin\_mode) | (Optional) Specifies the network plugin mode used for building the Kubernetes network. Possible value is `Overlay`. Changing this forces a new resource to be created. | `string` | `null` | no | | [network\_policy](#input\_network\_policy) | (Optional) Sets up network policy to be used with Azure CNI. Network policy allows us to control the traffic flow between pods. Currently supported values are calico and azure. Changing this forces a new resource to be created. | `string` | `null` | no | | [node\_os\_channel\_upgrade](#input\_node\_os\_channel\_upgrade) | (Optional) The upgrade channel for this Kubernetes Cluster Nodes' OS Image. Possible values are `Unmanaged`, `SecurityPatch`, `NodeImage` and `None`. | `string` | `null` | no | -| [node\_pools](#input\_node\_pools) | A map of node pools that need to be created and attached on the Kubernetes cluster. The key of the map can be the name of the node pool, and the key must be static string. The value of the map is a `node_pool` block as defined below:
map(object({
name = (Required) The name of the Node Pool which should be created within the Kubernetes Cluster. Changing this forces a new resource to be created. A Windows Node Pool cannot have a `name` longer than 6 characters. A random suffix of 4 characters is always added to the name to avoid clashes during recreates.
node\_count = (Optional) The initial number of nodes which should exist within this Node Pool. Valid values are between `0` and `1000` (inclusive) for user pools and between `1` and `1000` (inclusive) for system pools and must be a value in the range `min_count` - `max_count`.
tags = (Optional) A mapping of tags to assign to the resource. At this time there's a bug in the AKS API where Tags for a Node Pool are not stored in the correct case - you [may wish to use Terraform's `ignore_changes` functionality to ignore changes to the casing](https://www.terraform.io/language/meta-arguments/lifecycle#ignore_changess) until this is fixed in the AKS API.
vm\_size = (Required) The SKU which should be used for the Virtual Machines used in this Node Pool. Changing this forces a new resource to be created.
host\_group\_id = (Optional) The fully qualified resource ID of the Dedicated Host Group to provision virtual machines from. Changing this forces a new resource to be created.
capacity\_reservation\_group\_id = (Optional) Specifies the ID of the Capacity Reservation Group where this Node Pool should exist. Changing this forces a new resource to be created.
custom\_ca\_trust\_enabled = (Optional) Specifies whether to trust a Custom CA. This requires that the Preview Feature `Microsoft.ContainerService/CustomCATrustPreview` is enabled and the Resource Provider is re-registered, see [the documentation](https://learn.microsoft.com/en-us/azure/aks/custom-certificate-authority) for more information.
enable\_auto\_scaling = (Optional) Whether to enable [auto-scaler](https://docs.microsoft.com/azure/aks/cluster-autoscaler).
enable\_host\_encryption = (Optional) Should the nodes in this Node Pool have host encryption enabled? Changing this forces a new resource to be created.
enable\_node\_public\_ip = (Optional) Should each node have a Public IP Address? Changing this forces a new resource to be created.
eviction\_policy = (Optional) The Eviction Policy which should be used for Virtual Machines within the Virtual Machine Scale Set powering this Node Pool. Possible values are `Deallocate` and `Delete`. Changing this forces a new resource to be created. An Eviction Policy can only be configured when `priority` is set to `Spot` and will default to `Delete` unless otherwise specified.
kubelet\_config = optional(object({
cpu\_manager\_policy = (Optional) Specifies the CPU Manager policy to use. Possible values are `none` and `static`, Changing this forces a new resource to be created.
cpu\_cfs\_quota\_enabled = (Optional) Is CPU CFS quota enforcement for containers enabled? Changing this forces a new resource to be created.
cpu\_cfs\_quota\_period = (Optional) Specifies the CPU CFS quota period value. Changing this forces a new resource to be created.
image\_gc\_high\_threshold = (Optional) Specifies the percent of disk usage above which image garbage collection is always run. Must be between `0` and `100`. Changing this forces a new resource to be created.
image\_gc\_low\_threshold = (Optional) Specifies the percent of disk usage lower than which image garbage collection is never run. Must be between `0` and `100`. Changing this forces a new resource to be created.
topology\_manager\_policy = (Optional) Specifies the Topology Manager policy to use. Possible values are `none`, `best-effort`, `restricted` or `single-numa-node`. Changing this forces a new resource to be created.
allowed\_unsafe\_sysctls = (Optional) Specifies the allow list of unsafe sysctls command or patterns (ending in `*`). Changing this forces a new resource to be created.
container\_log\_max\_size\_mb = (Optional) Specifies the maximum size (e.g. 10MB) of container log file before it is rotated. Changing this forces a new resource to be created.
container\_log\_max\_files = (Optional) Specifies the maximum number of container log files that can be present for a container. must be at least 2. Changing this forces a new resource to be created.
pod\_max\_pid = (Optional) Specifies the maximum number of processes per pod. Changing this forces a new resource to be created.
}))
linux\_os\_config = optional(object({
sysctl\_config = optional(object({
fs\_aio\_max\_nr = (Optional) The sysctl setting fs.aio-max-nr. Must be between `65536` and `6553500`. Changing this forces a new resource to be created.
fs\_file\_max = (Optional) The sysctl setting fs.file-max. Must be between `8192` and `12000500`. Changing this forces a new resource to be created.
fs\_inotify\_max\_user\_watches = (Optional) The sysctl setting fs.inotify.max\_user\_watches. Must be between `781250` and `2097152`. Changing this forces a new resource to be created.
fs\_nr\_open = (Optional) The sysctl setting fs.nr\_open. Must be between `8192` and `20000500`. Changing this forces a new resource to be created.
kernel\_threads\_max = (Optional) The sysctl setting kernel.threads-max. Must be between `20` and `513785`. Changing this forces a new resource to be created.
net\_core\_netdev\_max\_backlog = (Optional) The sysctl setting net.core.netdev\_max\_backlog. Must be between `1000` and `3240000`. Changing this forces a new resource to be created.
net\_core\_optmem\_max = (Optional) The sysctl setting net.core.optmem\_max. Must be between `20480` and `4194304`. Changing this forces a new resource to be created.
net\_core\_rmem\_default = (Optional) The sysctl setting net.core.rmem\_default. Must be between `212992` and `134217728`. Changing this forces a new resource to be created.
net\_core\_rmem\_max = (Optional) The sysctl setting net.core.rmem\_max. Must be between `212992` and `134217728`. Changing this forces a new resource to be created.
net\_core\_somaxconn = (Optional) The sysctl setting net.core.somaxconn. Must be between `4096` and `3240000`. Changing this forces a new resource to be created.
net\_core\_wmem\_default = (Optional) The sysctl setting net.core.wmem\_default. Must be between `212992` and `134217728`. Changing this forces a new resource to be created.
net\_core\_wmem\_max = (Optional) The sysctl setting net.core.wmem\_max. Must be between `212992` and `134217728`. Changing this forces a new resource to be created.
net\_ipv4\_ip\_local\_port\_range\_min = (Optional) The sysctl setting net.ipv4.ip\_local\_port\_range min value. Must be between `1024` and `60999`. Changing this forces a new resource to be created.
net\_ipv4\_ip\_local\_port\_range\_max = (Optional) The sysctl setting net.ipv4.ip\_local\_port\_range max value. Must be between `1024` and `60999`. Changing this forces a new resource to be created.
net\_ipv4\_neigh\_default\_gc\_thresh1 = (Optional) The sysctl setting net.ipv4.neigh.default.gc\_thresh1. Must be between `128` and `80000`. Changing this forces a new resource to be created.
net\_ipv4\_neigh\_default\_gc\_thresh2 = (Optional) The sysctl setting net.ipv4.neigh.default.gc\_thresh2. Must be between `512` and `90000`. Changing this forces a new resource to be created.
net\_ipv4\_neigh\_default\_gc\_thresh3 = (Optional) The sysctl setting net.ipv4.neigh.default.gc\_thresh3. Must be between `1024` and `100000`. Changing this forces a new resource to be created.
net\_ipv4\_tcp\_fin\_timeout = (Optional) The sysctl setting net.ipv4.tcp\_fin\_timeout. Must be between `5` and `120`. Changing this forces a new resource to be created.
net\_ipv4\_tcp\_keepalive\_intvl = (Optional) The sysctl setting net.ipv4.tcp\_keepalive\_intvl. Must be between `10` and `75`. Changing this forces a new resource to be created.
net\_ipv4\_tcp\_keepalive\_probes = (Optional) The sysctl setting net.ipv4.tcp\_keepalive\_probes. Must be between `1` and `15`. Changing this forces a new resource to be created.
net\_ipv4\_tcp\_keepalive\_time = (Optional) The sysctl setting net.ipv4.tcp\_keepalive\_time. Must be between `30` and `432000`. Changing this forces a new resource to be created.
net\_ipv4\_tcp\_max\_syn\_backlog = (Optional) The sysctl setting net.ipv4.tcp\_max\_syn\_backlog. Must be between `128` and `3240000`. Changing this forces a new resource to be created.
net\_ipv4\_tcp\_max\_tw\_buckets = (Optional) The sysctl setting net.ipv4.tcp\_max\_tw\_buckets. Must be between `8000` and `1440000`. Changing this forces a new resource to be created.
net\_ipv4\_tcp\_tw\_reuse = (Optional) Is sysctl setting net.ipv4.tcp\_tw\_reuse enabled? Changing this forces a new resource to be created.
net\_netfilter\_nf\_conntrack\_buckets = (Optional) The sysctl setting net.netfilter.nf\_conntrack\_buckets. Must be between `65536` and `147456`. Changing this forces a new resource to be created.
net\_netfilter\_nf\_conntrack\_max = (Optional) The sysctl setting net.netfilter.nf\_conntrack\_max. Must be between `131072` and `1048576`. Changing this forces a new resource to be created.
vm\_max\_map\_count = (Optional) The sysctl setting vm.max\_map\_count. Must be between `65530` and `262144`. Changing this forces a new resource to be created.
vm\_swappiness = (Optional) The sysctl setting vm.swappiness. Must be between `0` and `100`. Changing this forces a new resource to be created.
vm\_vfs\_cache\_pressure = (Optional) The sysctl setting vm.vfs\_cache\_pressure. Must be between `0` and `100`. Changing this forces a new resource to be created.
}))
transparent\_huge\_page\_enabled = (Optional) Specifies the Transparent Huge Page enabled configuration. Possible values are `always`, `madvise` and `never`. Changing this forces a new resource to be created.
transparent\_huge\_page\_defrag = (Optional) specifies the defrag configuration for Transparent Huge Page. Possible values are `always`, `defer`, `defer+madvise`, `madvise` and `never`. Changing this forces a new resource to be created.
swap\_file\_size\_mb = (Optional) Specifies the size of swap file on each node in MB. Changing this forces a new resource to be created.
}))
fips\_enabled = (Optional) Should the nodes in this Node Pool have Federal Information Processing Standard enabled? Changing this forces a new resource to be created. FIPS support is in Public Preview - more information and details on how to opt into the Preview can be found in [this article](https://docs.microsoft.com/azure/aks/use-multiple-node-pools#add-a-fips-enabled-node-pool-preview).
kubelet\_disk\_type = (Optional) The type of disk used by kubelet. Possible values are `OS` and `Temporary`.
max\_count = (Optional) The maximum number of nodes which should exist within this Node Pool. Valid values are between `0` and `1000` and must be greater than or equal to `min_count`.
max\_pods = (Optional) The minimum number of nodes which should exist within this Node Pool. Valid values are between `0` and `1000` and must be less than or equal to `max_count`.
message\_of\_the\_day = (Optional) A base64-encoded string which will be written to /etc/motd after decoding. This allows customization of the message of the day for Linux nodes. It cannot be specified for Windows nodes and must be a static string (i.e. will be printed raw and not executed as a script). Changing this forces a new resource to be created.
mode = (Optional) Should this Node Pool be used for System or User resources? Possible values are `System` and `User`. Defaults to `User`.
min\_count = (Optional) The minimum number of nodes which should exist within this Node Pool. Valid values are between `0` and `1000` and must be less than or equal to `max_count`.
node\_network\_profile = optional(object({
node\_public\_ip\_tags = (Optional) Specifies a mapping of tags to the instance-level public IPs. Changing this forces a new resource to be created.
}))
node\_labels = (Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool.
node\_public\_ip\_prefix\_id = (Optional) Resource ID for the Public IP Addresses Prefix for the nodes in this Node Pool. `enable_node_public_ip` should be `true`. Changing this forces a new resource to be created.
node\_taints = (Optional) A list of Kubernetes taints which should be applied to nodes in the agent pool (e.g `key=value:NoSchedule`). Changing this forces a new resource to be created.
orchestrator\_version = (Optional) Version of Kubernetes used for the Agents. If not specified, the latest recommended version will be used at provisioning time (but won't auto-upgrade). AKS does not require an exact patch version to be specified, minor version aliases such as `1.22` are also supported. - The minor version's latest GA patch is automatically chosen in that case. More details can be found in [the documentation](https://docs.microsoft.com/en-us/azure/aks/supported-kubernetes-versions?tabs=azure-cli#alias-minor-version). This version must be supported by the Kubernetes Cluster - as such the version of Kubernetes used on the Cluster/Control Plane may need to be upgraded first.
os\_disk\_size\_gb = (Optional) The Agent Operating System disk size in GB. Changing this forces a new resource to be created.
os\_disk\_type = (Optional) The type of disk which should be used for the Operating System. Possible values are `Ephemeral` and `Managed`. Defaults to `Managed`. Changing this forces a new resource to be created.
os\_sku = (Optional) Specifies the OS SKU used by the agent pool. Possible values include: `Ubuntu`, `CBLMariner`, `Mariner`, `Windows2019`, `Windows2022`. If not specified, the default is `Ubuntu` if OSType=Linux or `Windows2019` if OSType=Windows. And the default Windows OSSKU will be changed to `Windows2022` after Windows2019 is deprecated. Changing this forces a new resource to be created.
os\_type = (Optional) The Operating System which should be used for this Node Pool. Changing this forces a new resource to be created. Possible values are `Linux` and `Windows`. Defaults to `Linux`.
pod\_subnet\_id = (Optional) The ID of the Subnet where the pods in the Node Pool should exist. Changing this forces a new resource to be created.
priority = (Optional) The Priority for Virtual Machines within the Virtual Machine Scale Set that powers this Node Pool. Possible values are `Regular` and `Spot`. Defaults to `Regular`. Changing this forces a new resource to be created.
proximity\_placement\_group\_id = (Optional) The ID of the Proximity Placement Group where the Virtual Machine Scale Set that powers this Node Pool will be placed. Changing this forces a new resource to be created. When setting `priority` to Spot - you must configure an `eviction_policy`, `spot_max_price` and add the applicable `node_labels` and `node_taints` [as per the Azure Documentation](https://docs.microsoft.com/azure/aks/spot-node-pool).
spot\_max\_price = (Optional) The maximum price you're willing to pay in USD per Virtual Machine. Valid values are `-1` (the current on-demand price for a Virtual Machine) or a positive value with up to five decimal places. Changing this forces a new resource to be created. This field can only be configured when `priority` is set to `Spot`.
scale\_down\_mode = (Optional) Specifies how the node pool should deal with scaled-down nodes. Allowed values are `Delete` and `Deallocate`. Defaults to `Delete`.
snapshot\_id = (Optional) The ID of the Snapshot which should be used to create this Node Pool. Changing this forces a new resource to be created.
ultra\_ssd\_enabled = (Optional) Used to specify whether the UltraSSD is enabled in the Node Pool. Defaults to `false`. See [the documentation](https://docs.microsoft.com/azure/aks/use-ultra-disks) for more information. Changing this forces a new resource to be created.
vnet\_subnet\_id = (Optional) The ID of the Subnet where this Node Pool should exist. Changing this forces a new resource to be created. A route table must be configured on this Subnet.
upgrade\_settings = optional(object({
max\_surge = string
}))
windows\_profile = optional(object({
outbound\_nat\_enabled = optional(bool, true)
}))
workload\_runtime = (Optional) Used to specify the workload runtime. Allowed values are `OCIContainer` and `WasmWasi`. WebAssembly System Interface node pools are in Public Preview - more information and details on how to opt into the preview can be found in [this article](https://docs.microsoft.com/azure/aks/use-wasi-node-pools)
zones = (Optional) Specifies a list of Availability Zones in which this Kubernetes Cluster Node Pool should be located. Changing this forces a new Kubernetes Cluster Node Pool to be created.
})) | 
map(object({
    name                          = string
    node_count                    = optional(number)
    tags                          = optional(map(string))
    vm_size                       = string
    host_group_id                 = optional(string)
    capacity_reservation_group_id = optional(string)
    custom_ca_trust_enabled       = optional(bool)
    enable_auto_scaling           = optional(bool)
    enable_host_encryption        = optional(bool)
    enable_node_public_ip         = optional(bool)
    eviction_policy               = optional(string)
    kubelet_config = optional(object({
      cpu_manager_policy        = optional(string)
      cpu_cfs_quota_enabled     = optional(bool)
      cpu_cfs_quota_period      = optional(string)
      image_gc_high_threshold   = optional(number)
      image_gc_low_threshold    = optional(number)
      topology_manager_policy   = optional(string)
      allowed_unsafe_sysctls    = optional(set(string))
      container_log_max_size_mb = optional(number)
      container_log_max_files   = optional(number)
      pod_max_pid               = optional(number)
    }))
    linux_os_config = optional(object({
      sysctl_config = optional(object({
        fs_aio_max_nr                      = optional(number)
        fs_file_max                        = optional(number)
        fs_inotify_max_user_watches        = optional(number)
        fs_nr_open                         = optional(number)
        kernel_threads_max                 = optional(number)
        net_core_netdev_max_backlog        = optional(number)
        net_core_optmem_max                = optional(number)
        net_core_rmem_default              = optional(number)
        net_core_rmem_max                  = optional(number)
        net_core_somaxconn                 = optional(number)
        net_core_wmem_default              = optional(number)
        net_core_wmem_max                  = optional(number)
        net_ipv4_ip_local_port_range_min   = optional(number)
        net_ipv4_ip_local_port_range_max   = optional(number)
        net_ipv4_neigh_default_gc_thresh1  = optional(number)
        net_ipv4_neigh_default_gc_thresh2  = optional(number)
        net_ipv4_neigh_default_gc_thresh3  = optional(number)
        net_ipv4_tcp_fin_timeout           = optional(number)
        net_ipv4_tcp_keepalive_intvl       = optional(number)
        net_ipv4_tcp_keepalive_probes      = optional(number)
        net_ipv4_tcp_keepalive_time        = optional(number)
        net_ipv4_tcp_max_syn_backlog       = optional(number)
        net_ipv4_tcp_max_tw_buckets        = optional(number)
        net_ipv4_tcp_tw_reuse              = optional(bool)
        net_netfilter_nf_conntrack_buckets = optional(number)
        net_netfilter_nf_conntrack_max     = optional(number)
        vm_max_map_count                   = optional(number)
        vm_swappiness                      = optional(number)
        vm_vfs_cache_pressure              = optional(number)
      }))
      transparent_huge_page_enabled = optional(string)
      transparent_huge_page_defrag  = optional(string)
      swap_file_size_mb             = optional(number)
    }))
    fips_enabled       = optional(bool)
    kubelet_disk_type  = optional(string)
    max_count          = optional(number)
    max_pods           = optional(number)
    message_of_the_day = optional(string)
    mode               = optional(string, "User")
    min_count          = optional(number)
    node_network_profile = optional(object({
      node_public_ip_tags = optional(map(string))
    }))
    node_labels                  = optional(map(string))
    node_public_ip_prefix_id     = optional(string)
    node_taints                  = optional(list(string))
    orchestrator_version         = optional(string)
    os_disk_size_gb              = optional(number)
    os_disk_type                 = optional(string, "Managed")
    os_sku                       = optional(string)
    os_type                      = optional(string, "Linux")
    pod_subnet_id                = optional(string)
    priority                     = optional(string, "Regular")
    proximity_placement_group_id = optional(string)
    spot_max_price               = optional(number)
    scale_down_mode              = optional(string, "Delete")
    snapshot_id                  = optional(string)
    ultra_ssd_enabled            = optional(bool)
    vnet_subnet_id               = optional(string)
    upgrade_settings = optional(object({
      max_surge = string
    }))
    windows_profile = optional(object({
      outbound_nat_enabled = optional(bool, true)
    }))
    workload_runtime = optional(string)
    zones            = optional(set(string))
  }))
| `{}` | no | +| [node\_pools](#input\_node\_pools) | A map of node pools that need to be created and attached on the Kubernetes cluster. The key of the map can be the name of the node pool, and the key must be static string. The value of the map is a `node_pool` block as defined below:
map(object({
name = (Required) The name of the Node Pool which should be created within the Kubernetes Cluster. Changing this forces a new resource to be created. A Windows Node Pool cannot have a `name` longer than 6 characters. A random suffix of 4 characters is always added to the name to avoid clashes during recreates.
node\_count = (Optional) The initial number of nodes which should exist within this Node Pool. Valid values are between `0` and `1000` (inclusive) for user pools and between `1` and `1000` (inclusive) for system pools and must be a value in the range `min_count` - `max_count`.
tags = (Optional) A mapping of tags to assign to the resource. At this time there's a bug in the AKS API where Tags for a Node Pool are not stored in the correct case - you [may wish to use Terraform's `ignore_changes` functionality to ignore changes to the casing](https://www.terraform.io/language/meta-arguments/lifecycle#ignore_changess) until this is fixed in the AKS API.
vm\_size = (Required) The SKU which should be used for the Virtual Machines used in this Node Pool. Changing this forces a new resource to be created.
host\_group\_id = (Optional) The fully qualified resource ID of the Dedicated Host Group to provision virtual machines from. Changing this forces a new resource to be created.
capacity\_reservation\_group\_id = (Optional) Specifies the ID of the Capacity Reservation Group where this Node Pool should exist. Changing this forces a new resource to be created.
custom\_ca\_trust\_enabled = (Optional) Specifies whether to trust a Custom CA. This requires that the Preview Feature `Microsoft.ContainerService/CustomCATrustPreview` is enabled and the Resource Provider is re-registered, see [the documentation](https://learn.microsoft.com/en-us/azure/aks/custom-certificate-authority) for more information.
enable\_auto\_scaling = (Optional) Whether to enable [auto-scaler](https://docs.microsoft.com/azure/aks/cluster-autoscaler).
enable\_host\_encryption = (Optional) Should the nodes in this Node Pool have host encryption enabled? Changing this forces a new resource to be created.
enable\_node\_public\_ip = (Optional) Should each node have a Public IP Address? Changing this forces a new resource to be created.
eviction\_policy = (Optional) The Eviction Policy which should be used for Virtual Machines within the Virtual Machine Scale Set powering this Node Pool. Possible values are `Deallocate` and `Delete`. Changing this forces a new resource to be created. An Eviction Policy can only be configured when `priority` is set to `Spot` and will default to `Delete` unless otherwise specified.
kubelet\_config = optional(object({
cpu\_manager\_policy = (Optional) Specifies the CPU Manager policy to use. Possible values are `none` and `static`, Changing this forces a new resource to be created.
cpu\_cfs\_quota\_enabled = (Optional) Is CPU CFS quota enforcement for containers enabled? Changing this forces a new resource to be created.
cpu\_cfs\_quota\_period = (Optional) Specifies the CPU CFS quota period value. Changing this forces a new resource to be created.
image\_gc\_high\_threshold = (Optional) Specifies the percent of disk usage above which image garbage collection is always run. Must be between `0` and `100`. Changing this forces a new resource to be created.
image\_gc\_low\_threshold = (Optional) Specifies the percent of disk usage lower than which image garbage collection is never run. Must be between `0` and `100`. Changing this forces a new resource to be created.
topology\_manager\_policy = (Optional) Specifies the Topology Manager policy to use. Possible values are `none`, `best-effort`, `restricted` or `single-numa-node`. Changing this forces a new resource to be created.
allowed\_unsafe\_sysctls = (Optional) Specifies the allow list of unsafe sysctls command or patterns (ending in `*`). Changing this forces a new resource to be created.
container\_log\_max\_size\_mb = (Optional) Specifies the maximum size (e.g. 10MB) of container log file before it is rotated. Changing this forces a new resource to be created.
container\_log\_max\_files = (Optional) Specifies the maximum number of container log files that can be present for a container. must be at least 2. Changing this forces a new resource to be created.
pod\_max\_pid = (Optional) Specifies the maximum number of processes per pod. Changing this forces a new resource to be created.
}))
linux\_os\_config = optional(object({
sysctl\_config = optional(object({
fs\_aio\_max\_nr = (Optional) The sysctl setting fs.aio-max-nr. Must be between `65536` and `6553500`. Changing this forces a new resource to be created.
fs\_file\_max = (Optional) The sysctl setting fs.file-max. Must be between `8192` and `12000500`. Changing this forces a new resource to be created.
fs\_inotify\_max\_user\_watches = (Optional) The sysctl setting fs.inotify.max\_user\_watches. Must be between `781250` and `2097152`. Changing this forces a new resource to be created.
fs\_nr\_open = (Optional) The sysctl setting fs.nr\_open. Must be between `8192` and `20000500`. Changing this forces a new resource to be created.
kernel\_threads\_max = (Optional) The sysctl setting kernel.threads-max. Must be between `20` and `513785`. Changing this forces a new resource to be created.
net\_core\_netdev\_max\_backlog = (Optional) The sysctl setting net.core.netdev\_max\_backlog. Must be between `1000` and `3240000`. Changing this forces a new resource to be created.
net\_core\_optmem\_max = (Optional) The sysctl setting net.core.optmem\_max. Must be between `20480` and `4194304`. Changing this forces a new resource to be created.
net\_core\_rmem\_default = (Optional) The sysctl setting net.core.rmem\_default. Must be between `212992` and `134217728`. Changing this forces a new resource to be created.
net\_core\_rmem\_max = (Optional) The sysctl setting net.core.rmem\_max. Must be between `212992` and `134217728`. Changing this forces a new resource to be created.
net\_core\_somaxconn = (Optional) The sysctl setting net.core.somaxconn. Must be between `4096` and `3240000`. Changing this forces a new resource to be created.
net\_core\_wmem\_default = (Optional) The sysctl setting net.core.wmem\_default. Must be between `212992` and `134217728`. Changing this forces a new resource to be created.
net\_core\_wmem\_max = (Optional) The sysctl setting net.core.wmem\_max. Must be between `212992` and `134217728`. Changing this forces a new resource to be created.
net\_ipv4\_ip\_local\_port\_range\_min = (Optional) The sysctl setting net.ipv4.ip\_local\_port\_range min value. Must be between `1024` and `60999`. Changing this forces a new resource to be created.
net\_ipv4\_ip\_local\_port\_range\_max = (Optional) The sysctl setting net.ipv4.ip\_local\_port\_range max value. Must be between `1024` and `60999`. Changing this forces a new resource to be created.
net\_ipv4\_neigh\_default\_gc\_thresh1 = (Optional) The sysctl setting net.ipv4.neigh.default.gc\_thresh1. Must be between `128` and `80000`. Changing this forces a new resource to be created.
net\_ipv4\_neigh\_default\_gc\_thresh2 = (Optional) The sysctl setting net.ipv4.neigh.default.gc\_thresh2. Must be between `512` and `90000`. Changing this forces a new resource to be created.
net\_ipv4\_neigh\_default\_gc\_thresh3 = (Optional) The sysctl setting net.ipv4.neigh.default.gc\_thresh3. Must be between `1024` and `100000`. Changing this forces a new resource to be created.
net\_ipv4\_tcp\_fin\_timeout = (Optional) The sysctl setting net.ipv4.tcp\_fin\_timeout. Must be between `5` and `120`. Changing this forces a new resource to be created.
net\_ipv4\_tcp\_keepalive\_intvl = (Optional) The sysctl setting net.ipv4.tcp\_keepalive\_intvl. Must be between `10` and `75`. Changing this forces a new resource to be created.
net\_ipv4\_tcp\_keepalive\_probes = (Optional) The sysctl setting net.ipv4.tcp\_keepalive\_probes. Must be between `1` and `15`. Changing this forces a new resource to be created.
net\_ipv4\_tcp\_keepalive\_time = (Optional) The sysctl setting net.ipv4.tcp\_keepalive\_time. Must be between `30` and `432000`. Changing this forces a new resource to be created.
net\_ipv4\_tcp\_max\_syn\_backlog = (Optional) The sysctl setting net.ipv4.tcp\_max\_syn\_backlog. Must be between `128` and `3240000`. Changing this forces a new resource to be created.
net\_ipv4\_tcp\_max\_tw\_buckets = (Optional) The sysctl setting net.ipv4.tcp\_max\_tw\_buckets. Must be between `8000` and `1440000`. Changing this forces a new resource to be created.
net\_ipv4\_tcp\_tw\_reuse = (Optional) Is sysctl setting net.ipv4.tcp\_tw\_reuse enabled? Changing this forces a new resource to be created.
net\_netfilter\_nf\_conntrack\_buckets = (Optional) The sysctl setting net.netfilter.nf\_conntrack\_buckets. Must be between `65536` and `147456`. Changing this forces a new resource to be created.
net\_netfilter\_nf\_conntrack\_max = (Optional) The sysctl setting net.netfilter.nf\_conntrack\_max. Must be between `131072` and `1048576`. Changing this forces a new resource to be created.
vm\_max\_map\_count = (Optional) The sysctl setting vm.max\_map\_count. Must be between `65530` and `262144`. Changing this forces a new resource to be created.
vm\_swappiness = (Optional) The sysctl setting vm.swappiness. Must be between `0` and `100`. Changing this forces a new resource to be created.
vm\_vfs\_cache\_pressure = (Optional) The sysctl setting vm.vfs\_cache\_pressure. Must be between `0` and `100`. Changing this forces a new resource to be created.
}))
transparent\_huge\_page\_enabled = (Optional) Specifies the Transparent Huge Page enabled configuration. Possible values are `always`, `madvise` and `never`. Changing this forces a new resource to be created.
transparent\_huge\_page\_defrag = (Optional) specifies the defrag configuration for Transparent Huge Page. Possible values are `always`, `defer`, `defer+madvise`, `madvise` and `never`. Changing this forces a new resource to be created.
swap\_file\_size\_mb = (Optional) Specifies the size of swap file on each node in MB. Changing this forces a new resource to be created.
}))
fips\_enabled = (Optional) Should the nodes in this Node Pool have Federal Information Processing Standard enabled? Changing this forces a new resource to be created. FIPS support is in Public Preview - more information and details on how to opt into the Preview can be found in [this article](https://docs.microsoft.com/azure/aks/use-multiple-node-pools#add-a-fips-enabled-node-pool-preview).
kubelet\_disk\_type = (Optional) The type of disk used by kubelet. Possible values are `OS` and `Temporary`.
max\_count = (Optional) The maximum number of nodes which should exist within this Node Pool. Valid values are between `0` and `1000` and must be greater than or equal to `min_count`.
max\_pods = (Optional) The minimum number of nodes which should exist within this Node Pool. Valid values are between `0` and `1000` and must be less than or equal to `max_count`.
message\_of\_the\_day = (Optional) A base64-encoded string which will be written to /etc/motd after decoding. This allows customization of the message of the day for Linux nodes. It cannot be specified for Windows nodes and must be a static string (i.e. will be printed raw and not executed as a script). Changing this forces a new resource to be created.
mode = (Optional) Should this Node Pool be used for System or User resources? Possible values are `System` and `User`. Defaults to `User`.
min\_count = (Optional) The minimum number of nodes which should exist within this Node Pool. Valid values are between `0` and `1000` and must be less than or equal to `max_count`.
node\_network\_profile = optional(object({
node\_public\_ip\_tags = (Optional) Specifies a mapping of tags to the instance-level public IPs. Changing this forces a new resource to be created.
}))
node\_labels = (Optional) A map of Kubernetes labels which should be applied to nodes in this Node Pool.
node\_public\_ip\_prefix\_id = (Optional) Resource ID for the Public IP Addresses Prefix for the nodes in this Node Pool. `enable_node_public_ip` should be `true`. Changing this forces a new resource to be created.
node\_taints = (Optional) A list of Kubernetes taints which should be applied to nodes in the agent pool (e.g `key=value:NoSchedule`). Changing this forces a new resource to be created.
orchestrator\_version = (Optional) Version of Kubernetes used for the Agents. If not specified, the latest recommended version will be used at provisioning time (but won't auto-upgrade). AKS does not require an exact patch version to be specified, minor version aliases such as `1.22` are also supported. - The minor version's latest GA patch is automatically chosen in that case. More details can be found in [the documentation](https://docs.microsoft.com/en-us/azure/aks/supported-kubernetes-versions?tabs=azure-cli#alias-minor-version). This version must be supported by the Kubernetes Cluster - as such the version of Kubernetes used on the Cluster/Control Plane may need to be upgraded first.
os\_disk\_size\_gb = (Optional) The Agent Operating System disk size in GB. Changing this forces a new resource to be created.
os\_disk\_type = (Optional) The type of disk which should be used for the Operating System. Possible values are `Ephemeral` and `Managed`. Defaults to `Managed`. Changing this forces a new resource to be created.
os\_sku = (Optional) Specifies the OS SKU used by the agent pool. Possible values include: `Ubuntu`, `CBLMariner`, `Mariner`, `Windows2019`, `Windows2022`. If not specified, the default is `Ubuntu` if OSType=Linux or `Windows2019` if OSType=Windows. And the default Windows OSSKU will be changed to `Windows2022` after Windows2019 is deprecated. Changing this forces a new resource to be created.
os\_type = (Optional) The Operating System which should be used for this Node Pool. Changing this forces a new resource to be created. Possible values are `Linux` and `Windows`. Defaults to `Linux`.
pod\_subnet\_id = (Optional) The ID of the Subnet where the pods in the Node Pool should exist. Changing this forces a new resource to be created.
priority = (Optional) The Priority for Virtual Machines within the Virtual Machine Scale Set that powers this Node Pool. Possible values are `Regular` and `Spot`. Defaults to `Regular`. Changing this forces a new resource to be created.
proximity\_placement\_group\_id = (Optional) The ID of the Proximity Placement Group where the Virtual Machine Scale Set that powers this Node Pool will be placed. Changing this forces a new resource to be created. When setting `priority` to Spot - you must configure an `eviction_policy`, `spot_max_price` and add the applicable `node_labels` and `node_taints` [as per the Azure Documentation](https://docs.microsoft.com/azure/aks/spot-node-pool).
spot\_max\_price = (Optional) The maximum price you're willing to pay in USD per Virtual Machine. Valid values are `-1` (the current on-demand price for a Virtual Machine) or a positive value with up to five decimal places. Changing this forces a new resource to be created. This field can only be configured when `priority` is set to `Spot`.
scale\_down\_mode = (Optional) Specifies how the node pool should deal with scaled-down nodes. Allowed values are `Delete` and `Deallocate`. Defaults to `Delete`.
snapshot\_id = (Optional) The ID of the Snapshot which should be used to create this Node Pool. Changing this forces a new resource to be created.
ultra\_ssd\_enabled = (Optional) Used to specify whether the UltraSSD is enabled in the Node Pool. Defaults to `false`. See [the documentation](https://docs.microsoft.com/azure/aks/use-ultra-disks) for more information. Changing this forces a new resource to be created.
vnet\_subnet\_id = (Optional) The ID of the Subnet where this Node Pool should exist. Changing this forces a new resource to be created. A route table must be configured on this Subnet.
upgrade\_settings = optional(object({
max\_surge = string
}))
windows\_profile = optional(object({
outbound\_nat\_enabled = optional(bool, true)
}))
workload\_runtime = (Optional) Used to specify the workload runtime. Allowed values are `OCIContainer` and `WasmWasi`. WebAssembly System Interface node pools are in Public Preview - more information and details on how to opt into the preview can be found in [this article](https://docs.microsoft.com/azure/aks/use-wasi-node-pools)
zones = (Optional) Specifies a list of Availability Zones in which this Kubernetes Cluster Node Pool should be located. Changing this forces a new Kubernetes Cluster Node Pool to be created.
create\_before\_destroy = (Optional) Create a new node pool before destroy the old one when Terraform must update an argument that cannot be updated in-place. Set this argument to `true` will add add a random suffix to pool's name to avoid conflict. Default to `true`.
})) | 
map(object({
    name                          = string
    node_count                    = optional(number)
    tags                          = optional(map(string))
    vm_size                       = string
    host_group_id                 = optional(string)
    capacity_reservation_group_id = optional(string)
    custom_ca_trust_enabled       = optional(bool)
    enable_auto_scaling           = optional(bool)
    enable_host_encryption        = optional(bool)
    enable_node_public_ip         = optional(bool)
    eviction_policy               = optional(string)
    kubelet_config = optional(object({
      cpu_manager_policy        = optional(string)
      cpu_cfs_quota_enabled     = optional(bool)
      cpu_cfs_quota_period      = optional(string)
      image_gc_high_threshold   = optional(number)
      image_gc_low_threshold    = optional(number)
      topology_manager_policy   = optional(string)
      allowed_unsafe_sysctls    = optional(set(string))
      container_log_max_size_mb = optional(number)
      container_log_max_files   = optional(number)
      pod_max_pid               = optional(number)
    }))
    linux_os_config = optional(object({
      sysctl_config = optional(object({
        fs_aio_max_nr                      = optional(number)
        fs_file_max                        = optional(number)
        fs_inotify_max_user_watches        = optional(number)
        fs_nr_open                         = optional(number)
        kernel_threads_max                 = optional(number)
        net_core_netdev_max_backlog        = optional(number)
        net_core_optmem_max                = optional(number)
        net_core_rmem_default              = optional(number)
        net_core_rmem_max                  = optional(number)
        net_core_somaxconn                 = optional(number)
        net_core_wmem_default              = optional(number)
        net_core_wmem_max                  = optional(number)
        net_ipv4_ip_local_port_range_min   = optional(number)
        net_ipv4_ip_local_port_range_max   = optional(number)
        net_ipv4_neigh_default_gc_thresh1  = optional(number)
        net_ipv4_neigh_default_gc_thresh2  = optional(number)
        net_ipv4_neigh_default_gc_thresh3  = optional(number)
        net_ipv4_tcp_fin_timeout           = optional(number)
        net_ipv4_tcp_keepalive_intvl       = optional(number)
        net_ipv4_tcp_keepalive_probes      = optional(number)
        net_ipv4_tcp_keepalive_time        = optional(number)
        net_ipv4_tcp_max_syn_backlog       = optional(number)
        net_ipv4_tcp_max_tw_buckets        = optional(number)
        net_ipv4_tcp_tw_reuse              = optional(bool)
        net_netfilter_nf_conntrack_buckets = optional(number)
        net_netfilter_nf_conntrack_max     = optional(number)
        vm_max_map_count                   = optional(number)
        vm_swappiness                      = optional(number)
        vm_vfs_cache_pressure              = optional(number)
      }))
      transparent_huge_page_enabled = optional(string)
      transparent_huge_page_defrag  = optional(string)
      swap_file_size_mb             = optional(number)
    }))
    fips_enabled       = optional(bool)
    kubelet_disk_type  = optional(string)
    max_count          = optional(number)
    max_pods           = optional(number)
    message_of_the_day = optional(string)
    mode               = optional(string, "User")
    min_count          = optional(number)
    node_network_profile = optional(object({
      node_public_ip_tags = optional(map(string))
    }))
    node_labels                  = optional(map(string))
    node_public_ip_prefix_id     = optional(string)
    node_taints                  = optional(list(string))
    orchestrator_version         = optional(string)
    os_disk_size_gb              = optional(number)
    os_disk_type                 = optional(string, "Managed")
    os_sku                       = optional(string)
    os_type                      = optional(string, "Linux")
    pod_subnet_id                = optional(string)
    priority                     = optional(string, "Regular")
    proximity_placement_group_id = optional(string)
    spot_max_price               = optional(number)
    scale_down_mode              = optional(string, "Delete")
    snapshot_id                  = optional(string)
    ultra_ssd_enabled            = optional(bool)
    vnet_subnet_id               = optional(string)
    upgrade_settings = optional(object({
      max_surge = string
    }))
    windows_profile = optional(object({
      outbound_nat_enabled = optional(bool, true)
    }))
    workload_runtime      = optional(string)
    zones                 = optional(set(string))
    create_before_destroy = optional(bool, true)
  }))
| `{}` | no | | [node\_resource\_group](#input\_node\_resource\_group) | The auto-generated Resource Group which contains the resources for this Managed Kubernetes Cluster. Changing this forces a new resource to be created. | `string` | `null` | no | | [oidc\_issuer\_enabled](#input\_oidc\_issuer\_enabled) | Enable or Disable the OIDC issuer URL. Defaults to false. | `bool` | `false` | no | | [only\_critical\_addons\_enabled](#input\_only\_critical\_addons\_enabled) | (Optional) Enabling this option will taint default node pool with `CriticalAddonsOnly=true:NoSchedule` taint. Changing this forces a new resource to be created. | `bool` | `null` | no | diff --git a/extra_node_pool.tf b/extra_node_pool.tf index 8631af97..7641b48c 100644 --- a/extra_node_pool.tf +++ b/extra_node_pool.tf @@ -1,5 +1,10 @@ -resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { - for_each = var.node_pools +moved { + from = azurerm_kubernetes_cluster_node_pool.node_pool + to = azurerm_kubernetes_cluster_node_pool.node_pool_create_before_destroy +} + +resource "azurerm_kubernetes_cluster_node_pool" "node_pool_create_before_destroy" { + for_each = local.node_pools_create_before_destroy kubernetes_cluster_id = azurerm_kubernetes_cluster.main.id name = "${each.value.name}${substr(md5(jsonencode(each.value)), 0, 4)}" @@ -34,14 +39,13 @@ resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { snapshot_id = each.value.snapshot_id spot_max_price = each.value.spot_max_price tags = merge(each.value.tags, (/**/ (var.tracing_tags_enabled ? { for k, v in /**/ { + avm_yor_name = "node_pool" avm_git_commit = "bc0c9fab9ee53296a64c7a682d2ed7e0726c6547" avm_git_file = "main.tf" avm_git_last_modified_at = "2023-05-04 05:02:32" avm_git_org = "Azure" avm_git_repo = "terraform-azurerm-aks" - avm_yor_trace = "6dcb4922-45aa-469d-ab2e-1945e2735ac5" - } /**/ : replace(k, "avm_", var.tracing_tags_prefix) => v } : {}) /**/), (/**/ (var.tracing_tags_enabled ? { for k, v in /**/ { - avm_yor_name = "node_pool" + avm_yor_trace = "2aa46696-a713-4c12-95c0-550c5ff7f8ec" } /**/ : replace(k, "avm_", var.tracing_tags_prefix) => v } : {}) /**/)) ultra_ssd_enabled = each.value.ultra_ssd_enabled vnet_subnet_id = each.value.vnet_subnet_id @@ -161,6 +165,160 @@ resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { } } +resource "azurerm_kubernetes_cluster_node_pool" "node_pool_create_after_destroy" { + for_each = local.node_pools_create_after_destroy + + kubernetes_cluster_id = azurerm_kubernetes_cluster.main.id + name = each.value.name + vm_size = each.value.vm_size + capacity_reservation_group_id = each.value.capacity_reservation_group_id + custom_ca_trust_enabled = each.value.custom_ca_trust_enabled + enable_auto_scaling = each.value.enable_auto_scaling + enable_host_encryption = each.value.enable_host_encryption + enable_node_public_ip = each.value.enable_node_public_ip + eviction_policy = each.value.eviction_policy + fips_enabled = each.value.fips_enabled + host_group_id = each.value.host_group_id + kubelet_disk_type = each.value.kubelet_disk_type + max_count = each.value.max_count + max_pods = each.value.max_pods + message_of_the_day = each.value.message_of_the_day + min_count = each.value.min_count + mode = each.value.mode + node_count = each.value.node_count + node_labels = each.value.node_labels + node_public_ip_prefix_id = each.value.node_public_ip_prefix_id + node_taints = each.value.node_taints + orchestrator_version = each.value.orchestrator_version + os_disk_size_gb = each.value.os_disk_size_gb + os_disk_type = each.value.os_disk_type + os_sku = each.value.os_sku + os_type = each.value.os_type + pod_subnet_id = each.value.pod_subnet_id + priority = each.value.priority + proximity_placement_group_id = each.value.proximity_placement_group_id + scale_down_mode = each.value.scale_down_mode + snapshot_id = each.value.snapshot_id + spot_max_price = each.value.spot_max_price + tags = merge(each.value.tags, (/**/ (var.tracing_tags_enabled ? { for k, v in /**/ { + avm_yor_name = "node_pool" + avm_git_commit = "bc0c9fab9ee53296a64c7a682d2ed7e0726c6547" + avm_git_file = "main.tf" + avm_git_last_modified_at = "2023-05-04 05:02:32" + avm_git_org = "Azure" + avm_git_repo = "terraform-azurerm-aks" + avm_yor_trace = "2aa46696-a713-4c12-95c0-550c5ff7f8ec" + } /**/ : replace(k, "avm_", var.tracing_tags_prefix) => v } : {}) /**/)) + ultra_ssd_enabled = each.value.ultra_ssd_enabled + vnet_subnet_id = each.value.vnet_subnet_id + workload_runtime = each.value.workload_runtime + zones = each.value.zones + + dynamic "kubelet_config" { + for_each = each.value.kubelet_config == null ? [] : ["kubelet_config"] + + content { + allowed_unsafe_sysctls = each.value.kubelet_config.allowed_unsafe_sysctls + container_log_max_line = each.value.kubelet_config.container_log_max_files + container_log_max_size_mb = each.value.kubelet_config.container_log_max_size_mb + cpu_cfs_quota_enabled = each.value.kubelet_config.cpu_cfs_quota_enabled + cpu_cfs_quota_period = each.value.kubelet_config.cpu_cfs_quota_period + cpu_manager_policy = each.value.kubelet_config.cpu_manager_policy + image_gc_high_threshold = each.value.kubelet_config.image_gc_high_threshold + image_gc_low_threshold = each.value.kubelet_config.image_gc_low_threshold + pod_max_pid = each.value.kubelet_config.pod_max_pid + topology_manager_policy = each.value.kubelet_config.topology_manager_policy + } + } + dynamic "linux_os_config" { + for_each = each.value.linux_os_config == null ? [] : ["linux_os_config"] + + content { + swap_file_size_mb = each.value.linux_os_config.swap_file_size_mb + transparent_huge_page_defrag = each.value.linux_os_config.transparent_huge_page_defrag + transparent_huge_page_enabled = each.value.linux_os_config.transparent_huge_page_enabled + + dynamic "sysctl_config" { + for_each = each.value.linux_os_config.sysctl_config == null ? [] : ["sysctl_config"] + + content { + fs_aio_max_nr = each.value.linux_os_config.sysctl_config.fs_aio_max_nr + fs_file_max = each.value.linux_os_config.sysctl_config.fs_file_max + fs_inotify_max_user_watches = each.value.linux_os_config.sysctl_config.fs_inotify_max_user_watches + fs_nr_open = each.value.linux_os_config.sysctl_config.fs_nr_open + kernel_threads_max = each.value.linux_os_config.sysctl_config.kernel_threads_max + net_core_netdev_max_backlog = each.value.linux_os_config.sysctl_config.net_core_netdev_max_backlog + net_core_optmem_max = each.value.linux_os_config.sysctl_config.net_core_optmem_max + net_core_rmem_default = each.value.linux_os_config.sysctl_config.net_core_rmem_default + net_core_rmem_max = each.value.linux_os_config.sysctl_config.net_core_rmem_max + net_core_somaxconn = each.value.linux_os_config.sysctl_config.net_core_somaxconn + net_core_wmem_default = each.value.linux_os_config.sysctl_config.net_core_wmem_default + net_core_wmem_max = each.value.linux_os_config.sysctl_config.net_core_wmem_max + net_ipv4_ip_local_port_range_max = each.value.linux_os_config.sysctl_config.net_ipv4_ip_local_port_range_max + net_ipv4_ip_local_port_range_min = each.value.linux_os_config.sysctl_config.net_ipv4_ip_local_port_range_min + net_ipv4_neigh_default_gc_thresh1 = each.value.linux_os_config.sysctl_config.net_ipv4_neigh_default_gc_thresh1 + net_ipv4_neigh_default_gc_thresh2 = each.value.linux_os_config.sysctl_config.net_ipv4_neigh_default_gc_thresh2 + net_ipv4_neigh_default_gc_thresh3 = each.value.linux_os_config.sysctl_config.net_ipv4_neigh_default_gc_thresh3 + net_ipv4_tcp_fin_timeout = each.value.linux_os_config.sysctl_config.net_ipv4_tcp_fin_timeout + net_ipv4_tcp_keepalive_intvl = each.value.linux_os_config.sysctl_config.net_ipv4_tcp_keepalive_intvl + net_ipv4_tcp_keepalive_probes = each.value.linux_os_config.sysctl_config.net_ipv4_tcp_keepalive_probes + net_ipv4_tcp_keepalive_time = each.value.linux_os_config.sysctl_config.net_ipv4_tcp_keepalive_time + net_ipv4_tcp_max_syn_backlog = each.value.linux_os_config.sysctl_config.net_ipv4_tcp_max_syn_backlog + net_ipv4_tcp_max_tw_buckets = each.value.linux_os_config.sysctl_config.net_ipv4_tcp_max_tw_buckets + net_ipv4_tcp_tw_reuse = each.value.linux_os_config.sysctl_config.net_ipv4_tcp_tw_reuse + net_netfilter_nf_conntrack_buckets = each.value.linux_os_config.sysctl_config.net_netfilter_nf_conntrack_buckets + net_netfilter_nf_conntrack_max = each.value.linux_os_config.sysctl_config.net_netfilter_nf_conntrack_max + vm_max_map_count = each.value.linux_os_config.sysctl_config.vm_max_map_count + vm_swappiness = each.value.linux_os_config.sysctl_config.vm_swappiness + vm_vfs_cache_pressure = each.value.linux_os_config.sysctl_config.vm_vfs_cache_pressure + } + } + } + } + dynamic "node_network_profile" { + for_each = each.value.node_network_profile == null ? [] : ["node_network_profile"] + + content { + node_public_ip_tags = each.value.node_network_profile.node_public_ip_tags + } + } + dynamic "upgrade_settings" { + for_each = each.value.upgrade_settings == null ? [] : ["upgrade_settings"] + + content { + max_surge = each.value.upgrade_settings.max_surge + } + } + dynamic "windows_profile" { + for_each = each.value.windows_profile == null ? [] : ["windows_profile"] + + content { + outbound_nat_enabled = each.value.windows_profile.outbound_nat_enabled + } + } + + depends_on = [azapi_update_resource.aks_cluster_post_create] + + lifecycle { + precondition { + condition = can(regex("[a-z0-9]{1,8}", each.value.name)) + error_message = "A Node Pools name must consist of alphanumeric characters and have a maximum lenght of 8 characters (4 random chars added)" + } + precondition { + condition = var.network_plugin_mode != "overlay" || each.value.os_type != "Windows" + error_message = "Windows Server 2019 node pools are not supported for Overlay and Windows support is still in preview" + } + precondition { + condition = var.network_plugin_mode != "overlay" || !can(regex("^Standard_DC[0-9]+s?_v2$", each.value.vm_size)) + error_message = "With with Azure CNI Overlay you can't use DCsv2-series virtual machines in node pools. " + } + precondition { + condition = var.agents_type == "VirtualMachineScaleSets" + error_message = "Multiple Node Pools are only supported when the Kubernetes Cluster is using Virtual Machine Scale Sets." + } + } +} + resource "null_resource" "pool_name_keeper" { for_each = var.node_pools diff --git a/locals.tf b/locals.tf index 03eb3edd..c81cc852 100644 --- a/locals.tf +++ b/locals.tf @@ -9,6 +9,7 @@ locals { (contains(["patch"], var.automatic_channel_upgrade) && can(regex("^[0-9]{1,}\\.[0-9]{1,}$", var.kubernetes_version)) && (can(regex("^[0-9]{1,}\\.[0-9]{1,}$", var.orchestrator_version)) || var.orchestrator_version == null)) || (contains(["rapid", "stable", "node-image"], var.automatic_channel_upgrade) && var.kubernetes_version == null && var.orchestrator_version == null) ) + cluster_name = coalesce(var.cluster_name, trim("${var.prefix}-aks", "-")) # Abstract the decision whether to create an Analytics Workspace or not. create_analytics_solution = var.log_analytics_workspace_enabled && var.log_analytics_solution == null create_analytics_workspace = var.log_analytics_workspace_enabled && var.log_analytics_workspace == null @@ -45,6 +46,8 @@ locals { resource_group_name = split("/", var.log_analytics_workspace.id)[4] } ) : null # Finally, the Log Analytics Workspace should be disabled. + node_pools_create_after_destroy = [for p in var.node_pools : p if p.create_before_destroy != true] + node_pools_create_before_destroy = [for p in var.node_pools : p if p.create_before_destroy == true] potential_subnet_ids = flatten(concat([ for pool in var.node_pools : [ pool.vnet_subnet_id, diff --git a/main.tf b/main.tf index d175390c..74981745 100644 --- a/main.tf +++ b/main.tf @@ -16,7 +16,7 @@ resource "tls_private_key" "ssh" { resource "azurerm_kubernetes_cluster" "main" { location = coalesce(var.location, data.azurerm_resource_group.main.location) - name = coalesce(var.cluster_name, trim("${var.prefix}-aks", "-")) + name = "${local.cluster_name}${var.cluster_name_random_suffix ? substr(md5(uuid()), 0, 4) : ""}" resource_group_name = data.azurerm_resource_group.main.name automatic_channel_upgrade = var.automatic_channel_upgrade azure_policy_enabled = var.azure_policy_enabled @@ -524,6 +524,11 @@ resource "azurerm_kubernetes_cluster" "main" { http_proxy_config[0].no_proxy, kubernetes_version, public_network_access_enabled, + # we might have a random suffix in cluster's name so we have to ignore it here, but we've traced user supplied cluster name by `null_resource.kubernetes_cluster_name_keeper` so when the name is changed we'll recreate this resource. + name, + ] + replace_triggered_by = [ + null_resource.kubernetes_cluster_name_keeper.id ] precondition { @@ -595,6 +600,12 @@ resource "azurerm_kubernetes_cluster" "main" { } } +resource "null_resource" "kubernetes_cluster_name_keeper" { + triggers = { + name = local.cluster_name + } +} + resource "null_resource" "kubernetes_version_keeper" { triggers = { version = var.kubernetes_version diff --git a/variables.tf b/variables.tf index 3ce8e17c..0211b657 100644 --- a/variables.tf +++ b/variables.tf @@ -401,6 +401,13 @@ variable "cluster_name" { description = "(Optional) The name for the AKS resources created in the specified Azure Resource Group. This variable overwrites the 'prefix' var (The 'prefix' var will still be applied to the dns_prefix if it is set)" } +variable "cluster_name_random_suffix" { + type = bool + default = false + description = "Whether to add a random suffix on Aks cluster's name or not. `azurerm_kubernetes_cluster` resource defined in this module is `create_before_destroy = true` implicity now(described [here](https://github.com/Azure/terraform-azurerm-aks/issues/389)), without this random suffix we'll not be able to recreate this cluster directly due to the naming conflict." + nullable = false +} + variable "confidential_computing" { type = object({ sgx_quote_helper_enabled = bool @@ -926,8 +933,9 @@ variable "node_pools" { windows_profile = optional(object({ outbound_nat_enabled = optional(bool, true) })) - workload_runtime = optional(string) - zones = optional(set(string)) + workload_runtime = optional(string) + zones = optional(set(string)) + create_before_destroy = optional(bool, true) })) default = {} description = <<-EOT @@ -1026,6 +1034,7 @@ variable "node_pools" { })) workload_runtime = (Optional) Used to specify the workload runtime. Allowed values are `OCIContainer` and `WasmWasi`. WebAssembly System Interface node pools are in Public Preview - more information and details on how to opt into the preview can be found in [this article](https://docs.microsoft.com/azure/aks/use-wasi-node-pools) zones = (Optional) Specifies a list of Availability Zones in which this Kubernetes Cluster Node Pool should be located. Changing this forces a new Kubernetes Cluster Node Pool to be created. + create_before_destroy = (Optional) Create a new node pool before destroy the old one when Terraform must update an argument that cannot be updated in-place. Set this argument to `true` will add add a random suffix to pool's name to avoid conflict. Default to `true`. })) EOT nullable = false From 0f4ac24147c0734bfd24ac01986dbbe5602d9496 Mon Sep 17 00:00:00 2001 From: hezijie Date: Thu, 11 Jan 2024 17:10:09 +0800 Subject: [PATCH 2/6] fix incorrect type --- locals.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/locals.tf b/locals.tf index c81cc852..2f4d1c3f 100644 --- a/locals.tf +++ b/locals.tf @@ -46,8 +46,8 @@ locals { resource_group_name = split("/", var.log_analytics_workspace.id)[4] } ) : null # Finally, the Log Analytics Workspace should be disabled. - node_pools_create_after_destroy = [for p in var.node_pools : p if p.create_before_destroy != true] - node_pools_create_before_destroy = [for p in var.node_pools : p if p.create_before_destroy == true] + node_pools_create_after_destroy = { for k, p in var.node_pools : k => p if p.create_before_destroy != true } + node_pools_create_before_destroy = { for k, p in var.node_pools : k => p if p.create_before_destroy == true } potential_subnet_ids = flatten(concat([ for pool in var.node_pools : [ pool.vnet_subnet_id, From f5d6af180002532f6eb873c889a5cda948760816 Mon Sep 17 00:00:00 2001 From: zjhe Date: Fri, 12 Jan 2024 11:24:39 +0800 Subject: [PATCH 3/6] add retry when ingress is not ready --- examples/application_gateway_ingress/k8s_workload.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/application_gateway_ingress/k8s_workload.tf b/examples/application_gateway_ingress/k8s_workload.tf index c4d6d0fa..ac9c8457 100644 --- a/examples/application_gateway_ingress/k8s_workload.tf +++ b/examples/application_gateway_ingress/k8s_workload.tf @@ -96,7 +96,7 @@ resource "kubernetes_ingress_v1" "ing" { } resource "time_sleep" "wait_for_ingress" { - create_duration = "15m" + create_duration = "5m" depends_on = [kubernetes_ingress_v1.ing] } From c9f88207b23c93ec639498856ce54a360c83549e Mon Sep 17 00:00:00 2001 From: hezijie Date: Mon, 19 Feb 2024 12:48:39 +0800 Subject: [PATCH 4/6] revert change to example --- examples/application_gateway_ingress/k8s_workload.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/application_gateway_ingress/k8s_workload.tf b/examples/application_gateway_ingress/k8s_workload.tf index ac9c8457..c4d6d0fa 100644 --- a/examples/application_gateway_ingress/k8s_workload.tf +++ b/examples/application_gateway_ingress/k8s_workload.tf @@ -96,7 +96,7 @@ resource "kubernetes_ingress_v1" "ing" { } resource "time_sleep" "wait_for_ingress" { - create_duration = "5m" + create_duration = "15m" depends_on = [kubernetes_ingress_v1.ing] } From 984612fe51afab775578891081525cae73b00280 Mon Sep 17 00:00:00 2001 From: hezijie Date: Mon, 19 Feb 2024 13:39:17 +0800 Subject: [PATCH 5/6] update extra node pool's name suffix, use uuid() as random seed --- examples/multiple_node_pools/main.tf | 1 + extra_node_pool.tf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/multiple_node_pools/main.tf b/examples/multiple_node_pools/main.tf index a9ee476a..be3e670b 100644 --- a/examples/multiple_node_pools/main.tf +++ b/examples/multiple_node_pools/main.tf @@ -38,6 +38,7 @@ locals { vm_size = "Standard_D2s_v3" node_count = 1 vnet_subnet_id = azurerm_subnet.test.id + create_before_destroy = i % 2 == 0 } } } diff --git a/extra_node_pool.tf b/extra_node_pool.tf index 7641b48c..f7a121a5 100644 --- a/extra_node_pool.tf +++ b/extra_node_pool.tf @@ -7,7 +7,7 @@ resource "azurerm_kubernetes_cluster_node_pool" "node_pool_create_before_destroy for_each = local.node_pools_create_before_destroy kubernetes_cluster_id = azurerm_kubernetes_cluster.main.id - name = "${each.value.name}${substr(md5(jsonencode(each.value)), 0, 4)}" + name = "${each.value.name}${substr(md5(uuid()), 0, 4)}" vm_size = each.value.vm_size capacity_reservation_group_id = each.value.capacity_reservation_group_id custom_ca_trust_enabled = each.value.custom_ca_trust_enabled From 4a20a4e033dc2a82403e903003a94c0edc63efdd Mon Sep 17 00:00:00 2001 From: hezijie Date: Mon, 19 Feb 2024 14:14:49 +0800 Subject: [PATCH 6/6] fix format issue --- examples/multiple_node_pools/main.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/multiple_node_pools/main.tf b/examples/multiple_node_pools/main.tf index be3e670b..2cba335b 100644 --- a/examples/multiple_node_pools/main.tf +++ b/examples/multiple_node_pools/main.tf @@ -34,10 +34,10 @@ resource "azurerm_subnet" "test" { locals { nodes = { for i in range(3) : "worker${i}" => { - name = substr("worker${i}${random_id.prefix.hex}", 0, 8) - vm_size = "Standard_D2s_v3" - node_count = 1 - vnet_subnet_id = azurerm_subnet.test.id + name = substr("worker${i}${random_id.prefix.hex}", 0, 8) + vm_size = "Standard_D2s_v3" + node_count = 1 + vnet_subnet_id = azurerm_subnet.test.id create_before_destroy = i % 2 == 0 } }