diff --git a/addons/cni-canal/canal-config.yaml b/addons/cni-canal/canal-config.yaml index 7aa62dd87..c226714d8 100644 --- a/addons/cni-canal/canal-config.yaml +++ b/addons/cni-canal/canal-config.yaml @@ -53,10 +53,10 @@ data: {{ end }} }, "policy": { - "type": "k8s" + "type": "k8s" }, "kubernetes": { - "kubeconfig": "__KUBECONFIG_FILEPATH__" + "kubeconfig": "__KUBECONFIG_FILEPATH__" } }, { diff --git a/examples/terraform/gce-ipv6/README.md b/examples/terraform/gce-ipv6/README.md new file mode 100644 index 000000000..348f43a08 --- /dev/null +++ b/examples/terraform/gce-ipv6/README.md @@ -0,0 +1,95 @@ +# GCE Quickstart Terraform configs + +The GCE Quickstart Terraform configs can be used to create the needed +infrastructure for a Kubernetes HA cluster. Check out the following +[Creating Infrastructure guide][docs-infrastructure] to learn more about how to +use the configs and how to provision a Kubernetes cluster using KubeOne. + +[docs-infrastructure]: https://docs.kubermatic.com/kubeone/v1.9/guides/using-terraform-configs/ + +## GCE Provider configuration + +### Credentials + +Per +either of the following ENV variables should be accessible: + +* `GOOGLE_CREDENTIALS` +* `GOOGLE_CLOUD_KEYFILE_JSON` +* `GCLOUD_KEYFILE_JSON` + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [google](#requirement\_google) | ~> 6 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | ~> 6 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_address.lb_ip](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | +| [google_compute_firewall.common](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_firewall) | resource | +| [google_compute_firewall.control_plane](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_firewall) | resource | +| [google_compute_firewall.internal](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_firewall) | resource | +| [google_compute_firewall.nodeports](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_firewall) | resource | +| [google_compute_forwarding_rule.control_plane](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_forwarding_rule) | resource | +| [google_compute_http_health_check.control_plane](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_http_health_check) | resource | +| [google_compute_instance.control_plane](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance) | resource | +| [google_compute_network.network](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network) | resource | +| [google_compute_subnetwork.subnet](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_subnetwork) | resource | +| [google_compute_target_pool.control_plane_pool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_target_pool) | resource | +| [google_compute_image.control_plane_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | +| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [apiserver\_alternative\_names](#input\_apiserver\_alternative\_names) | subject alternative names for the API Server signing cert. | `list(string)` | `[]` | no | +| [bastion\_host\_key](#input\_bastion\_host\_key) | Bastion SSH host public key | `string` | `null` | no | +| [cluster\_autoscaler\_max\_replicas](#input\_cluster\_autoscaler\_max\_replicas) | maximum number of replicas per MachineDeployment (requires cluster-autoscaler) | `number` | `0` | no | +| [cluster\_autoscaler\_min\_replicas](#input\_cluster\_autoscaler\_min\_replicas) | minimum number of replicas per MachineDeployment (requires cluster-autoscaler) | `number` | `0` | no | +| [cluster\_name](#input\_cluster\_name) | Name of the cluster | `string` | n/a | yes | +| [control\_plane\_image\_family](#input\_control\_plane\_image\_family) | Image family to use for provisioning instances | `string` | `"ubuntu-2404-lts-amd64"` | no | +| [control\_plane\_image\_project](#input\_control\_plane\_image\_project) | Project of the image to use for provisioning instances | `string` | `"ubuntu-os-cloud"` | no | +| [control\_plane\_target\_pool\_members\_count](#input\_control\_plane\_target\_pool\_members\_count) | n/a | `number` | `3` | no | +| [control\_plane\_type](#input\_control\_plane\_type) | GCE instance type | `string` | `"n1-standard-2"` | no | +| [control\_plane\_vm\_count](#input\_control\_plane\_vm\_count) | number of control plane instances | `number` | `3` | no | +| [control\_plane\_volume\_size](#input\_control\_plane\_volume\_size) | Size of the boot volume, in GB | `number` | `100` | no | +| [disable\_kubeapi\_loadbalancer](#input\_disable\_kubeapi\_loadbalancer) | E2E tests specific variable to disable usage of any loadbalancer in front of kubeapi-server | `bool` | `false` | no | +| [enable\_ula\_internal\_ipv6](#input\_enable\_ula\_internal\_ipv6) | Enable ULA internal ipv6 on this network. Enabling this feature will assign a /48 from google defined ULA prefix fd20::/20 | `bool` | `false` | no | +| [initial\_machinedeployment\_operating\_system\_profile](#input\_initial\_machinedeployment\_operating\_system\_profile) | Name of operating system profile for MachineDeployments, only applicable if operating-system-manager addon is enabled.
If not specified, the default value will be added by machine-controller addon. | `string` | `""` | no | +| [initial\_machinedeployment\_replicas](#input\_initial\_machinedeployment\_replicas) | Number of replicas per MachineDeployment | `number` | `2` | no | +| [ip\_cidr\_range](#input\_ip\_cidr\_range) | The range of internal addresses that are owned by this subnetwork. Ranges must
be unique and non-overlapping within a network. Only IPv4 is supported.value | `string` | `"10.255.0.0/16"` | no | +| [ipv6\_access\_type](#input\_ipv6\_access\_type) | The access type of IPv6 address this subnet holds. It's immutable and can only be specified during
creation or the first time the subnet is updated into IPV4\_IPV6 dual stack. If the ipv6\_type is
EXTERNAL then this subnet cannot enable direct path. Possible values: ["EXTERNAL", "INTERNAL"] | `string` | `"EXTERNAL"` | no | +| [network\_tier](#input\_network\_tier) | The service-level to be provided for IPv6 traffic when the subnet has an external subnet.
Only PREMIUM or STANDARD tier is valid for IPv6. | `string` | `"PREMIUM"` | no | +| [project](#input\_project) | Project to be used for all resources | `string` | n/a | yes | +| [region](#input\_region) | GCP region to speak to | `string` | `"europe-west3"` | no | +| [ssh\_agent\_socket](#input\_ssh\_agent\_socket) | SSH Agent socket, default to grab from $SSH\_AUTH\_SOCK | `string` | `"env:SSH_AUTH_SOCK"` | no | +| [ssh\_hosts\_keys](#input\_ssh\_hosts\_keys) | A list of SSH hosts public keys to verify | `list(string)` | `null` | no | +| [ssh\_port](#input\_ssh\_port) | SSH port to be used to provision instances | `number` | `22` | no | +| [ssh\_private\_key\_file](#input\_ssh\_private\_key\_file) | SSH private key file used to access instances | `string` | `""` | no | +| [ssh\_public\_key\_file](#input\_ssh\_public\_key\_file) | SSH public key file | `string` | `"~/.ssh/id_rsa.pub"` | no | +| [ssh\_username](#input\_ssh\_username) | SSH user, used only in output | `string` | `"root"` | no | +| [worker\_os](#input\_worker\_os) | OS to run on worker machines | `string` | `"ubuntu"` | no | +| [workers\_type](#input\_workers\_type) | GCE instance type | `string` | `"n1-standard-2"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [kubeone\_api](#output\_kubeone\_api) | kube-apiserver LB endpoint | +| [kubeone\_hosts](#output\_kubeone\_hosts) | Control plane endpoints to SSH to | +| [kubeone\_workers](#output\_kubeone\_workers) | Workers definitions, that will be transformed into MachineDeployment object | diff --git a/examples/terraform/gce-ipv6/README.md.in b/examples/terraform/gce-ipv6/README.md.in new file mode 100644 index 000000000..7791ce640 --- /dev/null +++ b/examples/terraform/gce-ipv6/README.md.in @@ -0,0 +1,20 @@ +# GCE Quickstart Terraform configs + +The GCE Quickstart Terraform configs can be used to create the needed +infrastructure for a Kubernetes HA cluster. Check out the following +[Creating Infrastructure guide][docs-infrastructure] to learn more about how to +use the configs and how to provision a Kubernetes cluster using KubeOne. + +[docs-infrastructure]: https://docs.kubermatic.com/kubeone/v1.9/guides/using-terraform-configs/ + +## GCE Provider configuration + +### Credentials + +Per +either of the following ENV variables should be accessible: + +* `GOOGLE_CREDENTIALS` +* `GOOGLE_CLOUD_KEYFILE_JSON` +* `GCLOUD_KEYFILE_JSON` + diff --git a/examples/terraform/gce-ipv6/main.tf b/examples/terraform/gce-ipv6/main.tf new file mode 100644 index 000000000..48ec02c97 --- /dev/null +++ b/examples/terraform/gce-ipv6/main.tf @@ -0,0 +1,209 @@ +/* +Copyright 2019 The KubeOne Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +provider "google" { + region = var.region + project = var.project +} + +locals { + zones_count = length(data.google_compute_zones.available.names) + zone_first = data.google_compute_zones.available.names[0] + kubeapi_endpoint = var.disable_kubeapi_loadbalancer ? google_compute_instance.control_plane.0.network_interface.0.network_ip : google_compute_address.lb_ip.0.address + loadbalancer_count = var.disable_kubeapi_loadbalancer ? 0 : 1 + + cluster_autoscaler_min_replicas = var.cluster_autoscaler_min_replicas > 0 ? var.cluster_autoscaler_min_replicas : var.initial_machinedeployment_replicas + cluster_autoscaler_max_replicas = var.cluster_autoscaler_max_replicas > 0 ? var.cluster_autoscaler_max_replicas : var.initial_machinedeployment_replicas +} + +data "google_compute_zones" "available" { +} + +data "google_compute_image" "control_plane_image" { + family = var.control_plane_image_family + project = var.control_plane_image_project +} + +resource "google_compute_network" "network" { + name = var.cluster_name + auto_create_subnetworks = false + enable_ula_internal_ipv6 = var.enable_ula_internal_ipv6 +} + +resource "google_compute_subnetwork" "subnet" { + name = var.cluster_name + network = google_compute_network.network.self_link + ip_cidr_range = var.ip_cidr_range + region = var.region + stack_type = "IPV4_IPV6" + ipv6_access_type = var.ipv6_access_type +} + +resource "google_compute_firewall" "common" { + name = "${var.cluster_name}-common" + network = google_compute_network.network.self_link + + allow { + protocol = "tcp" + ports = [var.ssh_port] + } + + source_ranges = [ + "0.0.0.0/0", + ] +} + +resource "google_compute_firewall" "control_plane" { + name = "${var.cluster_name}-control-plane" + network = google_compute_network.network.self_link + + allow { + protocol = "tcp" + ports = ["6443"] + } + + source_ranges = [ + "0.0.0.0/0", + ] +} + +resource "google_compute_firewall" "internal" { + name = "${var.cluster_name}-internal" + network = google_compute_network.network.self_link + + allow { + protocol = "tcp" + ports = ["0-65535"] + } + + allow { + protocol = "udp" + ports = ["0-65535"] + } + + allow { + protocol = "icmp" + } + + source_ranges = [ + google_compute_subnetwork.subnet.ip_cidr_range, + ] +} + +resource "google_compute_firewall" "nodeports" { + name = "${var.cluster_name}-nodeports" + network = google_compute_network.network.self_link + + allow { + protocol = "tcp" + ports = ["30000-32767"] + } + + source_ranges = [ + "0.0.0.0/0", + ] +} + + +resource "google_compute_address" "lb_ip" { + count = local.loadbalancer_count + + name = "${var.cluster_name}-lb-ip" +} + +resource "google_compute_http_health_check" "control_plane" { + name = "${var.cluster_name}-control-plane-health" + + port = 10256 + request_path = "/healthz" + + timeout_sec = 3 + check_interval_sec = 5 +} + +resource "google_compute_target_pool" "control_plane_pool" { + name = "${var.cluster_name}-control-plane" + + instances = slice( + google_compute_instance.control_plane.*.self_link, + 0, + var.control_plane_target_pool_members_count, + ) + + health_checks = [ + google_compute_http_health_check.control_plane.self_link, + ] +} + +resource "google_compute_forwarding_rule" "control_plane" { + count = local.loadbalancer_count + + name = "${var.cluster_name}-apiserver" + target = google_compute_target_pool.control_plane_pool.self_link + port_range = "6443-6443" + ip_address = google_compute_address.lb_ip.0.address +} + +resource "google_compute_instance" "control_plane" { + count = var.control_plane_vm_count + + name = "${var.cluster_name}-control-plane-${count.index + 1}" + machine_type = var.control_plane_type + zone = data.google_compute_zones.available.names[count.index % local.zones_count] + + # Changing the machine_type, min_cpu_platform, or service_account on an + # instance requires stopping it. To acknowledge this, + # allow_stopping_for_update = true is required + allow_stopping_for_update = true + + boot_disk { + initialize_params { + size = var.control_plane_volume_size + image = data.google_compute_image.control_plane_image.self_link + } + } + + network_interface { + subnetwork = google_compute_subnetwork.subnet.self_link + + access_config { + nat_ip = "" + } + + ipv6_access_config { + network_tier = var.network_tier + } + + stack_type = "IPV4_IPV6" + } + + metadata = { + sshKeys = "${var.ssh_username}:${file(var.ssh_public_key_file)}" + } + + # https://cloud.google.com/sdk/gcloud/reference/alpha/compute/instances/set-scopes#--scopes + # listing of possible scopes + service_account { + scopes = [ + "compute-rw", + "logging-write", + "monitoring-write", + "service-control", + "service-management", + "storage-ro", + ] + } +} diff --git a/examples/terraform/gce-ipv6/output.tf b/examples/terraform/gce-ipv6/output.tf new file mode 100644 index 000000000..b26960a99 --- /dev/null +++ b/examples/terraform/gce-ipv6/output.tf @@ -0,0 +1,104 @@ +/* +Copyright 2019 The KubeOne Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +output "kubeone_api" { + description = "kube-apiserver LB endpoint" + + value = { + endpoint = local.kubeapi_endpoint + apiserver_alternative_names = var.apiserver_alternative_names + } +} + +output "kubeone_hosts" { + description = "Control plane endpoints to SSH to" + + value = { + control_plane = { + cluster_name = var.cluster_name + cloud_provider = "gce" + private_address = google_compute_instance.control_plane.*.network_interface.0.network_ip + public_address = google_compute_instance.control_plane.*.network_interface.0.access_config.0.nat_ip + ipv6_addresses = [for ip in google_compute_instance.control_plane.*.network_interface.0.ipv6_access_config.0.external_ipv6 : [ip]] + hostnames = google_compute_instance.control_plane.*.name + ssh_agent_socket = var.ssh_agent_socket + ssh_port = var.ssh_port + ssh_private_key_file = var.ssh_private_key_file + ssh_user = var.ssh_username + ssh_hosts_keys = var.ssh_hosts_keys + bastion_host_key = var.bastion_host_key + } + } +} + +output "kubeone_workers" { + description = "Workers definitions, that will be transformed into MachineDeployment object" + + value = { + # following outputs will be parsed by kubeone and automatically merged into + # corresponding (by name) worker definition + "${var.cluster_name}-pool1" = { + replicas = var.initial_machinedeployment_replicas + providerSpec = { + annotations = { + "k8c.io/operating-system-profile" = var.initial_machinedeployment_operating_system_profile + "cluster.k8s.io/cluster-api-autoscaler-node-group-min-size" = tostring(local.cluster_autoscaler_min_replicas) + "cluster.k8s.io/cluster-api-autoscaler-node-group-max-size" = tostring(local.cluster_autoscaler_max_replicas) + } + sshPublicKeys = [file(var.ssh_public_key_file)] + operatingSystem = var.worker_os + operatingSystemSpec = { + distUpgradeOnBoot = false + } + # nodeAnnotations are applied on resulting Node objects + # nodeAnnotations = { + # "key" = "value" + # } + # machineObjectAnnotations are applied on resulting Machine objects + # uncomment to following to set those kubelet parameters. More into at: + # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/ + # machineObjectAnnotations = { + # "v1.kubelet-config.machine-controller.kubermatic.io/SystemReserved" = "cpu=200m,memory=200Mi" + # "v1.kubelet-config.machine-controller.kubermatic.io/KubeReserved" = "cpu=200m,memory=300Mi" + # "v1.kubelet-config.machine-controller.kubermatic.io/EvictionHard" = "" + # "v1.kubelet-config.machine-controller.kubermatic.io/MaxPods" = "110" + # } + cloudProviderSpec = { + # provider specific fields: + # see example under `cloudProviderSpec` section at: + # https://github.com/kubermatic/machine-controller/blob/main/examples/gce-machinedeployment.yaml + diskSize = 50 + diskType = "pd-ssd" + machineType = var.workers_type + network = google_compute_network.network.self_link + subnetwork = google_compute_subnetwork.subnet.self_link + zone = "${local.zone_first}" + preemptible = false + assignPublicIPAddress = true + # Enable support for multizone clusters + multizone = true + labels = { + "${var.cluster_name}-workers" = "pool1" + } + tags = ["firewall", "targets", "${var.cluster_name}-pool1"] + regional = false + # Use custom image (optional) + # customImage = "" + } + } + } + } +} diff --git a/examples/terraform/gce-ipv6/variables.tf b/examples/terraform/gce-ipv6/variables.tf new file mode 100644 index 000000000..95634cf80 --- /dev/null +++ b/examples/terraform/gce-ipv6/variables.tf @@ -0,0 +1,203 @@ +/* +Copyright 2019 The KubeOne Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +variable "cluster_name" { + description = "Name of the cluster" + type = string + + validation { + condition = can(regex("^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$", var.cluster_name)) + error_message = "Value of cluster_name should be lowercase and can only contain alphanumeric characters and hyphens(-)." + } +} + +variable "apiserver_alternative_names" { + description = "subject alternative names for the API Server signing cert." + default = [] + type = list(string) +} + +variable "worker_os" { + description = "OS to run on worker machines" + + # valid choices are: + # * ubuntu + default = "ubuntu" + type = string +} + +variable "ssh_public_key_file" { + description = "SSH public key file" + default = "~/.ssh/id_rsa.pub" + type = string +} + +variable "ssh_port" { + description = "SSH port to be used to provision instances" + default = 22 + type = number +} + +variable "ssh_username" { + description = "SSH user, used only in output" + default = "root" + type = string +} + +variable "ssh_private_key_file" { + description = "SSH private key file used to access instances" + default = "" + type = string +} + +variable "ssh_agent_socket" { + description = "SSH Agent socket, default to grab from $SSH_AUTH_SOCK" + default = "env:SSH_AUTH_SOCK" + type = string +} + +variable "ssh_hosts_keys" { + default = null + description = "A list of SSH hosts public keys to verify" + type = list(string) +} + +variable "bastion_host_key" { + description = "Bastion SSH host public key" + default = null + type = string +} + +variable "disable_kubeapi_loadbalancer" { + type = bool + default = false + description = "E2E tests specific variable to disable usage of any loadbalancer in front of kubeapi-server" +} + +variable "control_plane_vm_count" { + description = "number of control plane instances" + default = 3 + type = number +} + +# Provider specific settings + +variable "project" { + description = "Project to be used for all resources" + type = string +} + +variable "region" { + default = "europe-west3" + description = "GCP region to speak to" + type = string +} + +variable "control_plane_target_pool_members_count" { + default = 3 + type = number +} + +variable "control_plane_type" { + default = "n1-standard-2" + description = "GCE instance type" + type = string +} + +variable "control_plane_volume_size" { + default = 100 + description = "Size of the boot volume, in GB" + type = number +} + +variable "control_plane_image_family" { + default = "ubuntu-2404-lts-amd64" + description = "Image family to use for provisioning instances" + type = string +} + +variable "control_plane_image_project" { + default = "ubuntu-os-cloud" + description = "Project of the image to use for provisioning instances" + type = string +} + +variable "workers_type" { + default = "n1-standard-2" + description = "GCE instance type" + type = string +} + +variable "initial_machinedeployment_replicas" { + description = "Number of replicas per MachineDeployment" + default = 2 + type = number +} + +variable "cluster_autoscaler_min_replicas" { + default = 0 + description = "minimum number of replicas per MachineDeployment (requires cluster-autoscaler)" + type = number +} + +variable "cluster_autoscaler_max_replicas" { + default = 0 + description = "maximum number of replicas per MachineDeployment (requires cluster-autoscaler)" + type = number +} + +variable "initial_machinedeployment_operating_system_profile" { + default = "" + type = string + description = <