From 73cef83436b42d5871554d3996e3e6807acf949f Mon Sep 17 00:00:00 2001 From: Phillip Stagnet Date: Mon, 27 May 2024 17:59:09 +0200 Subject: [PATCH 1/2] Remove unused cloud provider code --- .prow/e2e-features.yaml | 134 - .prow/postsubmits.yaml | 63 - .prow/provider-alibaba.yaml | 47 - .prow/provider-anexia.yaml | 49 - .prow/provider-aws.yaml | 270 - .prow/provider-azure.yaml | 111 - .prow/provider-digitalocean.yaml | 45 - .prow/provider-equinix-metal.yaml | 46 - .prow/provider-gcp.yaml | 80 - .prow/provider-hetzner.yaml | 44 - .prow/provider-kubevirt.yaml | 47 - .prow/provider-linode.yaml | 46 - .prow/provider-nutanix.yaml | 45 - .prow/provider-openstack.yaml | 80 - .prow/provider-scaleway.yaml | 45 - .prow/provider-vmware-cloud-director.yaml | 47 - .prow/provider-vsphere.yaml | 174 - .prow/verify.yaml | 179 - .wwhrd.yml | 3 - README.md | 2 +- docs/anexia.md | 25 - docs/cloud-provider.md | 272 -- docs/howto-provider.md | 4 +- docs/kubevirt.md | 40 - docs/nutanix.md | 17 - docs/operating-system.md | 9 - docs/rhel-custom-image.md | 30 - docs/vmware-cloud-director.md | 14 - docs/vsphere.md | 266 - examples/alibaba-machinedeployment.yaml | 65 - examples/anexia-machinedeployment.yaml | 57 - examples/azure-machinedeployment.yaml | 94 - examples/cdi-operator-cr.yaml | 7 - examples/cdi-operator.yaml | 2223 --------- examples/digitalocean-machinedeployment.yaml | 60 - examples/equinixmetal-machinedeployment.yaml | 54 - examples/gce-machinedeployment.yaml | 93 - examples/hetzner-machinedeployment.yaml | 70 - examples/kubevirt-cr.yaml | 8 - examples/kubevirt-local-mounter.yaml | 43 - examples/kubevirt-local-provisioner.yaml | 122 - examples/kubevirt-machinedeployment.yaml | 72 - examples/kubevirt-operator-0.19.0.yaml | 4333 ----------------- examples/linode-machinedeployment.yaml | 56 - examples/nutanix-machinedeployment.yaml | 85 - examples/opennebula-machinedeployment.yaml | 70 - examples/operating-system-manager.yaml | 30 - examples/scaleway-machinedeployment.yaml | 64 - ...ware-cloud-director-machinedeployment.yaml | 92 - ...e-datastore-cluster-machinedeployment.yaml | 82 - examples/vsphere-machinedeployment.yaml | 82 - examples/vultr-machinedeployment.yaml | 73 - hack/ci/run-e2e-tests.sh | 6 - .../machinedeployments_validation.go | 8 - pkg/admission/machines.go | 8 - pkg/admission/util.go | 51 - .../providerconfig_to_providerspec_test.go | 79 - .../hetzner.yaml | 44 - .../hetzner.yaml | 51 - .../hetzner.yaml | 22 - .../machinesv1alpha1machine/azure.yaml | 47 - .../machinesv1alpha1machine/digitalocean.yaml | 28 - .../machinesv1alpha1machine/hetzner.yaml | 25 - .../machinesv1alpha1machine/linode.yaml | 26 - .../vsphere-static-ip.yaml | 39 - .../machinesv1alpha1machine/vsphere.yaml | 39 - .../azure.yaml | 48 - .../digitalocean.yaml | 29 - .../hetzner.yaml | 27 - .../linode.yaml | 27 - .../vsphere-static-ip.yaml | 39 - .../vsphere.yaml | 35 - .../hetzner.yaml | 46 - .../hetzner.yaml | 53 - .../hetzner.yaml | 26 - pkg/cloudprovider/provider.go | 79 - .../provider/alibaba/provider.go | 486 -- .../provider/alibaba/types/types.go | 42 - .../provider/anexia/helper_test.go | 155 - pkg/cloudprovider/provider/anexia/instance.go | 110 - .../provider/anexia/instance_test.go | 127 - pkg/cloudprovider/provider/anexia/provider.go | 687 --- .../provider/anexia/provider_test.go | 510 -- .../provider/anexia/reconcile_context.go | 52 - .../provider/anexia/types/errors.go | 49 - .../provider/anexia/types/types.go | 89 - .../provider/azure/create_delete_resources.go | 403 -- .../provider/azure/get_client.go | 112 - pkg/cloudprovider/provider/azure/provider.go | 1336 ----- .../provider/azure/types/cloudconfig.go | 50 - .../provider/azure/types/types.go | 77 - .../provider/digitalocean/provider.go | 570 --- .../provider/digitalocean/types/types.go | 39 - pkg/cloudprovider/provider/edge/provider.go | 104 - .../provider/equinixmetal/provider.go | 572 --- .../provider/equinixmetal/types/types.go | 38 - pkg/cloudprovider/provider/gce/config.go | 325 -- pkg/cloudprovider/provider/gce/instance.go | 131 - pkg/cloudprovider/provider/gce/provider.go | 412 -- .../provider/gce/provider_test.go | 179 - pkg/cloudprovider/provider/gce/service.go | 169 - .../provider/gce/types/cloudconfig.go | 79 - .../provider/gce/types/cloudconfig_test.go | 73 - pkg/cloudprovider/provider/gce/types/types.go | 86 - .../provider/hetzner/provider.go | 646 --- .../provider/hetzner/types/types.go | 42 - .../provider/kubevirt/provider.go | 961 ---- .../provider/kubevirt/provider_test.go | 345 -- .../kubevirt/testdata/affinity-no-values.yaml | 83 - .../provider/kubevirt/testdata/affinity.yaml | 86 - .../kubevirt/testdata/custom-local-disk.yaml | 78 - .../kubevirt/testdata/http-image-source.yaml | 77 - .../instancetype-preference-custom.yaml | 76 - .../instancetype-preference-standard.yaml | 76 - .../kubevirt/testdata/nominal-case.yaml | 77 - .../kubevirt/testdata/pvc-image-source.yaml | 78 - .../testdata/registry-image-source-pod.yaml | 78 - .../testdata/registry-image-source.yaml | 78 - .../kubevirt/testdata/secondary-disks.yaml | 115 - .../testdata/topologyspreadconstraints.yaml | 83 - .../provider/kubevirt/types/cloudconfig.go | 36 - .../provider/kubevirt/types/types.go | 129 - pkg/cloudprovider/provider/linode/provider.go | 469 -- .../provider/linode/types/types.go | 37 - pkg/cloudprovider/provider/nutanix/client.go | 471 -- .../provider/nutanix/provider.go | 454 -- .../provider/nutanix/types/types.go | 75 - .../provider/opennebula/provider.go | 471 -- .../provider/opennebula/types/types.go | 46 - .../provider/scaleway/provider.go | 458 -- .../provider/scaleway/types/types.go | 38 - .../provider/vmwareclouddirector/client.go | 185 - .../provider/vmwareclouddirector/helper.go | 305 -- .../provider/vmwareclouddirector/provider.go | 585 --- .../vmwareclouddirector/types/types.go | 72 - pkg/cloudprovider/provider/vsphere/client.go | 125 - pkg/cloudprovider/provider/vsphere/helper.go | 515 -- .../provider/vsphere/helper_test.go | 242 - pkg/cloudprovider/provider/vsphere/network.go | 81 - .../provider/vsphere/provider.go | 695 --- .../provider/vsphere/provider_test.go | 188 - pkg/cloudprovider/provider/vsphere/rule.go | 188 - .../provider/vsphere/types/cloudconfig.go | 143 - .../vsphere/types/cloudconfig_test.go | 138 - .../types/testdata/2-virtual-centers.golden | 33 - .../types/testdata/3-dual-stack.golden | 29 - .../types/testdata/simple-config.golden | 21 - .../provider/vsphere/types/types.go | 66 - pkg/cloudprovider/provider/vultr/provider.go | 656 --- .../provider/vultr/types/types.go | 42 - pkg/controller/machine/controller.go | 61 +- pkg/controller/machine/controller_test.go | 42 - pkg/providerconfig/types/types.go | 46 +- pkg/userdata/amzn2/provider.go | 6 - pkg/userdata/amzn2/provider_test.go | 38 - .../kubelet-v1.29.2-vsphere-mirrors.yaml | 471 -- .../kubelet-v1.29.2-vsphere-proxy.yaml | 478 -- .../testdata/kubelet-v1.29.2-vsphere.yaml | 462 -- pkg/userdata/centos/provider.go | 52 - pkg/userdata/centos/provider_test.go | 48 - .../testdata/kubelet-v1.21-vsphere.yaml | 452 -- .../centos/testdata/kubelet-v1.27-aws.yaml | 1 + .../centos/testdata/kubelet-v1.28-aws.yaml | 1 + .../centos/testdata/kubelet-v1.29-aws.yaml | 1 + .../kubelet-v1.29.2-aws-external.yaml | 1 + .../centos/testdata/kubelet-v1.29.2-aws.yaml | 1 + .../testdata/kubelet-v1.29.2-nutanix.yaml | 468 -- .../kubelet-v1.29.2-vsphere-mirrors.yaml | 477 -- .../kubelet-v1.29.2-vsphere-proxy.yaml | 484 -- .../testdata/kubelet-v1.29.2-vsphere.yaml | 468 -- pkg/userdata/flatcar/provider.go | 6 - pkg/userdata/flatcar/provider_test.go | 60 +- .../flatcar/testdata/cloud-init_v1.28.5.yaml | 12 +- .../flatcar/testdata/cloud-init_v1.29.0.yaml | 12 +- .../flatcar/testdata/cloud-init_v1.29.2.yaml | 12 +- .../flatcar/testdata/ignition_v1.28.5.json | 2 +- .../flatcar/testdata/ignition_v1.29.0.json | 2 +- .../flatcar/testdata/ignition_v1.29.2.json | 2 +- pkg/userdata/rhel/provider_test.go | 68 - .../rhel/testdata/kubelet-v1.28-nutanix.yaml | 541 -- .../rhel/testdata/kubelet-v1.29-nutanix.yaml | 541 -- .../kubelet-v1.29.2-vsphere-mirrors.yaml | 550 --- .../kubelet-v1.29.2-vsphere-proxy.yaml | 557 --- .../testdata/kubelet-v1.29.2-vsphere.yaml | 541 -- .../rhel/testdata/pod-cidr-azure-rhel.yaml | 538 -- pkg/userdata/rockylinux/provider_test.go | 48 - .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 456 -- .../testdata/kubelet-v1.21-vsphere.yaml | 447 -- .../testdata/kubelet-v1.29.2-nutanix.yaml | 475 -- .../kubelet-v1.29.2-vsphere-mirrors.yaml | 484 -- .../kubelet-v1.29.2-vsphere-proxy.yaml | 491 -- .../testdata/kubelet-v1.29.2-vsphere.yaml | 475 -- pkg/userdata/ubuntu/provider_test.go | 166 - .../digitalocean-dualstack-IPv6+IPv4.yaml | 472 -- .../testdata/digitalocean-dualstack.yaml | 472 -- pkg/userdata/ubuntu/testdata/nutanix.yaml | 470 -- .../ubuntu/testdata/vsphere-mirrors.yaml | 478 -- .../ubuntu/testdata/vsphere-proxy.yaml | 485 -- pkg/userdata/ubuntu/testdata/vsphere.yaml | 468 -- test/e2e/provisioning/all_e2e_test.go | 535 +- test/e2e/provisioning/helper.go | 47 - .../testdata/machine-invalid.yaml | 4 +- .../testdata/machinedeployment-alibaba.yaml | 44 - .../testdata/machinedeployment-anexia.yaml | 42 - ...ployment-azure-custom-image-reference.yaml | 61 - ...hinedeployment-azure-redhat-satellite.yaml | 58 - .../testdata/machinedeployment-azure.yaml | 58 - .../machinedeployment-digitalocean.yaml | 44 - .../machinedeployment-equinixmetal.yaml | 38 - .../testdata/machinedeployment-gce.yaml | 56 - .../testdata/machinedeployment-hetzner.yaml | 42 - .../testdata/machinedeployment-kubevirt.yaml | 56 - .../testdata/machinedeployment-linode.yaml | 42 - .../testdata/machinedeployment-nutanix.yaml | 46 - .../machinedeployment-opennebula.yaml | 55 - .../testdata/machinedeployment-scaleway.yaml | 42 - ...chinedeployment-vmware-cloud-director.yaml | 60 - ...chinedeployment-vsphere-anti-affinity.yaml | 54 - ...edeployment-vsphere-datastore-cluster.yaml | 56 - ...achinedeployment-vsphere-multiple-nic.yaml | 57 - ...chinedeployment-vsphere-resource-pool.yaml | 48 - .../machinedeployment-vsphere-static-ip.yaml | 55 - .../testdata/machinedeployment-vsphere.yaml | 54 - .../testdata/machinedeployment-vultr.yaml | 38 - test/e2e/provisioning/verify.go | 12 +- 225 files changed, 65 insertions(+), 43113 deletions(-) delete mode 100644 .prow/e2e-features.yaml delete mode 100644 .prow/postsubmits.yaml delete mode 100644 .prow/provider-alibaba.yaml delete mode 100644 .prow/provider-anexia.yaml delete mode 100644 .prow/provider-aws.yaml delete mode 100644 .prow/provider-azure.yaml delete mode 100644 .prow/provider-digitalocean.yaml delete mode 100644 .prow/provider-equinix-metal.yaml delete mode 100644 .prow/provider-gcp.yaml delete mode 100644 .prow/provider-hetzner.yaml delete mode 100644 .prow/provider-kubevirt.yaml delete mode 100644 .prow/provider-linode.yaml delete mode 100644 .prow/provider-nutanix.yaml delete mode 100644 .prow/provider-openstack.yaml delete mode 100644 .prow/provider-scaleway.yaml delete mode 100644 .prow/provider-vmware-cloud-director.yaml delete mode 100644 .prow/provider-vsphere.yaml delete mode 100644 .prow/verify.yaml delete mode 100644 docs/anexia.md delete mode 100644 docs/kubevirt.md delete mode 100644 docs/nutanix.md delete mode 100644 docs/vmware-cloud-director.md delete mode 100644 docs/vsphere.md delete mode 100644 examples/alibaba-machinedeployment.yaml delete mode 100644 examples/anexia-machinedeployment.yaml delete mode 100644 examples/azure-machinedeployment.yaml delete mode 100644 examples/cdi-operator-cr.yaml delete mode 100644 examples/cdi-operator.yaml delete mode 100644 examples/digitalocean-machinedeployment.yaml delete mode 100644 examples/equinixmetal-machinedeployment.yaml delete mode 100644 examples/gce-machinedeployment.yaml delete mode 100644 examples/hetzner-machinedeployment.yaml delete mode 100644 examples/kubevirt-cr.yaml delete mode 100644 examples/kubevirt-local-mounter.yaml delete mode 100644 examples/kubevirt-local-provisioner.yaml delete mode 100644 examples/kubevirt-machinedeployment.yaml delete mode 100644 examples/kubevirt-operator-0.19.0.yaml delete mode 100644 examples/linode-machinedeployment.yaml delete mode 100644 examples/nutanix-machinedeployment.yaml delete mode 100644 examples/opennebula-machinedeployment.yaml delete mode 100644 examples/scaleway-machinedeployment.yaml delete mode 100644 examples/vmware-cloud-director-machinedeployment.yaml delete mode 100644 examples/vsphere-datastore-cluster-machinedeployment.yaml delete mode 100644 examples/vsphere-machinedeployment.yaml delete mode 100644 examples/vultr-machinedeployment.yaml delete mode 100644 pkg/admission/util.go delete mode 100644 pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineDeploymentWithProviderConfig/hetzner.yaml delete mode 100644 pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineSetWithProviderConfig/hetzner.yaml delete mode 100644 pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/hetzner.yaml delete mode 100644 pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/azure.yaml delete mode 100644 pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/digitalocean.yaml delete mode 100644 pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/hetzner.yaml delete mode 100644 pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/linode.yaml delete mode 100644 pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/vsphere-static-ip.yaml delete mode 100644 pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/vsphere.yaml delete mode 100644 pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/azure.yaml delete mode 100644 pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/digitalocean.yaml delete mode 100644 pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/hetzner.yaml delete mode 100644 pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/linode.yaml delete mode 100644 pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/vsphere-static-ip.yaml delete mode 100644 pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/vsphere.yaml delete mode 100644 pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineDeploymentWithProviderConfig/hetzner.yaml delete mode 100644 pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineSetWithProviderConfig/hetzner.yaml delete mode 100644 pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/hetzner.yaml delete mode 100644 pkg/cloudprovider/provider/alibaba/provider.go delete mode 100644 pkg/cloudprovider/provider/alibaba/types/types.go delete mode 100644 pkg/cloudprovider/provider/anexia/helper_test.go delete mode 100644 pkg/cloudprovider/provider/anexia/instance.go delete mode 100644 pkg/cloudprovider/provider/anexia/instance_test.go delete mode 100644 pkg/cloudprovider/provider/anexia/provider.go delete mode 100644 pkg/cloudprovider/provider/anexia/provider_test.go delete mode 100644 pkg/cloudprovider/provider/anexia/reconcile_context.go delete mode 100644 pkg/cloudprovider/provider/anexia/types/errors.go delete mode 100644 pkg/cloudprovider/provider/anexia/types/types.go delete mode 100644 pkg/cloudprovider/provider/azure/create_delete_resources.go delete mode 100644 pkg/cloudprovider/provider/azure/get_client.go delete mode 100644 pkg/cloudprovider/provider/azure/provider.go delete mode 100644 pkg/cloudprovider/provider/azure/types/cloudconfig.go delete mode 100644 pkg/cloudprovider/provider/azure/types/types.go delete mode 100644 pkg/cloudprovider/provider/digitalocean/provider.go delete mode 100644 pkg/cloudprovider/provider/digitalocean/types/types.go delete mode 100644 pkg/cloudprovider/provider/edge/provider.go delete mode 100644 pkg/cloudprovider/provider/equinixmetal/provider.go delete mode 100644 pkg/cloudprovider/provider/equinixmetal/types/types.go delete mode 100644 pkg/cloudprovider/provider/gce/config.go delete mode 100644 pkg/cloudprovider/provider/gce/instance.go delete mode 100644 pkg/cloudprovider/provider/gce/provider.go delete mode 100644 pkg/cloudprovider/provider/gce/provider_test.go delete mode 100644 pkg/cloudprovider/provider/gce/service.go delete mode 100644 pkg/cloudprovider/provider/gce/types/cloudconfig.go delete mode 100644 pkg/cloudprovider/provider/gce/types/cloudconfig_test.go delete mode 100644 pkg/cloudprovider/provider/gce/types/types.go delete mode 100644 pkg/cloudprovider/provider/hetzner/provider.go delete mode 100644 pkg/cloudprovider/provider/hetzner/types/types.go delete mode 100644 pkg/cloudprovider/provider/kubevirt/provider.go delete mode 100644 pkg/cloudprovider/provider/kubevirt/provider_test.go delete mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml delete mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml delete mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml delete mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml delete mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml delete mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml delete mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml delete mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml delete mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml delete mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml delete mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml delete mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml delete mode 100644 pkg/cloudprovider/provider/kubevirt/types/cloudconfig.go delete mode 100644 pkg/cloudprovider/provider/kubevirt/types/types.go delete mode 100644 pkg/cloudprovider/provider/linode/provider.go delete mode 100644 pkg/cloudprovider/provider/linode/types/types.go delete mode 100644 pkg/cloudprovider/provider/nutanix/client.go delete mode 100644 pkg/cloudprovider/provider/nutanix/provider.go delete mode 100644 pkg/cloudprovider/provider/nutanix/types/types.go delete mode 100644 pkg/cloudprovider/provider/opennebula/provider.go delete mode 100644 pkg/cloudprovider/provider/opennebula/types/types.go delete mode 100644 pkg/cloudprovider/provider/scaleway/provider.go delete mode 100644 pkg/cloudprovider/provider/scaleway/types/types.go delete mode 100644 pkg/cloudprovider/provider/vmwareclouddirector/client.go delete mode 100644 pkg/cloudprovider/provider/vmwareclouddirector/helper.go delete mode 100644 pkg/cloudprovider/provider/vmwareclouddirector/provider.go delete mode 100644 pkg/cloudprovider/provider/vmwareclouddirector/types/types.go delete mode 100644 pkg/cloudprovider/provider/vsphere/client.go delete mode 100644 pkg/cloudprovider/provider/vsphere/helper.go delete mode 100644 pkg/cloudprovider/provider/vsphere/helper_test.go delete mode 100644 pkg/cloudprovider/provider/vsphere/network.go delete mode 100644 pkg/cloudprovider/provider/vsphere/provider.go delete mode 100644 pkg/cloudprovider/provider/vsphere/provider_test.go delete mode 100644 pkg/cloudprovider/provider/vsphere/rule.go delete mode 100644 pkg/cloudprovider/provider/vsphere/types/cloudconfig.go delete mode 100644 pkg/cloudprovider/provider/vsphere/types/cloudconfig_test.go delete mode 100644 pkg/cloudprovider/provider/vsphere/types/testdata/2-virtual-centers.golden delete mode 100644 pkg/cloudprovider/provider/vsphere/types/testdata/3-dual-stack.golden delete mode 100644 pkg/cloudprovider/provider/vsphere/types/testdata/simple-config.golden delete mode 100644 pkg/cloudprovider/provider/vsphere/types/types.go delete mode 100644 pkg/cloudprovider/provider/vultr/provider.go delete mode 100644 pkg/cloudprovider/provider/vultr/types/types.go delete mode 100644 pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml delete mode 100644 pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-proxy.yaml delete mode 100644 pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere.yaml delete mode 100644 pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml delete mode 100644 pkg/userdata/centos/testdata/kubelet-v1.29.2-nutanix.yaml delete mode 100644 pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml delete mode 100644 pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-proxy.yaml delete mode 100644 pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere.yaml delete mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.28-nutanix.yaml delete mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.29-nutanix.yaml delete mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml delete mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere-proxy.yaml delete mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere.yaml delete mode 100644 pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml delete mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml delete mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml delete mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-nutanix.yaml delete mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml delete mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-proxy.yaml delete mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/nutanix.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/vsphere.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-alibaba.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-anexia.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-azure-custom-image-reference.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-azure.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-digitalocean.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-gce.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-hetzner.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-linode.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-opennebula.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-scaleway.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-vsphere-anti-affinity.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-vsphere-multiple-nic.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-vultr.yaml diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml deleted file mode 100644 index a550bf6fe..000000000 --- a/.prow/e2e-features.yaml +++ /dev/null @@ -1,134 +0,0 @@ -# Copyright 2022 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -presubmits: - - name: pull-machine-controller-e2e-invalid-objects-get-rejected - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-azure: "true" - preset-digitalocean: "true" - preset-gce: "true" - preset-e2e-ssh: "true" - preset-hetzner: "true" - preset-openstack: "true" - preset-vsphere: "true" - preset-kubevirt: "true" - preset-alibaba: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestInvalidObjectsGetRejected" - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - - name: pull-machine-controller-e2e-custom-ca - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-openstack: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestCustomCAsAreApplied" - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - - name: pull-machine-controller-e2e-ubuntu-upgrade - # In-tree CCM is not supported for openstack starting from k8s 1.26. Please see https://github.com/kubermatic/machine-controller/issues/1626 for updates. - # run_if_changed: "(pkg/cloudprovider/provider/openstack/|pkg/userdata)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-openstack: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestUbuntuProvisioningWithUpgradeE2E" - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - - name: pull-machine-controller-e2e-deployment-upgrade - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestDeploymentControllerUpgradesMachineE2E" - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml deleted file mode 100644 index dd380aa5a..000000000 --- a/.prow/postsubmits.yaml +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2022 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -postsubmits: - - name: ci-push-machine-controller-image - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - branches: - - ^main$ - # Match on tags - - ^v\d+\.\d+\.\d+.* - labels: - preset-docker-push: "true" - preset-goproxy: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - /bin/bash - - -c - - | - set -euo pipefail - start-docker.sh - docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - docker login -u $QUAY_IO_USERNAME -p $QUAY_IO_PASSWORD quay.io - make download-gocache docker-image-publish - # docker-in-docker needs privileged mode - securityContext: - privileged: true - resources: - requests: - cpu: 2 - memory: 1Gi - - - name: ci-push-machine-controller-upload-gocache - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - branches: - - ^main$ - labels: - preset-goproxy: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/upload-gocache.sh" - resources: - requests: - cpu: 2 - memory: 1Gi diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml deleted file mode 100644 index 7e6e8d709..000000000 --- a/.prow/provider-alibaba.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2022 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -presubmits: - - name: pull-machine-controller-e2e-alibaba - optional: true - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - max_concurrency: 1 - labels: - preset-alibaba: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestAlibabaProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: alibaba - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml deleted file mode 100644 index defbcca80..000000000 --- a/.prow/provider-anexia.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2022 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -presubmits: - - name: pull-machine-controller-e2e-anexia - # We've made the E2E tests for Anexia optional since it doesn't support k8s v1.26 at the moment. - # the tests on k8s v1.26+ will fail. - # TODO: These tests shouldn't be marked as optional. - optional: true - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-anexia: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestAnexiaProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: anexia - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml deleted file mode 100644 index 3db39d632..000000000 --- a/.prow/provider-aws.yaml +++ /dev/null @@ -1,270 +0,0 @@ -# Copyright 2022 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -presubmits: - - name: pull-machine-controller-e2e-aws - # In-tree CCM is not supported for AWS starting from k8s 1.27. Please see https://github.com/kubermatic/machine-controller/issues/1626 for updates. - # run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestAWSProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: aws - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - - name: pull-machine-controller-e2e-aws-spot-instance-legacy-userdata - # In-tree CCM is not supported for AWS starting from k8s 1.27. Please see https://github.com/kubermatic/machine-controller/issues/1626 for updates. - # run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - env: - - name: OPERATING_SYSTEM_MANAGER - value: "false" - - name: CLOUD_PROVIDER - value: aws - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestAWSSpotInstanceProvisioningE2E" - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - - name: pull-machine-controller-e2e-aws-arm - run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestAWSARMProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: aws - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - - name: pull-machine-controller-e2e-aws-ebs-encryption-enabled - run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestAWSEbsEncryptionEnabledProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: aws - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - - name: pull-machine-controller-e2e-aws-spot-instance - # In-tree CCM is not supported for AWS starting from k8s 1.27. Please see https://github.com/kubermatic/machine-controller/issues/1626 for updates. - # run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - preset-rhel: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestAWSSpotInstanceProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: aws - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - - name: pull-machine-controller-e2e-aws-flatcar-coreos-cloud-init - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestAWSFlatcarCoreOSCloudInit8ProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: aws - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - - name: pull-machine-controller-e2e-aws-centos8 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestAWSCentOS8ProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: aws - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - - name: pull-machine-controller-e2e-aws-assume-role - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws-assume-role: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestAWSAssumeRoleProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: aws - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml deleted file mode 100644 index 1cb8c860b..000000000 --- a/.prow/provider-azure.yaml +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright 2022 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -presubmits: - - name: pull-machine-controller-e2e-azure - run_if_changed: "(pkg/cloudprovider/provider/azure/|pkg/userdata)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-azure: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestAzureProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: azure - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - - name: pull-machine-controller-e2e-azure-custom-image-reference - run_if_changed: "(pkg/cloudprovider/provider/azure/|pkg/userdata)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-azure: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestAzureCustomImageReferenceProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: azure - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - - name: pull-machine-controller-e2e-azure-redhat-satellite - optional: true - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-azure: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestAzureProvisioningE2ERedhatSatellite" - env: - - name: CLOUD_PROVIDER - value: azure - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml deleted file mode 100644 index bed60b3bd..000000000 --- a/.prow/provider-digitalocean.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2022 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -presubmits: - - name: pull-machine-controller-e2e-digitalocean - run_if_changed: "(pkg/cloudprovider/provider/digitalocean/|pkg/userdata)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-digitalocean: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestDigitalOceanProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: digitalocean - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml deleted file mode 100644 index 059766bcc..000000000 --- a/.prow/provider-equinix-metal.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2022 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -presubmits: - - name: pull-machine-controller-e2e-equinix-metal - optional: true - run_if_changed: "(pkg/cloudprovider/provider/equinixmetal/|pkg/userdata)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-equinix-metal: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestEquinixMetalProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: metal - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml deleted file mode 100644 index 5f3721c08..000000000 --- a/.prow/provider-gcp.yaml +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2022 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -presubmits: - - name: pull-machine-controller-e2e-gce - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-gce: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestGCEProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: gce - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - - name: pull-machine-controller-e2e-gce-legacy-userdata - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-gce: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestGCEProvisioningE2E" - env: - - name: OPERATING_SYSTEM_MANAGER - value: "false" - - name: CLOUD_PROVIDER - value: gce - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml deleted file mode 100644 index b8655e8f5..000000000 --- a/.prow/provider-hetzner.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2022 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -presubmits: - - name: pull-machine-controller-e2e-hetzner - run_if_changed: "(pkg/cloudprovider/provider/hetzner/|pkg/userdata)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestHetznerProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: hetzner - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml deleted file mode 100644 index adb808ae5..000000000 --- a/.prow/provider-kubevirt.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2022 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -presubmits: - - name: pull-machine-controller-e2e-kubevirt - run_if_changed: "(pkg/cloudprovider/provider/kubevirt/|pkg/userdata)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - max_concurrency: 1 - labels: - preset-kubevirt: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestKubevirtProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: kubevirt - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml deleted file mode 100644 index 12a0711ed..000000000 --- a/.prow/provider-linode.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2022 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -presubmits: - - name: pull-machine-controller-e2e-linode - always_run: false - optional: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-linode: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestLinodeProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: linode - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml deleted file mode 100644 index 31e53cf54..000000000 --- a/.prow/provider-nutanix.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2022 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -presubmits: - - name: pull-machine-controller-e2e-nutanix - run_if_changed: "(pkg/cloudprovider/provider/nutanix/)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-nutanix: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestNutanixProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: nutanix - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml deleted file mode 100644 index 22ff68180..000000000 --- a/.prow/provider-openstack.yaml +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2022 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -presubmits: - - name: pull-machine-controller-e2e-openstack - # In-tree CCM is not supported for openstack starting from k8s 1.26. Please see https://github.com/kubermatic/machine-controller/issues/1626 for updates. - # run_if_changed: "(pkg/cloudprovider/provider/openstack/|pkg/userdata)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-openstack: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestOpenstackProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: openstack - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - - name: pull-machine-controller-e2e-openstack-project-auth - # In-tree CCM is not supported for openstack starting from k8s 1.26. Please see https://github.com/kubermatic/machine-controller/issues/1626 for updates. - # run_if_changed: "(pkg/cloudprovider/provider/openstack/|pkg/userdata)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-openstack: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestOpenstackProjectAuthProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: openstack - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml deleted file mode 100644 index 0fa091d99..000000000 --- a/.prow/provider-scaleway.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2022 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -presubmits: - - name: pull-machine-controller-e2e-scaleway - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-scaleway: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestScalewayProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: scaleway - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml deleted file mode 100644 index 8be8ec733..000000000 --- a/.prow/provider-vmware-cloud-director.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2022 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -presubmits: - - name: pull-machine-controller-e2e-vmware-cloud-director - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - run_if_changed: "(pkg/cloudprovider/provider/vmwareclouddirector/)" - labels: - preset-vcloud-director: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestVMwareCloudDirectorProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: vcd - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml deleted file mode 100644 index a75d1f864..000000000 --- a/.prow/provider-vsphere.yaml +++ /dev/null @@ -1,174 +0,0 @@ -# Copyright 2022 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -presubmits: - - name: pull-machine-controller-e2e-vsphere - run_if_changed: "(pkg/cloudprovider/provider/vsphere/|pkg/userdata)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-vsphere: "true" - preset-rhel: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestVsphereProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: vsphere - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - - name: pull-machine-controller-e2e-vsphere-datastore-cluster - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-vsphere: "true" - preset-rhel: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestVsphereDatastoreClusterProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: vsphere - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - - name: pull-machine-controller-e2e-vsphere-resource-pool - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-vsphere: "true" - preset-rhel: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestVsphereResourcePoolProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: vsphere - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - - name: pull-machine-controller-e2e-vsphere-multiple-networks - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-vsphere: "true" - preset-rhel: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestVsphereMultipleNICProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: vsphere - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - - name: pull-machine-controller-e2e-vsphere-anti-affinity - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-vsphere: "true" - preset-rhel: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestVsphereAntiAffinityProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: vsphere - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi diff --git a/.prow/verify.yaml b/.prow/verify.yaml deleted file mode 100644 index 0ba98100b..000000000 --- a/.prow/verify.yaml +++ /dev/null @@ -1,179 +0,0 @@ -# Copyright 2022 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -presubmits: - - name: pull-machine-controller-build - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-goproxy: "true" - spec: - containers: - - image: golang:1.22.1 - command: - - make - args: - - download-gocache - - all - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - - name: pull-machine-controller-dependencies - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-goproxy: "true" - spec: - containers: - - image: golang:1.22.1 - command: - - make - args: - - check-dependencies - resources: - requests: - memory: 32Mi - cpu: 50m - limits: - memory: 256Mi - cpu: 250m - - - name: pull-machine-controller-lint - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-goproxy: "true" - spec: - containers: - - image: golangci/golangci-lint:v1.56.0 - command: - - make - args: - - lint - resources: - requests: - cpu: 800m - memory: 7Gi - limits: - memory: 7Gi - - - name: pull-machine-controller-yamllint - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-goproxy: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-3 - command: - - make - args: - - yamllint - resources: - requests: - memory: 32Mi - cpu: 50m - limits: - memory: 256Mi - cpu: 250m - - - name: pre-machine-controller-verify-shfmt - run_if_changed: "^hack/" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - spec: - containers: - - image: docker.io/mvdan/shfmt:v3.3.1 - command: - - "/bin/shfmt" - args: - # -l list files whose formatting differs from shfmt's - # -d error with a diff when the formatting differs - # -i uint indent: 0 for tabs (default), >0 for number of spaces - # -sr redirect operators will be followed by a space - - "-l" - - "-sr" - - "-i" - - "2" - - "-d" - - "hack" - resources: - requests: - memory: 32Mi - cpu: 50m - limits: - memory: 256Mi - cpu: 250m - - - name: pull-machine-controller-verify-boilerplate - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - spec: - containers: - - image: quay.io/kubermatic-labs/boilerplate:v0.2.0 - command: - - "./hack/verify-boilerplate.sh" - resources: - requests: - memory: 32Mi - cpu: 50m - limits: - memory: 256Mi - cpu: 250m - - - name: pull-machine-controller-license-validation - run_if_changed: "^go.(mod|sum)$" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-goproxy: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 - command: - - ./hack/verify-licenses.sh - resources: - requests: - memory: 2Gi - cpu: 2 - - - name: pull-machine-controller-test - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-goproxy: "true" - spec: - containers: - - image: golang:1.22.1 - command: - - make - args: - - download-gocache - - test-unit - resources: - requests: - cpu: 3 - memory: 7Gi - limits: - memory: 7Gi diff --git a/.wwhrd.yml b/.wwhrd.yml index 677ba2b2f..3e22417c3 100644 --- a/.wwhrd.yml +++ b/.wwhrd.yml @@ -27,9 +27,6 @@ allowlist: exceptions: - github.com/hashicorp/golang-lru # MPL-2.0 - github.com/hashicorp/golang-lru/simplelru # MPL-2.0 - - github.com/embik/nutanix-client-go/pkg/client # MPL-2.0 - - github.com/embik/nutanix-client-go/pkg/client/v3 # MPL-2.0 - - github.com/embik/nutanix-client-go/internal/utils # MPL-2.0 - github.com/ajeddeloh/go-json # Since it's a fork, https://github.com/golang/go/blob/master/LICENSE - github.com/hashicorp/go-version # MPL-2.0 - github.com/hashicorp/go-cleanhttp # MPL-2.0 diff --git a/README.md b/README.md index f0e41d67f..409a6455e 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ ### What Works -- Creation of worker nodes on AWS, Digitalocean, Openstack, Azure, Google Cloud Platform, Nutanix, VMWare Cloud Director, VMWare vSphere, Hetzner Cloud and Kubevirt +- Creation of worker nodes on AWS and Openstack - Using Ubuntu, Flatcar, CentOS 7 or Rocky Linux 8 distributions ([not all distributions work on all providers](/docs/operating-system.md)) ### Supported Kubernetes Versions diff --git a/docs/anexia.md b/docs/anexia.md deleted file mode 100644 index cf826f8fa..000000000 --- a/docs/anexia.md +++ /dev/null @@ -1,25 +0,0 @@ -# Anexia Engine - -This provider implementation is currently in **alpha** state. - -## Supported Operating Systems - -Only flatcar linux is currently supported and you explicitly have to set the provisioning mechanism to cloud-init by setting `machine.spec.providerSpec.value.operatingSystemSpec.provisioningUtility` to "cloud-init". - -An example machine deployment can be found here: [examples/anexia-machinedeployment.yaml](../examples/anexia-machinedeployment.yaml) - -## Templates - -You can configure the template to use by its name (using the attribute `template`) or its identifier (using the attribute `templateID`). - -When specifying the template by its name, the template build to use can optionally be set (attribute `templateBuild`). Omitting `templateBuild` will yield the latest available build (at time the time of creating the `Machine`) for the specified named template. - -Template identifiers (attribute `templateID`) always link to a given `template`-`templateBuild` combination, so using the identifier in configuration has the same drawback as specifying an exact build to use. - -Templates are rotated pretty often to include security patches and other updates. Outdated versions of templates are not retained and get removed after some time. Because of this, we do not recommend using the `templateID` attribute or pinning to a fixed build unless really required. - -To retrieve all available templates against a given location: - -``` -https://engine.anexia-it.com/api/vsphere/v1/provisioning/templates.json//templates?page=1&limit=50&api_key= -``` diff --git a/docs/cloud-provider.md b/docs/cloud-provider.md index 74edb7a4a..8e4c3162d 100644 --- a/docs/cloud-provider.md +++ b/docs/cloud-provider.md @@ -1,49 +1,5 @@ # Cloud providers -## Scaleway - -### machine.spec.providerConfig.cloudProviderSpec -```yaml -# your scaleway access key -accessKey: "<< SCW_ACCESS_KEY >>" -# your scaleway secret key -secretKey: "<< SCW_SECRET_KEY >>" -# your scaleway project ID -projectId: "<< SCW_DEFAULT_PROJECT_ID >>" -# server zone -zone: "fr-par-1" -# server commercial type -commercialType: "DEV1-M" -# enable ipv6 for the server -ipv6: false -# add the following tags to the server -tags: -- "machine-controller" -``` - -## Digitalocean - -### machine.spec.providerConfig.cloudProviderSpec -```yaml -# your digitalocean token -token: "<< YOUR_DO_TOKEN >>" -# droplet region -region: "fra1" -# droplet size -size: "2gb" -# enable backups for the droplet -backups: false -# enable ipv6 for the droplet -ipv6: false- Add operating system config -# enable private networking for the droplet -private_networking: true -# enable monitoring for the droplet -monitoring: true -# add the following tags to the droplet -tags: -- "machine-controller" -``` - ## AWS ### machine.spec.providerConfig.cloudProviderSpec @@ -145,231 +101,3 @@ nodeVolumeAttachLimit: 20 tags: tagKey: tagValue ``` - -## OpenNebula - -**Note:** This is a [community provider](../README.md#community-providers). - -### machine.spec.providerConfig.cloudProviderSpec - -```yaml -# XML-RPC endpoint of your OpenNebula installation -endpoint: "" -# your OpenNebula username -username: "" -# your OpenNebula password -password: "" - -# cpu (float64) -cpu: 1 -# vcpu -vcpu: 2 -# memory in MB -memory: 1024 - -# the name of the image to use, needs to be owned by the current user -image: "Amazon Linux 2" -# which datastore to use for the image -datastore: "" -# size of the disk in MB -diskSize: 51200 - -# network name, needs to be owned by the current user -network: "" - -# whether to enable the VNC console -enableVNC: true - -# optional key/value pairs to add to the VM template -vmTemplateExtra: - # useful for e.g. setting the placement attributes as defined in https://docs.opennebula.io/6.4/management_and_operations/references/template.html#template-placement-section - SCHED_REQUIREMENTS: 'RACK="G4"' -``` - -## Google Cloud Platform - -### machine.spec.providerConfig.cloudProviderSpec - -```yaml -# The service account needs to be base64-encoded. -serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>" -# See https://cloud.google.com/compute/docs/regions-zones/ -zone: "europe-west3-a" -# See https://cloud.google.com/compute/docs/machine-types -machineType: "n1-standard-2" -# See https://cloud.google.com/compute/docs/instances/preemptible -preemptible: false -# In GB -diskSize: 25 -# Can be 'pd-standard' or 'pd-ssd' -diskType: "pd-standard" -# The name or self_link of the network and subnetwork to attach this interface to; -# either of both can be provided, otherwise default network will taken -# in case if both empty — default network will be used -network: "my-cool-network" -subnetwork: "my-cool-subnetwork" -# assign a public IP Address. Required for Internet access -assignPublicIPAddress: true -# if true, does not inject the Service Account from the controller in the machine, leaving it empty -disableMachineServiceAccount: false -# set node labels -labels: - "kubernetesCluster": "my-cluster" -``` - -## Hetzner cloud - -### machine.spec.providerConfig.cloudProviderSpec -```yaml -token: "<< HETZNER_API_TOKEN >>" -serverType: "cx11" -datacenter: "" -location: "fsn1" -# Optional: network IDs or names -networks: - - "<< YOUR_NETWORK >>" -# set node labels -labels: - "kubernetesCluster": "my-cluster" -``` - -## Linode - -**Note:** This is a [community provider](../README.md#community-providers). - -### machine.spec.providerConfig.cloudProviderSpec -```yaml -# your linode token -token: "<< YOUR_LINODE_TOKEN >>" -# linode region -region: "eu-west" -# linode size -type: "g6-standard-2" -# enable backups for the linode -backups: false -# enable private networking for the linode -private_networking: true -# add the following tags to the linode -tags: -- "machine-controller" -``` - -## Alibaba - -### machine.spec.providerConfig.cloudProviderSpec -```yaml -# If empty, can be set via ALIBABA_ACCESS_KEY_ID env var -accessKeyID: "<< YOUR ACCESS ID >>" -accessKeySecret: "<< YOUR ACCESS SECRET >>" -# instance type -instanceType: "ecs.t1.xsmall" -# instance name -instanceName: "alibaba-instance" -# region -regionID: eu-central-1 -# image id -imageID: "aliyun_2_1903_64_20G_alibase_20190829.vhd" -# disk type -diskType: "cloud_efficiency" -# disk size in GB -diskSize: "40" -# set an existing vSwitch ID to use, VPC default is used if not set. -vSwitchID: -labels: - "kubernetesCluster": "my-cluster" -``` - - -## Azure - -### machine.spec.providerConfig.cloudProviderSpec -```yaml -# Can also be set via the env var 'AZURE_TENANT_ID' on the machine-controller -tenantID: "<< AZURE_TENANT_ID >>" -# Can also be set via the env var 'AZURE_CLIENT_ID' on the machine-controller -clientID: "<< AZURE_CLIENT_ID >>" -# Can also be set via the env var 'AZURE_CLIENT_SECRET' on the machine-controller -clientSecret: "<< AZURE_CLIENT_SECRET >>" -# Can also be set via the env var 'AZURE_SUBSCRIPTION_ID' on the machine-controller -subscriptionID: "<< AZURE_SUBSCRIPTION_ID >>" -# Azure location -location: "westeurope" -# Azure resource group -resourceGroup: "<< YOUR_RESOURCE_GROUP >>" -# Azure resource group of the vnet -vnetResourceGroup: "<< YOUR_VNET_RESOURCE_GROUP >>" -# Azure availability set -availabilitySet: "<< YOUR AVAILABILITY SET >>" -# VM size -vmSize: "Standard_B1ms" -# optional OS and Data disk size values in GB. If not set, the defaults for the vmSize will be used. -osDiskSize: 30 -dataDiskSize: 30 -# network name -vnetName: "<< VNET_NAME >>" -# subnet name -subnetName: "<< SUBNET_NAME >>" -# route able name -routeTableName: "<< ROUTE_TABLE_NAME >>" -# assign public IP addresses for nodes, required for Internet access -assignPublicIP: true -# security group -securityGroupName: my-security-group -# node tags -tags: - "kubernetesCluster": "my-cluster" -``` - -## Equinix Metal - -### machine.spec.providerConfig.cloudProviderSpec -```yaml -# If empty, can be set via METAL_AUTH_TOKEN env var -token: "<< METAL_AUTH_TOKEN >>" -# instance type -instanceType: "t1.small.x86" -# Equinix Metal project ID -projectID: "<< PROJECT_ID >>" -# Equinix Metal facilities -facilities: - - "ewr1" -# Equinix Metal billingCycle -billingCycle: "" -# node tags -tags: - "kubernetesCluster": "my-cluster" -``` - -## KubeVirt - -### machine.spec.providerConfig.cloudProviderSpec -```yaml -# base64-encoded kubeconfig to access KubeVirt cluster -kubeconfig: '<< KUBECONFIG_BASE64 >>' -# KubeVirt namespace -namespace: kube-system -# kubernetes storage class -storageClassName: kubermatic-fast -# storage PVC size -pvcSize: "10Gi" -# OS Image URL -sourceURL: http://10.109.79.210/<< OS_NAME >>.img -# instance resources -cpus: "1" -memory: "2048M" -``` - -## vSphere - -Refer to the [VSphere](./vsphere.md#provider-configuration) specific documentation. - -## Vultr - -**Note:** This is a [community provider](../README.md#community-providers). - -### machine.spec.providerConfig.cloudProviderSpec -```yaml -apiKey: "<< VULTR_API_KEY >>" -plan: "vhf-8c-32gb" -region: "" -osId: 127 diff --git a/docs/howto-provider.md b/docs/howto-provider.md index f797c4260..2b139d536 100644 --- a/docs/howto-provider.md +++ b/docs/howto-provider.md @@ -66,7 +66,7 @@ SetMetricsForMachines(machines v1alpha1.MachineList) error ### Implementation hints -Provider implementations are located in individual packages in `github.com/kubermatic/machine-controller/pkg/cloudprovider/provider`. Here see e.g. `hetzner` as a straight and good understandable implementation. Other implementations are there too, helping to understand the needed tasks inside and around the `Provider` interface implementation. +Provider implementations are located in individual packages in `github.com/kubermatic/machine-controller/pkg/cloudprovider/provider`. Here see e.g. `openstack` as a straight and good understandable implementation. Other implementations are there too, helping to understand the needed tasks inside and around the `Provider` interface implementation. When retrieving the individual configuration from the provider specification a type for unmarshalling is needed. Here first the provider configuration is read and based on it the individual values of the configuration are retrieved. Typically the access data (token, ID/key combination, document with all information) alternatively can be passed via an environment variable. According methods of the used `providerconfig.ConfigVarResolver` do support this. @@ -90,6 +90,6 @@ Now the provider is ready to be added into the project for CI tests. ## References - [Cloud Provider Interface](https://github.com/kubermatic/machine-controller/blob/main/pkg/cloudprovider/cloud/provider.go) -- [Implementation for Hetzner](https://github.com/kubermatic/machine-controller/blob/main/pkg/cloudprovider/provider/hetzner/provider.go) +- [Implementation for Openstack](https://github.com/kubermatic/machine-controller/blob/main/pkg/cloudprovider/provider/openstack/provider.go) - [Cloud Provider Type Definition](https://github.com/kubermatic/machine-controller/blob/main/pkg/providerconfig/types.go) - [Registration of supported Cloud Providers](https://github.com/kubermatic/machine-controller/blob/main/pkg/cloudprovider/provider.go) diff --git a/docs/kubevirt.md b/docs/kubevirt.md deleted file mode 100644 index 819d84140..000000000 --- a/docs/kubevirt.md +++ /dev/null @@ -1,40 +0,0 @@ -# Kubevirt - -In order to use the machine-controller to create machines using [Kubevirt](https://kubevirt.io) -you must first install the latter. We provide a manifest for this, simply run `kubectl apply -f examples/kubevirt-operator-0.19.0.yaml`. -We strongly recommend installing a version which is equal or higher than `0.19.0`. Machine Controller also uses the KubeVirt CDI which can be found -under `examples/cdi-operator.yaml` to provision storage. It is important to have a basic understanding of Kubernetes storage. For more -information regarding which types of storage can be used please refer to [KubeVirt documentation](https://github.com/kubevirt/containerized-data-importer/blob/main/doc/basic_pv_pvc_dv.md). - - -Afterwards, you can use the provided `examples/kubevirt-machinedeployment.yaml` as base. There -are some things you need to keep in mind: - -* The machine-controller will create `VMIs` that have the same name as the underlying `machine`. To -avoid collisions, use one namespace per cluster that runs the `machine-controller` -* EvictionStratey of `VMIs` is set to external, so VMI eviction needs to handled properly by a custom external controller or manual action -* Service CIDR range: The CIDR ranges of the cluster that runs Kubevirt and the cluster that hosts the machine-controller must not overlap, -otherwise routing of services that run in the kubevirt cluster won't work anymore. This is especially important for the DNS ClusterIP. -* `clusterName` is used to [label VMs](https://github.com/kubevirt/cloud-provider-kubevirt#prerequisites) for LoadBalancer selection - -## Serving Supported Images - -For KubeVirt clusters, we use Containerized Data Importer (CDI), which is is a utility to import, upload and clone -Virtual Machine images for use with KubeVirt. At a high level, a persistent volume claim (PVC), which defines VM-suitable -storage via a storage class, is created. - -The Containerized Data Importer is capable of performing certain functions that streamline its use with KubeVirt. It automatically -decompresses gzip and xz files, and un-tar’s tar archives. Also, qcow2 images are converted into the raw format which is required by KubeVirt, -resulting in the final file being a simple .img file. - -Supported file formats are: - -- Tar archive -- Gzip compressed file -- XZ compressed file -- Raw image data -- ISO image data -- Qemu qcow2 image data - -KubeVirt reads those images from an http endpoint which is passed to the `MachineDeployment` spec. The field that should be used -for to import those images is `sourceURL`. diff --git a/docs/nutanix.md b/docs/nutanix.md deleted file mode 100644 index b3c87432d..000000000 --- a/docs/nutanix.md +++ /dev/null @@ -1,17 +0,0 @@ -# Nutanix Prism Central - -Currently the `machine-controller` implementation of Nutanix supports the [Prism v3 API](https://www.nutanix.dev/reference/prism_central/v3/) to create `Machines`. - -## Prerequisites - -The `nutanix` provider assumes several things to be preexisting. You need: - -- Credentials and access information for a Nutanix Prism Central instance (endpoint, port, username and password). -- The name of a Nutanix cluster to create the VMs for Machines on. -- The name of a subnet on the given Nutanix cluster that the VMs' network interfaces will be created on. -- An image name that will be used to create the VM for (must match the configured operating system). -- **Optional**: The name of a project that the given credentials have access to, to create the VMs in. If none is provided, the VMs are created without a project. - -## Configuration Options - -An example `MachineDeployment` can be found [here](../examples/nutanix-machinedeployment.yaml). diff --git a/docs/operating-system.md b/docs/operating-system.md index fd6994ece..fd99ac400 100644 --- a/docs/operating-system.md +++ b/docs/operating-system.md @@ -7,16 +7,7 @@ | | Ubuntu | CentOS | Flatcar | RHEL | Amazon Linux 2 | Rocky Linux | |---|---|---|---|---|---|---| | AWS | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Azure | ✓ | ✓ | ✓ | ✓ | x | ✓ | -| Digitalocean | ✓ | ✓ | x | x | x | ✓ | -| Equinix Metal | ✓ | ✓ | ✓ | x | x | ✓ | -| Google Cloud Platform | ✓ | x | ✓ | x | x | x | -| Hetzner | ✓ | x | x | x | x | ✓ | -| KubeVirt | ✓ | ✓ | ✓ | ✓ | x | ✓ | -| Nutanix | ✓ | ✓ | x | x | x | x | | Openstack | ✓ | ✓ | ✓ | ✓ | x | ✓ | -| VMware Cloud Director | ✓ | x | x | x | x | x | -| VSphere | ✓ | ✓ | ✓ | ✓ | x | ✓ | ## Configuring a operating system diff --git a/docs/rhel-custom-image.md b/docs/rhel-custom-image.md index 81d9efb68..86cbcc9d7 100644 --- a/docs/rhel-custom-image.md +++ b/docs/rhel-custom-image.md @@ -3,11 +3,7 @@ Cloud providers which are listed below, support using RHEL as an operating system option: - AWS -- Azure -- GCE -- KubeVirt - Openstack -- vSphere #### AWS First of all the RHEL gold image AMIs have to be enabled from the [RedHat Customer Portal](https://access.redhat.com/public-cloud/aws) (this requires a [cloud-provider subscription](https://access.redhat.com/public-cloud)). @@ -15,32 +11,6 @@ First of all the RHEL gold image AMIs have to be enabled from the [RedHat Custom Afterwards, new images will be added to the aws account under EC2-> Images-> AMIs-> Private Images. Once the images are available in the aws account, the image id for RHEL (supported versions are mentioned [here](./operating-system.md#supported-os-versions)) should be then added to the `MachineDeployment` spec to the field `ami`. -#### Azure -RedHat provides images for Azure, [documentation](https://access.redhat.com/articles/uploading-rhel-image-to-azure) is available on RH customer portal. - -The `MachineDeployment` field `.spec.template.spec.providerSpec.value.cloudProviderSpec.imageID` should reference the ID of the uploaded VM. - -**Note:** -Azure RHEL images starting from 7.6.x don't support cloud-init as their documentation states [here](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/using-cloud-init#rhel). -Thus, custom images can be used with a cloud-init pre-installed to solve this issue. Follow this [documentation](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/cloudinit-prepare-custom-image) -to prepare an image with cloud-init support. - -#### GCE -RedHat also provides Gold Access Image for GCE and those can be fetched just like aws and azure. The `MachineDeployment` field `.spec.template.spec.providerSpec.value.cloudProviderSpec.customImage` should reference the ID of the used image. - -**Note:** -RHEL images in GCE don't support cloud-init. Thus, custom images can be used with a cloud-init pre-installed -to solve this issue. Follow this [documentation](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/8/html/deploying_red_hat_enterprise_linux_8_on_public_cloud_platforms/assembly_deploying-a-rhel-image-as-a-compute-engine-instance-on-google-cloud-platform_deploying-a-virtual-machine-on-aws) to upload custom RHEL images. - -#### KubeVirt -In order to create machines which run RHEL as an operating system in KubeVirt cloud provider, the image should be available and fetched -via an endpoint. This endpoint should be then added to the `MachineDeployment` field `.spec.template.spec.providerSpec.value.cloudProviderSpec.sourceURL`. For more information about -the supported images please refer to this documentation from KubeVirt CDI [here](https://kubevirt.io/2018/containerized-data-importer.html) - #### Openstack Once RHEL images(e.g: Red Hat Enterprise Linux 8.x KVM Guest Image) is uploaded to openstack, the image name should be used in the `MachineDeployment` field `.spec.template.spec.providerSpec.value.cloudProviderSpec.image`. - -#### vSphere - -Find [here](./vsphere.md#RHEL) how to deploy a template VM in vSphere. diff --git a/docs/vmware-cloud-director.md b/docs/vmware-cloud-director.md deleted file mode 100644 index c38cf06cc..000000000 --- a/docs/vmware-cloud-director.md +++ /dev/null @@ -1,14 +0,0 @@ -# VMware Cloud Director - -## Prerequisites - -The following things should be configured before managing machines on VMware Cloud Director: - -- Dedicated Organization VDC has been created. -- Required catalog and templates for creating VMs have been added to the organization VDC. -- VApp has been created that will be used to encapsulate all the VMs. -- Direct, routed or isolated network has been created. And the virtual machines within the vApp can communicate over that network. - -## Configuration Options - -An example `MachineDeployment` can be found [here](../examples/vmware-cloud-director-machinedeployment.yaml). diff --git a/docs/vsphere.md b/docs/vsphere.md deleted file mode 100644 index ab6962c94..000000000 --- a/docs/vsphere.md +++ /dev/null @@ -1,266 +0,0 @@ -# VMware vSphere - -## Supported versions - -* 6.5 -* 6.7 - -## Template VMs preparation - -To use the machine-controller to create machines on VMWare vsphere, you must first -create a VM to be used as a template. - -*Note that:* -`template VMs` in this document refers to regular VMs and not -[VM Templates][vm_templates] according to vSphere terminology. -The difference is quite subtle, but VM Templates are not supported yet by -`machine controller`. - -### Create template VM from OVA - -To see where to locate the OVAs go to the OS specific section. - -#### WebUI procedure - -1. Go into the vSphere WebUI, select your datacenter, right click onto it and choose "Deploy OVF Template" -2. Fill in the "URL" field with the appropriate url pointing to the `OVA` file -3. Click through the dialog until "Select storage" -4. Select the same storage you want to use for your machines -5. Select the same network you want to use for your machines -6. Leave everyhting in the "Customize Template" and "Ready to complete" dialog as it is -7. Wait until the VM got fully imported and the "Snapshots" => "Create Snapshot" button is not grayed out anymore - -#### Command-line procedure - -Prerequisites: - -* [GOVC](https://github.com/vmware/govmomi/tree/master/govc): tested on version 0.22.1 -* [jq](https://stedolan.github.io/jq/) - -Procedure: - -1. Download the `OVA` for the targeted OS. - - ``` - curl -sL "${OVA_URL}" -O . - ``` - -2. Extract the specs from the `OVA`: - - ``` - govc import.spec $(basename "${OVA_URL}") | jq -r > options.json - ``` - -3. Edit the `options.json` file with your text editor of choice. - - * Edit the `NetworkMapping` to point to the correct network. - * Make sure that `PowerOn` is set to `false`. - * Make sure that `MarkAsTemplate` is set to `false`. - * Verify the other properties and customize according to your needs. - e.g. - - ```json - { - "DiskProvisioning": "flat", - "IPAllocationPolicy": "dhcpPolicy", - "IPProtocol": "IPv4", - "PropertyMapping": [ - { - "Key": "guestinfo.hostname", - "Value": "" - }, - { - "Key": "guestinfo.flatcar.config.data", - "Value": "" - }, - { - "Key": "guestinfo.flatcar.config.url", - "Value": "" - }, - { - "Key": "guestinfo.flatcar.config.data.encoding", - "Value": "" - }, - { - "Key": "guestinfo.interface.0.name", - "Value": "" - }, - { - "Key": "guestinfo.interface.0.mac", - "Value": "" - }, - { - "Key": "guestinfo.interface.0.dhcp", - "Value": "no" - }, - { - "Key": "guestinfo.interface.0.role", - "Value": "public" - }, - { - "Key": "guestinfo.interface.0.ip.0.address", - "Value": "" - }, - { - "Key": "guestinfo.interface.0.route.0.gateway", - "Value": "" - }, - { - "Key": "guestinfo.interface.0.route.0.destination", - "Value": "" - }, - { - "Key": "guestinfo.dns.server.0", - "Value": "" - }, - { - "Key": "guestinfo.dns.server.1", - "Value": "" - } - ], - "NetworkMapping": [ - { - "Name": "VM Network", - "Network": "Kubermatic Default" - } - ], - "MarkAsTemplate": false, - "PowerOn": false, - "InjectOvfEnv": false, - "WaitForIP": false, - "Name": null - } - ``` - -4. Create a VM from the `OVA`: - - ``` - govc import.ova -options=options.json $(basename "${OVA_URL}") - ``` - -### Create template VM from qcow2 - -Prerequisites: - -* vSphere (tested on version 6.7) -* GOVC (tested on version 0.22.1) -* qemu-img (tested on version 4.2.0) -* curl or wget - -Procedure: - -1. Download the guest image in qcow2 format end export an environment variable - with the name of the file. - - ``` - # The URL below is just an example - image_url="https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2" - image_name="$(basename -- "${image_url}" | sed 's/.qcow2$//g')" - curl -sL "${image_url}" -O . - ``` - -2. Convert it to vmdk e.g. - - ``` - qemu-img convert -O vmdk -o subformat=streamOptimized "./${image_name}.qcow2" "${image_name}.vmdk" - ``` - -3. Upload to vSphere using WebUI or GOVC: - - Make sure to replace the parameters on the command below with the correct - values specific to yout vSphere environment. - - ``` - govc import.vmdk -dc=dc-1 -pool=/dc-1/host/cl-1/Resources -ds=ds-1 "./${image_name}.vmdk" - ``` - -4. Inflate the created disk (see vmware [documentation][inflate_thin_virtual_disks] for more details) - - ``` - govc datastore.disk.inflate -dc dc-1 -ds ds-1 "${image_name}/${image_name}.vmdk" - ``` - -5. Create a new virtual machine using that image with vSphere WebUI. -6. During the `Customize Hardware` step: - 1. Remove the disk present by default - 2. Click on `ADD NEW DEVICE`, select `Existing Hard Disk` and select the - disk previously created. -7. The vm is ready to be used by the `MachineController` by referencing its name in the field `.spec.template.spec.providerSpec.value.cloudProviderSpec.templateVMName` of the `MachineDeployment`. - -### OS images - -Information about supported OS versions can be found [here](./operating-system.md#supported-os-versions). - -#### Ubuntu - -Ubuntu OVA template can be foud at . - -Follow [OVA](#create-template-vm-from-ova) template VM creation guide. - -#### RHEL - -Red Hat Enterprise Linux 8.x KVM Guest Image can be found at [Red Hat Customer Portal][rh_portal_rhel8]. - -Follow [qcow2](#create-template-vm-from-qcow2) template VM creation guide. - -#### CentOS - -CentOS 7 image can be found at the following link: . - -Follow [qcow2](#create-template-vm-from-qcow2) template VM creation guide. - -## Provider configuration - -VSphere provider accepts the following configuration parameters: - -```yaml -# Can also be set via the env var 'VSPHERE_USERNAME' on the machine-controller -username: '<< VSPHERE_USERNAME >>' -# Can also be set via the env var 'VSPHERE_ADDRESS' on the machine-controller -# example: 'https://your-vcenter:8443'. '/sdk' gets appended automatically -vsphereURL: '<< VSPHERE_ADDRESS >>' -# Can also be set via the env var 'VSPHERE_PASSWORD' on the machine-controller -password: "<< VSPHERE_PASSWORD >>" -# datacenter name -datacenter: datacenter1 -# VM template name -templateVMName: ubuntu-template -# Optional. Sets the networks on the VM. If no network is specified, the template default will be used. -networks: -- network1 -# Optional -folder: folder1 -# Optional: Force VMs to be provisoned to the specified resourcePool -# Default is to use the resourcePool of the template VM -# example: kubeone or /DC/host/Cluster01/Resources/kubeone -resourcePool: kubeone -cluster: cluster1 -# either datastore or datastoreCluster have to be provided. -datastore: datastore1 -datastoreCluster: datastore-cluster1 -# Can also be set via the env var 'VSPHERE_ALLOW_INSECURE' on the machine-controller -allowInsecure: true -# instance resources -cpus: 2 -memoryMB: 2048 -# Optional: Resize the root disk to this size. Must be bigger than the existing size -# Default is to leave the disk at the same size as the template -diskSizeGB: 10 -``` - -### Datastore and DatastoreCluster - -A `Datastore` is the basic unit of storage abstraction in vSphere storage (more details [here][datastore]). - -A `DatastoreCluster` (sometimes referred to as StoragePod) is a logical grouping of `Datastores`, it provides some resource management capabilities (more details [here][datastore_cluster]). - -VSphere provider configuration in a `MachineDeployment` should specify either a `Datastore` or a `DatastoreCluster`. If both are specified or if one of the two is missing the `MachineDeployment` validation will fail. - -*Note that* -the `datastore` or `datastoreCluster` specified in the `MachineDeployment` will be only used for the placement of VM and disk files related to the VMs provisioned by the `machine controller`. They do not influence the placement of persistent volumes used by PODs, that only depends on the cloud configuration given to the k8s cloud provider running in control plane. - -[vm_templates]: https://docs.vmware.com/en/VMware-vSphere/6.7/com.vmware.vsphere.vm_admin.doc/GUID-F7BF0E6B-7C4F-4E46-8BBF-76229AEA7220.html?hWord=N4IghgNiBcIG4FsAEAXApggDhM6DOIAvkA -[datastore]: https://docs.vmware.com/en/VMware-vSphere/6.7/com.vmware.vsphere.storage.doc/GUID-3CC7078E-9C30-402C-B2E1-2542BEE67E8F.html -[datastore_cluster]: https://docs.vmware.com/en/VMware-vSphere/6.7/com.vmware.vsphere.resmgmt.doc/GUID-598DF695-107E-406B-9C95-0AF961FC227A.html -[inflate_thin_virtual_disks]: https://docs.vmware.com/en/VMware-vSphere/6.7/com.vmware.vsphere.storage.doc/GUID-C371B88F-C407-4A69-8F3B-FA877D6955F8.html -[rh_portal_rhel8]: https://access.redhat.com/downloads/content/479/ver=/rhel---8/8.1/x86_64/product-software diff --git a/examples/alibaba-machinedeployment.yaml b/examples/alibaba-machinedeployment.yaml deleted file mode 100644 index 154b6196c..000000000 --- a/examples/alibaba-machinedeployment.yaml +++ /dev/null @@ -1,65 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - # If you change the namespace/name, you must also - # adjust the rbac rules - name: machine-controller-alibaba - namespace: kube-system -type: Opaque -stringData: - accessKeyID: << ALIBABA_ACCESS_KEY_ID >> - accessKeySecret: << ALIBABA_ACCESS_SECRET >> ---- -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: alibaba-machinedeployment - namespace: kube-system -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - foo: bar - template: - metadata: - labels: - foo: bar - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "alibaba" - cloudProviderSpec: - # If empty, can be set via ALIBABA_ACCESS_KEY_ID env var - accessKeyID: - secretKeyRef: - namespace: kube-system - name: machine-controller-alibaba - key: accessKeyID - accessKeySecret: - secretKeyRef: - namespace: kube-system - name: machine-controller-alibaba - key: accessKeySecret - instanceType: "ecs.t1.xsmall" - instanceName: "alibaba-instance" - internetMaxBandwidthOut: 10 - regionID: eu-central-1 - vSwitchID: "vswitchID" - zoneID: eu-central-1a - diskType: "cloud_efficiency" - diskSize: "40" - operatingSystem: "ubuntu" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - versions: - kubelet: 1.28.5 diff --git a/examples/anexia-machinedeployment.yaml b/examples/anexia-machinedeployment.yaml deleted file mode 100644 index 7e9745903..000000000 --- a/examples/anexia-machinedeployment.yaml +++ /dev/null @@ -1,57 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: anexia-machinedeployment - namespace: kube-system -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: anexia-machinedeployment - template: - metadata: - labels: - name: anexia-machinedeployment - spec: - providerSpec: - value: - cloudProvider: anexia - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProviderSpec: - # If empty, can be set via ANXCLOUD_TOKEN env var - token: - secretKeyRef: - namespace: kube-system - name: machine-controller-anexia - key: token - vlanID: "<< ANEXIA_VLAN_ID >>" - # Currently only the "Flatcar Linux Stable" template is supported. - # Use templateBuild to specify a build. If empty => latest - # Alternatively use templateID for a specific template. - template: "<< ANEXIA_TEMPLATE_NAME >>" - locationID: "<< ANEXIA_LOCATION_ID >>" - cpus: 2 - memory: 2048 - - disks: - - size: 60 - performanceType: ENT6 - - # You may have this old disk config attribute in your config - please migrate to the disks attribute. - # For now it is still recognized though. - #diskSize: 60 - # Flatcar is the only supported operating system - operatingSystem: "flatcar" - operatingSystemSpec: - # Force cloud-init instead of ignition. Anexia supports cloud-init only. - provisioningUtility: "cloud-init" - distUpgradeOnBoot: false - disableAutoUpdate: true - versions: - kubelet: 1.28.5 diff --git a/examples/azure-machinedeployment.yaml b/examples/azure-machinedeployment.yaml deleted file mode 100644 index 755a1e5a7..000000000 --- a/examples/azure-machinedeployment.yaml +++ /dev/null @@ -1,94 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: machine-controller-azure - namespace: kube-system -type: Opaque -stringData: - tenantID: "<< AZURE_TENANT_ID >>" - clientID: "<< AZURE_CLIENT_ID >>" - clientSecret: "<< AZURE_CLIENT_SECRET >>" - subscriptionID: "<< AZURE_SUBSCRIPTION_ID >>" ---- -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: azure-machinedeployment - namespace: kube-system -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - foo: bar - template: - metadata: - labels: - foo: bar - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "azure" - cloudProviderSpec: - # Can also be set via the env var 'AZURE_TENANT_ID' on the machine-controller - tenantID: - secretKeyRef: - namespace: kube-system - name: machine-controller-azure - key: tenantID - # Can also be set via the env var 'AZURE_CLIENT_ID' on the machine-controller - clientID: - secretKeyRef: - namespace: kube-system - name: machine-controller-azure - key: clientID - # Can also be set via the env var 'AZURE_CLIENT_SECRET' on the machine-controller - clientSecret: - secretKeyRef: - namespace: kube-system - name: machine-controller-azure - key: clientSecret - # Can also be set via the env var 'AZURE_SUBSCRIPTION_ID' on the machine-controller - subscriptionID: - secretKeyRef: - namespace: kube-system - name: machine-controller-azure - key: subscriptionID - location: "westeurope" - resourceGroup: "<< YOUR_RESOURCE_GROUP >>" - vnetResourceGroup: "<< YOUR_VNET_RESOURCE_GROUP >>" - vmSize: "Standard_F2" - # optional disk size values in GB. If not set, the defaults for the vmSize will be used. - osDiskSize: 30 - dataDiskSize: 30 - vnetName: "<< VNET_NAME >>" - subnetName: "<< SUBNET_NAME >>" - routeTableName: "<< ROUTE_TABLE_NAME >>" - imageID: "myImageID" - assignPublicIP: false - securityGroupName: my-security-group - # zones is an optional field and it represents Availability Zones is a high-availability offering - # that protects your applications and data from datacenter failures. - zones: - - "1" - operatingSystem: "flatcar" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` - rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" - # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` - rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" - # 'rhsmOfflineToken' if it was provided red hat systems subscriptions will be removed upon machines deletions, and if wasn't - # provided the rhsm will be disabled and any created subscription won't be removed automatically - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" - versions: - kubelet: 1.28.5 diff --git a/examples/cdi-operator-cr.yaml b/examples/cdi-operator-cr.yaml deleted file mode 100644 index 8e0397078..000000000 --- a/examples/cdi-operator-cr.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: cdi.kubevirt.io/v1alpha1 -kind: CDI -metadata: - name: cdi - namespace: cdi -spec: - imagePullPolicy: IfNotPresent diff --git a/examples/cdi-operator.yaml b/examples/cdi-operator.yaml deleted file mode 100644 index f801539af..000000000 --- a/examples/cdi-operator.yaml +++ /dev/null @@ -1,2223 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - cdi.kubevirt.io: "" - name: cdi ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - labels: - operator.cdi.kubevirt.io: "" - name: cdis.cdi.kubevirt.io -spec: - conversion: - strategy: None - group: cdi.kubevirt.io - names: - kind: CDI - listKind: CDIList - plural: cdis - shortNames: - - cdi - - cdis - singular: cdi - scope: Cluster - versions: - - additionalPrinterColumns: - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - - jsonPath: .status.phase - name: Phase - type: string - name: v1alpha1 - schema: - openAPIV3Schema: - description: CDI is the CDI Operator CRD - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: CDISpec defines our specification for the CDI installation - properties: - certConfig: - description: certificate configuration - properties: - ca: - description: CA configuration CA certs are kept in the CA bundle as long as they are valid - properties: - duration: - description: The requested 'duration' (i.e. lifetime) of the Certificate. - type: string - renewBefore: - description: The amount of time before the currently issued certificate's `notAfter` time that we will begin to attempt to renew the certificate. - type: string - type: object - server: - description: Server configuration Certs are rotated and discarded - properties: - duration: - description: The requested 'duration' (i.e. lifetime) of the Certificate. - type: string - renewBefore: - description: The amount of time before the currently issued certificate's `notAfter` time that we will begin to attempt to renew the certificate. - type: string - type: object - type: object - cloneStrategyOverride: - description: 'Clone strategy override: should we use a host-assisted copy even if snapshots are available?' - enum: - - copy - - snapshot - type: string - config: - description: CDIConfig at CDI level - properties: - featureGates: - description: FeatureGates are a list of specific enabled feature gates - items: - type: string - type: array - filesystemOverhead: - description: FilesystemOverhead describes the space reserved for overhead when using Filesystem volumes. A value is between 0 and 1, if not defined it is 0.055 (5.5% overhead) - properties: - global: - description: Global is how much space of a Filesystem volume should be reserved for overhead. This value is used unless overridden by a more specific value (per storageClass) - pattern: ^(0(?:\.\d{1,3})?|1)$ - type: string - storageClass: - additionalProperties: - description: 'Percent is a string that can only be a value between [0,1) (Note: we actually rely on reconcile to reject invalid values)' - pattern: ^(0(?:\.\d{1,3})?|1)$ - type: string - description: StorageClass specifies how much space of a Filesystem volume should be reserved for safety. The keys are the storageClass and the values are the overhead. This value overrides the global value - type: object - type: object - importProxy: - description: ImportProxy contains importer pod proxy configuration. - properties: - HTTPProxy: - description: HTTPProxy is the URL http://:@: of the import proxy for HTTP requests. Empty means unset and will not result in the import pod env var. - type: string - HTTPSProxy: - description: HTTPSProxy is the URL https://:@: of the import proxy for HTTPS requests. Empty means unset and will not result in the import pod env var. - type: string - noProxy: - description: NoProxy is a comma-separated list of hostnames and/or CIDRs for which the proxy should not be used. Empty means unset and will not result in the import pod env var. - type: string - trustedCAProxy: - description: "TrustedCAProxy is the name of a ConfigMap in the cdi namespace that contains a user-provided trusted certificate authority (CA) bundle. The TrustedCAProxy field is consumed by the import controller that is resposible for coping it to a config map named trusted-ca-proxy-bundle-cm in the cdi namespace. Here is an example of the ConfigMap (in yaml): \n apiVersion: v1 kind: ConfigMap metadata: name: trusted-ca-proxy-bundle-cm namespace: cdi data: ca.pem: | -----BEGIN CERTIFICATE----- \t ... ... \t -----END CERTIFICATE-----" - type: string - type: object - insecureRegistries: - description: InsecureRegistries is a list of TLS disabled registries - items: - type: string - type: array - podResourceRequirements: - description: ResourceRequirements describes the compute resource requirements. - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - type: object - preallocation: - description: Preallocation controls whether storage for DataVolumes should be allocated in advance. - type: boolean - scratchSpaceStorageClass: - description: 'Override the storage class to used for scratch space during transfer operations. The scratch space storage class is determined in the following order: 1. value of scratchSpaceStorageClass, if that doesn''t exist, use the default storage class, if there is no default storage class, use the storage class of the DataVolume, if no storage class specified, use no storage class for scratch space' - type: string - uploadProxyURLOverride: - description: Override the URL used when uploading to a DataVolume - type: string - type: object - imagePullPolicy: - description: PullPolicy describes a policy for if/when to pull a container image - enum: - - Always - - IfNotPresent - - Never - type: string - infra: - description: Rules on which nodes CDI infrastructure pods will be scheduled - properties: - affinity: - description: affinity enables pod affinity/anti-affinity placement expanding the types of constraints that can be expressed with nodeSelector. affinity is going to be applied to the relevant kind of pods in parallel with nodeSelector See https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity - properties: - nodeAffinity: - description: Describes node affinity scheduling rules for the pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred. - items: - description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated with the corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements by node's labels. - items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements by node's fields. - items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. - items: - description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements by node's labels. - items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements by node's fields. - items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - nodeSelector: - additionalProperties: - type: string - description: 'nodeSelector is the node selector applied to the relevant kind of pods It specifies a map of key-value pairs: for the pod to be eligible to run on a node, the node must have each of the indicated key-value pairs as labels (it can have additional labels as well). See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector' - type: object - tolerations: - description: tolerations is a list of tolerations applied to the relevant kind of pods See https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ for more info. These are additional tolerations other than default ones. - items: - description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . - properties: - effect: - description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - type: object - uninstallStrategy: - description: CDIUninstallStrategy defines the state to leave CDI on uninstall - enum: - - RemoveWorkloads - - BlockUninstallIfWorkloadsExist - type: string - workload: - description: Restrict on which nodes CDI workload pods will be scheduled - properties: - affinity: - description: affinity enables pod affinity/anti-affinity placement expanding the types of constraints that can be expressed with nodeSelector. affinity is going to be applied to the relevant kind of pods in parallel with nodeSelector See https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity - properties: - nodeAffinity: - description: Describes node affinity scheduling rules for the pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred. - items: - description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated with the corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements by node's labels. - items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements by node's fields. - items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. - items: - description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements by node's labels. - items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements by node's fields. - items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - nodeSelector: - additionalProperties: - type: string - description: 'nodeSelector is the node selector applied to the relevant kind of pods It specifies a map of key-value pairs: for the pod to be eligible to run on a node, the node must have each of the indicated key-value pairs as labels (it can have additional labels as well). See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector' - type: object - tolerations: - description: tolerations is a list of tolerations applied to the relevant kind of pods See https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ for more info. These are additional tolerations other than default ones. - items: - description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . - properties: - effect: - description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - type: object - type: object - status: - description: CDIStatus defines the status of the installation - properties: - conditions: - description: A list of current conditions of the resource - items: - description: Condition represents the state of the operator's reconciliation functionality. - properties: - lastHeartbeatTime: - format: date-time - type: string - lastTransitionTime: - format: date-time - type: string - message: - type: string - reason: - type: string - status: - type: string - type: - description: ConditionType is the state of the operator's reconciliation functionality. - type: string - required: - - status - - type - type: object - type: array - observedVersion: - description: The observed version of the resource - type: string - operatorVersion: - description: The version of the resource as defined by the operator - type: string - phase: - description: Phase is the current phase of the deployment - type: string - targetVersion: - description: The desired version of the resource - type: string - type: object - required: - - spec - type: object - served: true - storage: false - subresources: {} - - additionalPrinterColumns: - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - - jsonPath: .status.phase - name: Phase - type: string - name: v1beta1 - schema: - openAPIV3Schema: - description: CDI is the CDI Operator CRD - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: CDISpec defines our specification for the CDI installation - properties: - certConfig: - description: certificate configuration - properties: - ca: - description: CA configuration CA certs are kept in the CA bundle as long as they are valid - properties: - duration: - description: The requested 'duration' (i.e. lifetime) of the Certificate. - type: string - renewBefore: - description: The amount of time before the currently issued certificate's `notAfter` time that we will begin to attempt to renew the certificate. - type: string - type: object - server: - description: Server configuration Certs are rotated and discarded - properties: - duration: - description: The requested 'duration' (i.e. lifetime) of the Certificate. - type: string - renewBefore: - description: The amount of time before the currently issued certificate's `notAfter` time that we will begin to attempt to renew the certificate. - type: string - type: object - type: object - cloneStrategyOverride: - description: 'Clone strategy override: should we use a host-assisted copy even if snapshots are available?' - enum: - - copy - - snapshot - type: string - config: - description: CDIConfig at CDI level - properties: - featureGates: - description: FeatureGates are a list of specific enabled feature gates - items: - type: string - type: array - filesystemOverhead: - description: FilesystemOverhead describes the space reserved for overhead when using Filesystem volumes. A value is between 0 and 1, if not defined it is 0.055 (5.5% overhead) - properties: - global: - description: Global is how much space of a Filesystem volume should be reserved for overhead. This value is used unless overridden by a more specific value (per storageClass) - pattern: ^(0(?:\.\d{1,3})?|1)$ - type: string - storageClass: - additionalProperties: - description: 'Percent is a string that can only be a value between [0,1) (Note: we actually rely on reconcile to reject invalid values)' - pattern: ^(0(?:\.\d{1,3})?|1)$ - type: string - description: StorageClass specifies how much space of a Filesystem volume should be reserved for safety. The keys are the storageClass and the values are the overhead. This value overrides the global value - type: object - type: object - importProxy: - description: ImportProxy contains importer pod proxy configuration. - properties: - HTTPProxy: - description: HTTPProxy is the URL http://:@: of the import proxy for HTTP requests. Empty means unset and will not result in the import pod env var. - type: string - HTTPSProxy: - description: HTTPSProxy is the URL https://:@: of the import proxy for HTTPS requests. Empty means unset and will not result in the import pod env var. - type: string - noProxy: - description: NoProxy is a comma-separated list of hostnames and/or CIDRs for which the proxy should not be used. Empty means unset and will not result in the import pod env var. - type: string - trustedCAProxy: - description: "TrustedCAProxy is the name of a ConfigMap in the cdi namespace that contains a user-provided trusted certificate authority (CA) bundle. The TrustedCAProxy field is consumed by the import controller that is resposible for coping it to a config map named trusted-ca-proxy-bundle-cm in the cdi namespace. Here is an example of the ConfigMap (in yaml): \n apiVersion: v1 kind: ConfigMap metadata: name: trusted-ca-proxy-bundle-cm namespace: cdi data: ca.pem: | -----BEGIN CERTIFICATE----- \t ... ... \t -----END CERTIFICATE-----" - type: string - type: object - insecureRegistries: - description: InsecureRegistries is a list of TLS disabled registries - items: - type: string - type: array - podResourceRequirements: - description: ResourceRequirements describes the compute resource requirements. - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - type: object - preallocation: - description: Preallocation controls whether storage for DataVolumes should be allocated in advance. - type: boolean - scratchSpaceStorageClass: - description: 'Override the storage class to used for scratch space during transfer operations. The scratch space storage class is determined in the following order: 1. value of scratchSpaceStorageClass, if that doesn''t exist, use the default storage class, if there is no default storage class, use the storage class of the DataVolume, if no storage class specified, use no storage class for scratch space' - type: string - uploadProxyURLOverride: - description: Override the URL used when uploading to a DataVolume - type: string - type: object - imagePullPolicy: - description: PullPolicy describes a policy for if/when to pull a container image - enum: - - Always - - IfNotPresent - - Never - type: string - infra: - description: Rules on which nodes CDI infrastructure pods will be scheduled - properties: - affinity: - description: affinity enables pod affinity/anti-affinity placement expanding the types of constraints that can be expressed with nodeSelector. affinity is going to be applied to the relevant kind of pods in parallel with nodeSelector See https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity - properties: - nodeAffinity: - description: Describes node affinity scheduling rules for the pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred. - items: - description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated with the corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements by node's labels. - items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements by node's fields. - items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. - items: - description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements by node's labels. - items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements by node's fields. - items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - nodeSelector: - additionalProperties: - type: string - description: 'nodeSelector is the node selector applied to the relevant kind of pods It specifies a map of key-value pairs: for the pod to be eligible to run on a node, the node must have each of the indicated key-value pairs as labels (it can have additional labels as well). See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector' - type: object - tolerations: - description: tolerations is a list of tolerations applied to the relevant kind of pods See https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ for more info. These are additional tolerations other than default ones. - items: - description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . - properties: - effect: - description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - type: object - uninstallStrategy: - description: CDIUninstallStrategy defines the state to leave CDI on uninstall - enum: - - RemoveWorkloads - - BlockUninstallIfWorkloadsExist - type: string - workload: - description: Restrict on which nodes CDI workload pods will be scheduled - properties: - affinity: - description: affinity enables pod affinity/anti-affinity placement expanding the types of constraints that can be expressed with nodeSelector. affinity is going to be applied to the relevant kind of pods in parallel with nodeSelector See https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity - properties: - nodeAffinity: - description: Describes node affinity scheduling rules for the pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred. - items: - description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated with the corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements by node's labels. - items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements by node's fields. - items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. - items: - description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements by node's labels. - items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements by node's fields. - items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - nodeSelector: - additionalProperties: - type: string - description: 'nodeSelector is the node selector applied to the relevant kind of pods It specifies a map of key-value pairs: for the pod to be eligible to run on a node, the node must have each of the indicated key-value pairs as labels (it can have additional labels as well). See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector' - type: object - tolerations: - description: tolerations is a list of tolerations applied to the relevant kind of pods See https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ for more info. These are additional tolerations other than default ones. - items: - description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . - properties: - effect: - description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - type: object - type: object - status: - description: CDIStatus defines the status of the installation - properties: - conditions: - description: A list of current conditions of the resource - items: - description: Condition represents the state of the operator's reconciliation functionality. - properties: - lastHeartbeatTime: - format: date-time - type: string - lastTransitionTime: - format: date-time - type: string - message: - type: string - reason: - type: string - status: - type: string - type: - description: ConditionType is the state of the operator's reconciliation functionality. - type: string - required: - - status - - type - type: object - type: array - observedVersion: - description: The observed version of the resource - type: string - operatorVersion: - description: The version of the resource as defined by the operator - type: string - phase: - description: Phase is the current phase of the deployment - type: string - targetVersion: - description: The desired version of the resource - type: string - type: object - required: - - spec - type: object - served: true - storage: true - subresources: {} - ---- -apiVersion: v1 -kind: ConfigMap -metadata: - labels: - operator.cdi.kubevirt.io: "" - name: cdi-operator-leader-election-helper - namespace: cdi - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cdi.kubevirt.io:operator - labels: - operator.cdi.kubevirt.io: "" - rbac.authorization.k8s.io/aggregate-to-admin: "true" -rules: -- apiGroups: - - cdi.kubevirt.io - resources: - - cdis - verbs: - - create - - patch - - get - - list - - delete - - watch - - deletecollection ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - operator.cdi.kubevirt.io: "" - name: cdi-operator - namespace: cdi ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - cdi.kubevirt.io: "" - name: cdi-operator-cluster -rules: -- apiGroups: - - rbac.authorization.k8s.io - resources: - - roles - - rolebindings - - clusterrolebindings - - clusterroles - verbs: - - '*' -- apiGroups: - - security.openshift.io - resources: - - securitycontextconstraints - verbs: - - get - - list - - watch -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - get - - patch - - update -- apiGroups: - - "" - resources: - - serviceaccounts - - services - verbs: - - '*' -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch - - update - - patch -- apiGroups: - - extensions - resources: - - deployments - verbs: - - '*' -- apiGroups: - - extensions - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - configmaps - verbs: - - watch - - create - - delete - - get - - update - - patch - - list -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - delete - - get - - update - - patch - - list -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - create - - delete - - get - - update - - patch - - list - - watch -- apiGroups: - - apps - resources: - - deployments - - daemonstes - verbs: - - create - - get - - list - - delete - - watch - - update -- apiGroups: - - admissionregistration.k8s.io - resources: - - validatingwebhookconfigurations - verbs: - - get - - create - - update -- apiGroups: - - apiregistration.k8s.io - resources: - - apiservices - verbs: - - get - - list - - watch - - create - - update - - patch -- apiGroups: - - cdi.kubevirt.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list -- apiGroups: - - "" - resources: - - events - verbs: - - create - - update - - patch -- apiGroups: - - "" - resources: - - pods - - persistentvolumeclaims - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - persistentvolumeclaims/finalizers - - pods/finalizers - verbs: - - update -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch - - create - - delete -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch - - create -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get - - list -- apiGroups: - - route.openshift.io - resources: - - routes - verbs: - - get - - list - - watch - - create - - update - - patch -- apiGroups: - - route.openshift.io - resources: - - routes/custom-host - verbs: - - create - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - cdi.kubevirt.io: "" - name: cdi-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cdi-operator-cluster -subjects: -- kind: ServiceAccount - name: cdi-operator - namespace: cdi - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cdi-operator - namespace: cdi -spec: - replicas: 1 - selector: - matchLabels: - name: cdi-operator - operator.cdi.kubevirt.io: "" - strategy: {} - template: - metadata: - labels: - name: cdi-operator - operator.cdi.kubevirt.io: "" - spec: - containers: - - env: - - name: DEPLOY_CLUSTER_RESOURCES - value: "true" - - name: DOCKER_REPO - value: kubevirt - - name: DOCKER_TAG - value: v1.9.3 - - name: CONTROLLER_IMAGE - value: cdi-controller - - name: IMPORTER_IMAGE - value: cdi-importer - - name: CLONER_IMAGE - value: cdi-cloner - - name: APISERVER_IMAGE - value: cdi-apiserver - - name: UPLOAD_SERVER_IMAGE - value: cdi-uploadserver - - name: UPLOAD_PROXY_IMAGE - value: cdi-uploadproxy - - name: VERBOSITY - value: "1" - - name: PULL_POLICY - value: IfNotPresent - image: kubevirt/cdi-operator:v1.9.3 - imagePullPolicy: IfNotPresent - name: cdi-operator - ports: - - containerPort: 60000 - name: metrics - protocol: TCP - resources: {} - serviceAccountName: cdi-operator diff --git a/examples/digitalocean-machinedeployment.yaml b/examples/digitalocean-machinedeployment.yaml deleted file mode 100644 index 804c8c3b4..000000000 --- a/examples/digitalocean-machinedeployment.yaml +++ /dev/null @@ -1,60 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - # If you change the namespace/name, you must also - # adjust the rbac rules - name: machine-controller-digitalocean - namespace: kube-system -type: Opaque -stringData: - token: << DIGITALOCEAN_TOKEN >> ---- -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: digitalocean-machinedeployment - namespace: kube-system -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - foo: bar - template: - metadata: - labels: - foo: bar - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "digitalocean" - cloudProviderSpec: - # If empty, can be set via DO_TOKEN env var - token: - secretKeyRef: - namespace: kube-system - name: machine-controller-digitalocean - key: token - region: fra1 - size: 2gb - backups: false - ipv6: false - private_networking: true - # Monitoring must be turned off for Flatcar Container Linux - monitoring: false - tags: - - "machine-controller" - # Can be 'ubuntu' or 'centos' - operatingSystem: "ubuntu" - operatingSystemSpec: - disableAutoUpdate: true - versions: - kubelet: 1.28.5 diff --git a/examples/equinixmetal-machinedeployment.yaml b/examples/equinixmetal-machinedeployment.yaml deleted file mode 100644 index dbce45cce..000000000 --- a/examples/equinixmetal-machinedeployment.yaml +++ /dev/null @@ -1,54 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - # If you change the namespace/name, you must also - # adjust the rbac rules - name: machine-controller-equinixmetal - namespace: kube-system -type: Opaque -stringData: - token: << METAL_AUTH_TOKEN >> ---- -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: equinixmetal-machinedeployment - namespace: kube-system -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - foo: bar - template: - metadata: - labels: - foo: bar - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "equinixmetal" - cloudProviderSpec: - # If empty, can be set via METAL_TOKEN env var - token: - secretKeyRef: - namespace: kube-system - name: machine-controller-equinixmetal - key: token - instanceType: "t1.small.x86" - projectID: "<< PROJECT_ID >>" - facilities: - - "ewr1" - operatingSystem: "ubuntu" - operatingSystemSpec: - distUpgradeOnBoot: false - versions: - kubelet: 1.28.5 diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml deleted file mode 100644 index 7c598af34..000000000 --- a/examples/gce-machinedeployment.yaml +++ /dev/null @@ -1,93 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - # If you change the namespace/name, you must also - # adjust the rbac rules - name: machine-controller-gce - namespace: kube-system -type: Opaque -data: - # The base64 encoding here is only to satisfy Kubernetes' - # Secret storage and to prevent multiline string replacement - # issues if we used stringData here (because the GCP SA is - # a multiline JSON string). - serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>" - ---- -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: gce-machinedeployment - namespace: kube-system -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - foo: bar - template: - metadata: - labels: - foo: bar - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "gce" - cloudProviderSpec: - # If empty, can be set via GOOGLE_SERVICE_ACCOUNT env var - serviceAccount: - secretKeyRef: - namespace: kube-system - name: machine-controller-gce - key: serviceAccount - # See https://cloud.google.com/compute/docs/regions-zones/ - zone: "europe-west3-a" - # Is the id of the GCP project that can be used to create machines in. Usually this id is taken from the - # service account however, it should be possible to create a machine in another project, as long as the - # machine controller has the right permissions - projectID: "" - # See https://cloud.google.com/compute/docs/machine-types - machineType: "n1-standard-2" - # In GB - diskSize: 50 - # The name or self_link of the network and subnetwork to attach this interface to; - # either of both can be provided, otherwise default network will taken - # in case if both empty — default network will be used - network: "my-cool-network" - subnetwork: "my-cool-subnetwork" - # See https://cloud.google.com/compute/docs/instances/preemptible - preemptible: false - # Can be 'pd-standard' or 'pd-ssd' - diskType: "pd-standard" - labels: - "kubernetes_cluster": "my-cluster" - # Whether to assign a public IP Address. Required for Internet access - assignPublicIPAddress: true - customImage: "myCustomImage" - disableMachineServiceAccount: false - enableNestedVirtualization: false - minCPUPlatform: "Intel Haswell" - guestOSFeatures: - - "VIRTIO_SCSI_MULTIQUEUE" - - "GVNIC" - # Can be 'ubuntu' or 'rhel' - operatingSystem: "ubuntu" - operatingSystemSpec: - disableAutoUpdate: true - # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` - rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" - # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` - rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" - # 'rhsmOfflineToken' if it was provided red hat systems subscriptions will be removed upon machines deletions, and if wasn't - # provided the rhsm will be disabled and any created subscription won't be removed automatically - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" - versions: - kubelet: 1.28.5 diff --git a/examples/hetzner-machinedeployment.yaml b/examples/hetzner-machinedeployment.yaml deleted file mode 100644 index 1d3adb228..000000000 --- a/examples/hetzner-machinedeployment.yaml +++ /dev/null @@ -1,70 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - # If you change the namespace/name, you must also - # adjust the rbac rules - name: machine-controller-hetzner - namespace: kube-system -type: Opaque -stringData: - token: << HETZNER_TOKEN >> ---- -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: hetzner-machinedeployment - namespace: kube-system -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - foo: bar - template: - metadata: - labels: - foo: bar - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "hetzner" - cloudProviderSpec: - # If empty, can be set via HZ_TOKEN env var - token: - secretKeyRef: - namespace: kube-system - name: machine-controller-hetzner - key: token - serverType: "cx21" - # Optional - datacenter: "" - location: "fsn1" - image: "ubuntu-20.04" - # Optional: placement group prefix - placementGroupPrefix: "<< YOUR_PLACEMENT_GROUP_PREFIX >>" - # Optional: network IDs or names - networks: - - "<< YOUR_NETWORK >>" - # Optional: assignPublicIPv4 whether a public ipv4 should be assigned or not - assignPublicIPv4: true - # Optional: assignPublicIPv4 whether an ipv6 should be assigned or not - assignPublicIPv6: true - # Optional: firewall IDs or names - firewalls: - - "<< YOUR_FIREWALL >>" - # Optional - labels: - my: label - operatingSystem: "ubuntu" - operatingSystemSpec: - distUpgradeOnBoot: false - versions: - kubelet: 1.28.5 diff --git a/examples/kubevirt-cr.yaml b/examples/kubevirt-cr.yaml deleted file mode 100644 index 0208bb865..000000000 --- a/examples/kubevirt-cr.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -apiVersion: kubevirt.io/v1alpha3 -kind: KubeVirt -metadata: - name: kubevirt - namespace: kubevirt -spec: - imagePullPolicy: IfNotPresent diff --git a/examples/kubevirt-local-mounter.yaml b/examples/kubevirt-local-mounter.yaml deleted file mode 100644 index 302d99e81..000000000 --- a/examples/kubevirt-local-mounter.yaml +++ /dev/null @@ -1,43 +0,0 @@ -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: create-bind-mounts - namespace: kube-system -spec: - selector: - matchLabels: - app: create-bind-mounts - template: - metadata: - labels: - app: create-bind-mounts - spec: - hostPID: true - containers: - - name: startup-script - image: quay.io/kubermatic/startup-script:v0.1.0 - securityContext: - privileged: true - env: - - name: STARTUP_SCRIPT - value: | - for ((i=1;i<=50;i++)); - do - mkdir -p /opt/kube-disks/${i} /mnt/local-volumes/${i} - cat >/etc/systemd/system/mnt-local\\x2dvolumes-${i}.mount <>" - cloudProvider: "kubevirt" - cloudProviderSpec: - clusterName: cluster-name - auth: - kubeconfig: - # Can also be set via the env var 'KUBEVIRT_KUBECONFIG' on the machine-controller. - # If instead specified directly, this value should be a base64 encoded kubeconfig. - value: "<< KUBECONFIG_BASE64 >>" - virtualMachine: - instancetype: - name: "standard-2" - kind: "VirtualMachineInstancetype" # Allowed values: "VirtualMachineInstancetype"/"VirtualMachineClusterInstancetype" - preference: - name: "sockets-advantage" - kind: "VirtualMachinePreference" # Allowed values: "VirtualMachinePreference"/"VirtualMachineClusterPreference" - template: - cpus: "1" - memory: "2048M" - primaryDisk: - osImage: http://10.109.79.210/<< OS_NAME >>.img - size: "10Gi" - storageClassName: kubermatic-fast - affinity: - nodeAffinityPreset: - type: "" # Allowed values: "", "soft", "hard" - key: "foo" - values: - - bar - topologySpreadConstraints: - - maxSkew: "1" - topologyKey: "kubernetes.io/hostname" - whenUnsatisfiable: "" # Allowed values: "DoNotSchedule", "ScheduleAnyway" - # Can also be `centos`, must align with he configured registryImage above - operatingSystem: "ubuntu" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` - rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" - # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` - rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" - # 'rhsmOfflineToken' if it was provided red hat systems subscriptions will be removed upon machines deletions, and if wasn't - # provided the rhsm will be disabled and any created subscription won't be removed automatically - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" - versions: - kubelet: 1.28.5 diff --git a/examples/kubevirt-operator-0.19.0.yaml b/examples/kubevirt-operator-0.19.0.yaml deleted file mode 100644 index ae0b30886..000000000 --- a/examples/kubevirt-operator-0.19.0.yaml +++ /dev/null @@ -1,4333 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - labels: - kubevirt.io: "" - name: kubevirt ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - labels: - operator.kubevirt.io: "" - name: kubevirts.kubevirt.io -spec: - group: kubevirt.io - names: - categories: - - all - kind: KubeVirt - plural: kubevirts - shortNames: - - kv - - kvs - singular: kubevirt - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - - jsonPath: .status.phase - name: Phase - type: string - name: v1 - schema: - openAPIV3Schema: - description: KubeVirt represents the object deploying all KubeVirt resources - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - certificateRotateStrategy: - properties: - selfSigned: - properties: - ca: - description: CA configuration CA certs are kept in the CA - bundle as long as they are valid - properties: - duration: - description: The requested 'duration' (i.e. lifetime) - of the Certificate. - type: string - renewBefore: - description: The amount of time before the currently issued - certificate's "notAfter" time that we will begin to - attempt to renew the certificate. - type: string - type: object - caOverlapInterval: - description: Deprecated. Use CA.Duration and CA.RenewBefore - instead - type: string - caRotateInterval: - description: Deprecated. Use CA.Duration instead - type: string - certRotateInterval: - description: Deprecated. Use Server.Duration instead - type: string - server: - description: Server configuration Certs are rotated and discarded - properties: - duration: - description: The requested 'duration' (i.e. lifetime) - of the Certificate. - type: string - renewBefore: - description: The amount of time before the currently issued - certificate's "notAfter" time that we will begin to - attempt to renew the certificate. - type: string - type: object - type: object - type: object - configuration: - description: holds kubevirt configurations. same as the virt-configMap - properties: - cpuModel: - type: string - cpuRequest: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - defaultRuntimeClass: - type: string - developerConfiguration: - description: DeveloperConfiguration holds developer options - properties: - cpuAllocationRatio: - type: integer - diskVerification: - description: DiskVerification holds container disks verification - limits - properties: - memoryLimit: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - memoryLimit - type: object - featureGates: - items: - type: string - type: array - logVerbosity: - description: LogVerbosity sets log verbosity level of various - components - properties: - nodeVerbosity: - additionalProperties: - type: integer - description: NodeVerbosity represents a map of nodes with - a specific verbosity level - type: object - virtAPI: - type: integer - virtController: - type: integer - virtHandler: - type: integer - virtLauncher: - type: integer - virtOperator: - type: integer - type: object - memoryOvercommit: - type: integer - minimumClusterTSCFrequency: - description: Allow overriding the automatically determined - minimum TSC frequency of the cluster and fixate the minimum - to this frequency. - format: int64 - type: integer - minimumReservePVCBytes: - format: int64 - type: integer - nodeSelectors: - additionalProperties: - type: string - type: object - pvcTolerateLessSpaceUpToPercent: - type: integer - useEmulation: - type: boolean - type: object - emulatedMachines: - items: - type: string - type: array - imagePullPolicy: - description: PullPolicy describes a policy for if/when to pull - a container image - type: string - machineType: - type: string - memBalloonStatsPeriod: - format: int32 - type: integer - migrations: - description: MigrationConfiguration holds migration options - properties: - allowAutoConverge: - type: boolean - allowPostCopy: - type: boolean - bandwidthPerMigration: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - completionTimeoutPerGiB: - format: int64 - type: integer - disableTLS: - type: boolean - nodeDrainTaintKey: - type: string - parallelMigrationsPerCluster: - format: int32 - type: integer - parallelOutboundMigrationsPerNode: - format: int32 - type: integer - progressTimeout: - format: int64 - type: integer - unsafeMigrationOverride: - type: boolean - type: object - minCPUModel: - type: string - network: - description: NetworkConfiguration holds network options - properties: - defaultNetworkInterface: - type: string - permitBridgeInterfaceOnPodNetwork: - type: boolean - permitSlirpInterface: - type: boolean - type: object - obsoleteCPUModels: - additionalProperties: - type: boolean - type: object - ovmfPath: - type: string - permittedHostDevices: - description: PermittedHostDevices holds inforamtion about devices - allowed for passthrough - properties: - mediatedDevices: - items: - description: MediatedHostDevice represents a host mediated - device allowed for passthrough - properties: - externalResourceProvider: - type: boolean - mdevNameSelector: - type: string - resourceName: - type: string - required: - - mdevNameSelector - - resourceName - type: object - type: array - x-kubernetes-list-type: atomic - pciHostDevices: - items: - description: PciHostDevice represents a host PCI device - allowed for passthrough - properties: - externalResourceProvider: - description: If true, KubeVirt will leave the allocation - and monitoring to an external device plugin - type: boolean - pciVendorSelector: - description: The vendor_id:product_id tuple of the PCI - device - type: string - resourceName: - description: The name of the resource that is representing - the device. Exposed by a device plugin and requested - by VMs. Typically of the form vendor.com/product_nameThe - name of the resource that is representing the device. - Exposed by a device plugin and requested by VMs. Typically - of the form vendor.com/product_name - type: string - required: - - pciVendorSelector - - resourceName - type: object - type: array - x-kubernetes-list-type: atomic - type: object - selinuxLauncherType: - type: string - smbios: - properties: - family: - type: string - manufacturer: - type: string - product: - type: string - sku: - type: string - version: - type: string - type: object - supportedGuestAgentVersions: - description: deprecated - items: - type: string - type: array - virtualMachineInstancesPerNode: - type: integer - type: object - customizeComponents: - properties: - flags: - description: Configure the value used for deployment and daemonset - resources - properties: - api: - additionalProperties: - type: string - type: object - controller: - additionalProperties: - type: string - type: object - handler: - additionalProperties: - type: string - type: object - type: object - patches: - items: - properties: - patch: - type: string - resourceName: - minLength: 1 - type: string - resourceType: - minLength: 1 - type: string - type: - type: string - required: - - patch - - resourceName - - resourceType - - type - type: object - type: array - x-kubernetes-list-type: atomic - type: object - imagePullPolicy: - description: The ImagePullPolicy to use. - type: string - imageRegistry: - description: The image registry to pull the container images from - Defaults to the same registry the operator's container image is - pulled from. - type: string - imageTag: - description: The image tag to use for the continer images installed. - Defaults to the same tag as the operator's container image. - type: string - infra: - description: selectors and tolerations that should apply to KubeVirt - infrastructure components - properties: - nodePlacement: - description: nodePlacement decsribes scheduling confiuguration - for specific KubeVirt components - properties: - affinity: - description: affinity enables pod affinity/anti-affinity placement - expanding the types of constraints that can be expressed - with nodeSelector. affinity is going to be applied to the - relevant kind of pods in parallel with nodeSelector See - https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity - properties: - nodeAffinity: - description: Describes node affinity scheduling rules - for the pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule - pods to nodes that satisfy the affinity expressions - specified by this field, but it may choose a node - that violates one or more of the expressions. The - node that is most preferred is the one with the - greatest sum of weights, i.e. for each node that - meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, - etc.), compute a sum by iterating through the elements - of this field and adding "weight" to the sum if - the node matches the corresponding matchExpressions; - the node(s) with the highest sum are the most preferred. - items: - description: An empty preferred scheduling term - matches all objects with implicit weight 0 (i.e. - it's a no-op). A null preferred scheduling term - matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated - with the corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that the - selector applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. - If the operator is Gt or Lt, the - values array must have a single - element, which will be interpreted - as an integer. This array is replaced - during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that the - selector applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. - If the operator is Gt or Lt, the - values array must have a single - element, which will be interpreted - as an integer. This array is replaced - during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - description: Weight associated with matching - the corresponding nodeSelectorTerm, in the - range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified - by this field are not met at scheduling time, the - pod will not be scheduled onto the node. If the - affinity requirements specified by this field cease - to be met at some point during pod execution (e.g. - due to an update), the system may or may not try - to eventually evict the pod from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector - terms. The terms are ORed. - items: - description: A null or empty node selector term - matches no objects. The requirements of them - are ANDed. The TopologySelectorTerm type implements - a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that the - selector applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. - If the operator is Gt or Lt, the - values array must have a single - element, which will be interpreted - as an integer. This array is replaced - during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that the - selector applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. - If the operator is Gt or Lt, the - values array must have a single - element, which will be interpreted - as an integer. This array is replaced - during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. - co-locate this pod in the same node, zone, etc. as some - other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule - pods to nodes that satisfy the affinity expressions - specified by this field, but it may choose a node - that violates one or more of the expressions. The - node that is most preferred is the one with the - greatest sum of weights, i.e. for each node that - meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, - etc.), compute a sum by iterating through the elements - of this field and adding "weight" to the sum if - the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum - are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, - associated with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of - resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The - requirements are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: operator represents - a key's relationship to a set - of values. Valid operators are - In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array - of string values. If the operator - is In or NotIn, the values array - must be non-empty. If the operator - is Exists or DoesNotExist, the - values array must be empty. - This array is replaced during - a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of - {key,value} pairs. A single {key,value} - in the matchLabels map is equivalent - to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are - ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which - namespaces the labelSelector applies to - (matches against); null or empty list - means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located - (affinity) or not co-located (anti-affinity) - with the pods matching the labelSelector - in the specified namespaces, where co-located - is defined as running on a node whose - value of the label with key topologyKey - matches that of any node on which any - of the selected pods is running. Empty - topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching - the corresponding podAffinityTerm, in the - range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified - by this field are not met at scheduling time, the - pod will not be scheduled onto the node. If the - affinity requirements specified by this field cease - to be met at some point during pod execution (e.g. - due to a pod label update), the system may or may - not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes - corresponding to each podAffinityTerm are intersected, - i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those - matching the labelSelector relative to the given - namespace(s)) that this pod should be co-located - (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node - whose value of the label with key - matches that of any node on which a pod of the - set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules - (e.g. avoid putting this pod in the same node, zone, - etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule - pods to nodes that satisfy the anti-affinity expressions - specified by this field, but it may choose a node - that violates one or more of the expressions. The - node that is most preferred is the one with the - greatest sum of weights, i.e. for each node that - meets all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity - expressions, etc.), compute a sum by iterating through - the elements of this field and adding "weight" to - the sum if the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum - are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, - associated with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of - resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The - requirements are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: operator represents - a key's relationship to a set - of values. Valid operators are - In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array - of string values. If the operator - is In or NotIn, the values array - must be non-empty. If the operator - is Exists or DoesNotExist, the - values array must be empty. - This array is replaced during - a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of - {key,value} pairs. A single {key,value} - in the matchLabels map is equivalent - to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are - ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which - namespaces the labelSelector applies to - (matches against); null or empty list - means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located - (affinity) or not co-located (anti-affinity) - with the pods matching the labelSelector - in the specified namespaces, where co-located - is defined as running on a node whose - value of the label with key topologyKey - matches that of any node on which any - of the selected pods is running. Empty - topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching - the corresponding podAffinityTerm, in the - range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements specified - by this field are not met at scheduling time, the - pod will not be scheduled onto the node. If the - anti-affinity requirements specified by this field - cease to be met at some point during pod execution - (e.g. due to a pod label update), the system may - or may not try to eventually evict the pod from - its node. When there are multiple elements, the - lists of nodes corresponding to each podAffinityTerm - are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those - matching the labelSelector relative to the given - namespace(s)) that this pod should be co-located - (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node - whose value of the label with key - matches that of any node on which a pod of the - set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - nodeSelector: - additionalProperties: - type: string - description: 'nodeSelector is the node selector applied to - the relevant kind of pods It specifies a map of key-value - pairs: for the pod to be eligible to run on a node, the - node must have each of the indicated key-value pairs as - labels (it can have additional labels as well). See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector' - type: object - tolerations: - description: tolerations is a list of tolerations applied - to the relevant kind of pods See https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ - for more info. These are additional tolerations other than - default ones. - items: - description: The pod this Toleration is attached to tolerates - any taint that matches the triple using - the matching operator . - properties: - effect: - description: Effect indicates the taint effect to match. - Empty means match all taint effects. When specified, - allowed values are NoSchedule, PreferNoSchedule and - NoExecute. - type: string - key: - description: Key is the taint key that the toleration - applies to. Empty means match all taint keys. If the - key is empty, operator must be Exists; this combination - means to match all values and all keys. - type: string - operator: - description: Operator represents a key's relationship - to the value. Valid operators are Exists and Equal. - Defaults to Equal. Exists is equivalent to wildcard - for value, so that a pod can tolerate all taints of - a particular category. - type: string - tolerationSeconds: - description: TolerationSeconds represents the period - of time the toleration (which must be of effect NoExecute, - otherwise this field is ignored) tolerates the taint. - By default, it is not set, which means tolerate the - taint forever (do not evict). Zero and negative values - will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: Value is the taint value the toleration - matches to. If the operator is Exists, the value should - be empty, otherwise just a regular string. - type: string - type: object - type: array - type: object - type: object - monitorAccount: - description: The name of the Prometheus service account that needs - read-access to KubeVirt endpoints Defaults to prometheus-k8s - type: string - monitorNamespace: - description: The namespace Prometheus is deployed in Defaults to openshift-monitor - type: string - productName: - description: Designate the apps.kubevirt.io/part-of label for KubeVirt - components. Useful if KubeVirt is included as part of a product. - If ProductName is not specified, the part-of label will be omitted. - type: string - productVersion: - description: Designate the apps.kubevirt.io/version label for KubeVirt - components. Useful if KubeVirt is included as part of a product. - If ProductVersion is not specified, KubeVirt's version will be used. - type: string - uninstallStrategy: - description: Specifies if kubevirt can be deleted if workloads are - still present. This is mainly a precaution to avoid accidental data - loss - type: string - workloadUpdateStrategy: - description: WorkloadUpdateStrategy defines at the cluster level how - to handle automated workload updates - properties: - batchEvictionInterval: - description: "BatchEvictionInterval Represents the interval to - wait before issuing the next batch of shutdowns \n Defaults - to 1 minute" - type: string - batchEvictionSize: - description: "BatchEvictionSize Represents the number of VMIs - that can be forced updated per the BatchShutdownInteral interval - \n Defaults to 10" - type: integer - workloadUpdateMethods: - description: "WorkloadUpdateMethods defines the methods that can - be used to disrupt workloads during automated workload updates. - When multiple methods are present, the least disruptive method - takes precedence over more disruptive methods. For example if - both LiveMigrate and Shutdown methods are listed, only VMs which - are not live migratable will be restarted/shutdown \n An empty - list defaults to no automated workload updating" - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - workloads: - description: selectors and tolerations that should apply to KubeVirt - workloads - properties: - nodePlacement: - description: nodePlacement decsribes scheduling confiuguration - for specific KubeVirt components - properties: - affinity: - description: affinity enables pod affinity/anti-affinity placement - expanding the types of constraints that can be expressed - with nodeSelector. affinity is going to be applied to the - relevant kind of pods in parallel with nodeSelector See - https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity - properties: - nodeAffinity: - description: Describes node affinity scheduling rules - for the pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule - pods to nodes that satisfy the affinity expressions - specified by this field, but it may choose a node - that violates one or more of the expressions. The - node that is most preferred is the one with the - greatest sum of weights, i.e. for each node that - meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, - etc.), compute a sum by iterating through the elements - of this field and adding "weight" to the sum if - the node matches the corresponding matchExpressions; - the node(s) with the highest sum are the most preferred. - items: - description: An empty preferred scheduling term - matches all objects with implicit weight 0 (i.e. - it's a no-op). A null preferred scheduling term - matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated - with the corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that the - selector applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. - If the operator is Gt or Lt, the - values array must have a single - element, which will be interpreted - as an integer. This array is replaced - during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that the - selector applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. - If the operator is Gt or Lt, the - values array must have a single - element, which will be interpreted - as an integer. This array is replaced - during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - description: Weight associated with matching - the corresponding nodeSelectorTerm, in the - range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified - by this field are not met at scheduling time, the - pod will not be scheduled onto the node. If the - affinity requirements specified by this field cease - to be met at some point during pod execution (e.g. - due to an update), the system may or may not try - to eventually evict the pod from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector - terms. The terms are ORed. - items: - description: A null or empty node selector term - matches no objects. The requirements of them - are ANDed. The TopologySelectorTerm type implements - a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that the - selector applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. - If the operator is Gt or Lt, the - values array must have a single - element, which will be interpreted - as an integer. This array is replaced - during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that the - selector applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. - If the operator is Gt or Lt, the - values array must have a single - element, which will be interpreted - as an integer. This array is replaced - during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. - co-locate this pod in the same node, zone, etc. as some - other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule - pods to nodes that satisfy the affinity expressions - specified by this field, but it may choose a node - that violates one or more of the expressions. The - node that is most preferred is the one with the - greatest sum of weights, i.e. for each node that - meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, - etc.), compute a sum by iterating through the elements - of this field and adding "weight" to the sum if - the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum - are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, - associated with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of - resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The - requirements are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: operator represents - a key's relationship to a set - of values. Valid operators are - In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array - of string values. If the operator - is In or NotIn, the values array - must be non-empty. If the operator - is Exists or DoesNotExist, the - values array must be empty. - This array is replaced during - a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of - {key,value} pairs. A single {key,value} - in the matchLabels map is equivalent - to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are - ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which - namespaces the labelSelector applies to - (matches against); null or empty list - means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located - (affinity) or not co-located (anti-affinity) - with the pods matching the labelSelector - in the specified namespaces, where co-located - is defined as running on a node whose - value of the label with key topologyKey - matches that of any node on which any - of the selected pods is running. Empty - topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching - the corresponding podAffinityTerm, in the - range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified - by this field are not met at scheduling time, the - pod will not be scheduled onto the node. If the - affinity requirements specified by this field cease - to be met at some point during pod execution (e.g. - due to a pod label update), the system may or may - not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes - corresponding to each podAffinityTerm are intersected, - i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those - matching the labelSelector relative to the given - namespace(s)) that this pod should be co-located - (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node - whose value of the label with key - matches that of any node on which a pod of the - set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules - (e.g. avoid putting this pod in the same node, zone, - etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule - pods to nodes that satisfy the anti-affinity expressions - specified by this field, but it may choose a node - that violates one or more of the expressions. The - node that is most preferred is the one with the - greatest sum of weights, i.e. for each node that - meets all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity - expressions, etc.), compute a sum by iterating through - the elements of this field and adding "weight" to - the sum if the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum - are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, - associated with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of - resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The - requirements are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: operator represents - a key's relationship to a set - of values. Valid operators are - In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array - of string values. If the operator - is In or NotIn, the values array - must be non-empty. If the operator - is Exists or DoesNotExist, the - values array must be empty. - This array is replaced during - a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of - {key,value} pairs. A single {key,value} - in the matchLabels map is equivalent - to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are - ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which - namespaces the labelSelector applies to - (matches against); null or empty list - means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located - (affinity) or not co-located (anti-affinity) - with the pods matching the labelSelector - in the specified namespaces, where co-located - is defined as running on a node whose - value of the label with key topologyKey - matches that of any node on which any - of the selected pods is running. Empty - topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching - the corresponding podAffinityTerm, in the - range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements specified - by this field are not met at scheduling time, the - pod will not be scheduled onto the node. If the - anti-affinity requirements specified by this field - cease to be met at some point during pod execution - (e.g. due to a pod label update), the system may - or may not try to eventually evict the pod from - its node. When there are multiple elements, the - lists of nodes corresponding to each podAffinityTerm - are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those - matching the labelSelector relative to the given - namespace(s)) that this pod should be co-located - (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node - whose value of the label with key - matches that of any node on which a pod of the - set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - nodeSelector: - additionalProperties: - type: string - description: 'nodeSelector is the node selector applied to - the relevant kind of pods It specifies a map of key-value - pairs: for the pod to be eligible to run on a node, the - node must have each of the indicated key-value pairs as - labels (it can have additional labels as well). See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector' - type: object - tolerations: - description: tolerations is a list of tolerations applied - to the relevant kind of pods See https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ - for more info. These are additional tolerations other than - default ones. - items: - description: The pod this Toleration is attached to tolerates - any taint that matches the triple using - the matching operator . - properties: - effect: - description: Effect indicates the taint effect to match. - Empty means match all taint effects. When specified, - allowed values are NoSchedule, PreferNoSchedule and - NoExecute. - type: string - key: - description: Key is the taint key that the toleration - applies to. Empty means match all taint keys. If the - key is empty, operator must be Exists; this combination - means to match all values and all keys. - type: string - operator: - description: Operator represents a key's relationship - to the value. Valid operators are Exists and Equal. - Defaults to Equal. Exists is equivalent to wildcard - for value, so that a pod can tolerate all taints of - a particular category. - type: string - tolerationSeconds: - description: TolerationSeconds represents the period - of time the toleration (which must be of effect NoExecute, - otherwise this field is ignored) tolerates the taint. - By default, it is not set, which means tolerate the - taint forever (do not evict). Zero and negative values - will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: Value is the taint value the toleration - matches to. If the operator is Exists, the value should - be empty, otherwise just a regular string. - type: string - type: object - type: array - type: object - type: object - type: object - status: - description: KubeVirtStatus represents information pertaining to a KubeVirt - deployment. - properties: - conditions: - items: - description: KubeVirtCondition represents a condition of a KubeVirt - deployment - properties: - lastProbeTime: - format: date-time - nullable: true - type: string - lastTransitionTime: - format: date-time - nullable: true - type: string - message: - type: string - reason: - type: string - status: - type: string - type: - type: string - required: - - status - - type - type: object - type: array - generations: - items: - description: GenerationStatus keeps track of the generation for - a given resource so that decisions about forced updates can be - made. - properties: - group: - description: group is the group of the thing you're tracking - type: string - hash: - description: hash is an optional field set for resources without - generation that are content sensitive like secrets and configmaps - type: string - lastGeneration: - description: lastGeneration is the last generation of the workload - controller involved - format: int64 - type: integer - name: - description: name is the name of the thing you're tracking - type: string - namespace: - description: namespace is where the thing you're tracking is - type: string - resource: - description: resource is the resource type of the thing you're - tracking - type: string - required: - - group - - lastGeneration - - name - - resource - type: object - type: array - x-kubernetes-list-type: atomic - observedDeploymentConfig: - type: string - observedDeploymentID: - type: string - observedKubeVirtRegistry: - type: string - observedKubeVirtVersion: - type: string - operatorVersion: - type: string - outdatedVirtualMachineInstanceWorkloads: - type: integer - phase: - description: KubeVirtPhase is a label for the phase of a KubeVirt - deployment at the current time. - type: string - targetDeploymentConfig: - type: string - targetDeploymentID: - type: string - targetKubeVirtRegistry: - type: string - targetKubeVirtVersion: - type: string - type: object - required: - - spec - type: object - served: true - storage: false - subresources: - status: {} - - additionalPrinterColumns: - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - - jsonPath: .status.phase - name: Phase - type: string - name: v1alpha3 - schema: - openAPIV3Schema: - description: KubeVirt represents the object deploying all KubeVirt resources - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - certificateRotateStrategy: - properties: - selfSigned: - properties: - ca: - description: CA configuration CA certs are kept in the CA - bundle as long as they are valid - properties: - duration: - description: The requested 'duration' (i.e. lifetime) - of the Certificate. - type: string - renewBefore: - description: The amount of time before the currently issued - certificate's "notAfter" time that we will begin to - attempt to renew the certificate. - type: string - type: object - caOverlapInterval: - description: Deprecated. Use CA.Duration and CA.RenewBefore - instead - type: string - caRotateInterval: - description: Deprecated. Use CA.Duration instead - type: string - certRotateInterval: - description: Deprecated. Use Server.Duration instead - type: string - server: - description: Server configuration Certs are rotated and discarded - properties: - duration: - description: The requested 'duration' (i.e. lifetime) - of the Certificate. - type: string - renewBefore: - description: The amount of time before the currently issued - certificate's "notAfter" time that we will begin to - attempt to renew the certificate. - type: string - type: object - type: object - type: object - configuration: - description: holds kubevirt configurations. same as the virt-configMap - properties: - cpuModel: - type: string - cpuRequest: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - defaultRuntimeClass: - type: string - developerConfiguration: - description: DeveloperConfiguration holds developer options - properties: - cpuAllocationRatio: - type: integer - diskVerification: - description: DiskVerification holds container disks verification - limits - properties: - memoryLimit: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - memoryLimit - type: object - featureGates: - items: - type: string - type: array - logVerbosity: - description: LogVerbosity sets log verbosity level of various - components - properties: - nodeVerbosity: - additionalProperties: - type: integer - description: NodeVerbosity represents a map of nodes with - a specific verbosity level - type: object - virtAPI: - type: integer - virtController: - type: integer - virtHandler: - type: integer - virtLauncher: - type: integer - virtOperator: - type: integer - type: object - memoryOvercommit: - type: integer - minimumClusterTSCFrequency: - description: Allow overriding the automatically determined - minimum TSC frequency of the cluster and fixate the minimum - to this frequency. - format: int64 - type: integer - minimumReservePVCBytes: - format: int64 - type: integer - nodeSelectors: - additionalProperties: - type: string - type: object - pvcTolerateLessSpaceUpToPercent: - type: integer - useEmulation: - type: boolean - type: object - emulatedMachines: - items: - type: string - type: array - imagePullPolicy: - description: PullPolicy describes a policy for if/when to pull - a container image - type: string - machineType: - type: string - memBalloonStatsPeriod: - format: int32 - type: integer - migrations: - description: MigrationConfiguration holds migration options - properties: - allowAutoConverge: - type: boolean - allowPostCopy: - type: boolean - bandwidthPerMigration: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - completionTimeoutPerGiB: - format: int64 - type: integer - disableTLS: - type: boolean - nodeDrainTaintKey: - type: string - parallelMigrationsPerCluster: - format: int32 - type: integer - parallelOutboundMigrationsPerNode: - format: int32 - type: integer - progressTimeout: - format: int64 - type: integer - unsafeMigrationOverride: - type: boolean - type: object - minCPUModel: - type: string - network: - description: NetworkConfiguration holds network options - properties: - defaultNetworkInterface: - type: string - permitBridgeInterfaceOnPodNetwork: - type: boolean - permitSlirpInterface: - type: boolean - type: object - obsoleteCPUModels: - additionalProperties: - type: boolean - type: object - ovmfPath: - type: string - permittedHostDevices: - description: PermittedHostDevices holds inforamtion about devices - allowed for passthrough - properties: - mediatedDevices: - items: - description: MediatedHostDevice represents a host mediated - device allowed for passthrough - properties: - externalResourceProvider: - type: boolean - mdevNameSelector: - type: string - resourceName: - type: string - required: - - mdevNameSelector - - resourceName - type: object - type: array - x-kubernetes-list-type: atomic - pciHostDevices: - items: - description: PciHostDevice represents a host PCI device - allowed for passthrough - properties: - externalResourceProvider: - description: If true, KubeVirt will leave the allocation - and monitoring to an external device plugin - type: boolean - pciVendorSelector: - description: The vendor_id:product_id tuple of the PCI - device - type: string - resourceName: - description: The name of the resource that is representing - the device. Exposed by a device plugin and requested - by VMs. Typically of the form vendor.com/product_nameThe - name of the resource that is representing the device. - Exposed by a device plugin and requested by VMs. Typically - of the form vendor.com/product_name - type: string - required: - - pciVendorSelector - - resourceName - type: object - type: array - x-kubernetes-list-type: atomic - type: object - selinuxLauncherType: - type: string - smbios: - properties: - family: - type: string - manufacturer: - type: string - product: - type: string - sku: - type: string - version: - type: string - type: object - supportedGuestAgentVersions: - description: deprecated - items: - type: string - type: array - virtualMachineInstancesPerNode: - type: integer - type: object - customizeComponents: - properties: - flags: - description: Configure the value used for deployment and daemonset - resources - properties: - api: - additionalProperties: - type: string - type: object - controller: - additionalProperties: - type: string - type: object - handler: - additionalProperties: - type: string - type: object - type: object - patches: - items: - properties: - patch: - type: string - resourceName: - minLength: 1 - type: string - resourceType: - minLength: 1 - type: string - type: - type: string - required: - - patch - - resourceName - - resourceType - - type - type: object - type: array - x-kubernetes-list-type: atomic - type: object - imagePullPolicy: - description: The ImagePullPolicy to use. - type: string - imageRegistry: - description: The image registry to pull the container images from - Defaults to the same registry the operator's container image is - pulled from. - type: string - imageTag: - description: The image tag to use for the continer images installed. - Defaults to the same tag as the operator's container image. - type: string - infra: - description: selectors and tolerations that should apply to KubeVirt - infrastructure components - properties: - nodePlacement: - description: nodePlacement decsribes scheduling confiuguration - for specific KubeVirt components - properties: - affinity: - description: affinity enables pod affinity/anti-affinity placement - expanding the types of constraints that can be expressed - with nodeSelector. affinity is going to be applied to the - relevant kind of pods in parallel with nodeSelector See - https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity - properties: - nodeAffinity: - description: Describes node affinity scheduling rules - for the pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule - pods to nodes that satisfy the affinity expressions - specified by this field, but it may choose a node - that violates one or more of the expressions. The - node that is most preferred is the one with the - greatest sum of weights, i.e. for each node that - meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, - etc.), compute a sum by iterating through the elements - of this field and adding "weight" to the sum if - the node matches the corresponding matchExpressions; - the node(s) with the highest sum are the most preferred. - items: - description: An empty preferred scheduling term - matches all objects with implicit weight 0 (i.e. - it's a no-op). A null preferred scheduling term - matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated - with the corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that the - selector applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. - If the operator is Gt or Lt, the - values array must have a single - element, which will be interpreted - as an integer. This array is replaced - during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that the - selector applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. - If the operator is Gt or Lt, the - values array must have a single - element, which will be interpreted - as an integer. This array is replaced - during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - description: Weight associated with matching - the corresponding nodeSelectorTerm, in the - range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified - by this field are not met at scheduling time, the - pod will not be scheduled onto the node. If the - affinity requirements specified by this field cease - to be met at some point during pod execution (e.g. - due to an update), the system may or may not try - to eventually evict the pod from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector - terms. The terms are ORed. - items: - description: A null or empty node selector term - matches no objects. The requirements of them - are ANDed. The TopologySelectorTerm type implements - a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that the - selector applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. - If the operator is Gt or Lt, the - values array must have a single - element, which will be interpreted - as an integer. This array is replaced - during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that the - selector applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. - If the operator is Gt or Lt, the - values array must have a single - element, which will be interpreted - as an integer. This array is replaced - during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. - co-locate this pod in the same node, zone, etc. as some - other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule - pods to nodes that satisfy the affinity expressions - specified by this field, but it may choose a node - that violates one or more of the expressions. The - node that is most preferred is the one with the - greatest sum of weights, i.e. for each node that - meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, - etc.), compute a sum by iterating through the elements - of this field and adding "weight" to the sum if - the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum - are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, - associated with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of - resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The - requirements are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: operator represents - a key's relationship to a set - of values. Valid operators are - In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array - of string values. If the operator - is In or NotIn, the values array - must be non-empty. If the operator - is Exists or DoesNotExist, the - values array must be empty. - This array is replaced during - a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of - {key,value} pairs. A single {key,value} - in the matchLabels map is equivalent - to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are - ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which - namespaces the labelSelector applies to - (matches against); null or empty list - means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located - (affinity) or not co-located (anti-affinity) - with the pods matching the labelSelector - in the specified namespaces, where co-located - is defined as running on a node whose - value of the label with key topologyKey - matches that of any node on which any - of the selected pods is running. Empty - topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching - the corresponding podAffinityTerm, in the - range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified - by this field are not met at scheduling time, the - pod will not be scheduled onto the node. If the - affinity requirements specified by this field cease - to be met at some point during pod execution (e.g. - due to a pod label update), the system may or may - not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes - corresponding to each podAffinityTerm are intersected, - i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those - matching the labelSelector relative to the given - namespace(s)) that this pod should be co-located - (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node - whose value of the label with key - matches that of any node on which a pod of the - set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules - (e.g. avoid putting this pod in the same node, zone, - etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule - pods to nodes that satisfy the anti-affinity expressions - specified by this field, but it may choose a node - that violates one or more of the expressions. The - node that is most preferred is the one with the - greatest sum of weights, i.e. for each node that - meets all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity - expressions, etc.), compute a sum by iterating through - the elements of this field and adding "weight" to - the sum if the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum - are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, - associated with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of - resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The - requirements are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: operator represents - a key's relationship to a set - of values. Valid operators are - In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array - of string values. If the operator - is In or NotIn, the values array - must be non-empty. If the operator - is Exists or DoesNotExist, the - values array must be empty. - This array is replaced during - a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of - {key,value} pairs. A single {key,value} - in the matchLabels map is equivalent - to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are - ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which - namespaces the labelSelector applies to - (matches against); null or empty list - means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located - (affinity) or not co-located (anti-affinity) - with the pods matching the labelSelector - in the specified namespaces, where co-located - is defined as running on a node whose - value of the label with key topologyKey - matches that of any node on which any - of the selected pods is running. Empty - topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching - the corresponding podAffinityTerm, in the - range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements specified - by this field are not met at scheduling time, the - pod will not be scheduled onto the node. If the - anti-affinity requirements specified by this field - cease to be met at some point during pod execution - (e.g. due to a pod label update), the system may - or may not try to eventually evict the pod from - its node. When there are multiple elements, the - lists of nodes corresponding to each podAffinityTerm - are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those - matching the labelSelector relative to the given - namespace(s)) that this pod should be co-located - (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node - whose value of the label with key - matches that of any node on which a pod of the - set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - nodeSelector: - additionalProperties: - type: string - description: 'nodeSelector is the node selector applied to - the relevant kind of pods It specifies a map of key-value - pairs: for the pod to be eligible to run on a node, the - node must have each of the indicated key-value pairs as - labels (it can have additional labels as well). See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector' - type: object - tolerations: - description: tolerations is a list of tolerations applied - to the relevant kind of pods See https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ - for more info. These are additional tolerations other than - default ones. - items: - description: The pod this Toleration is attached to tolerates - any taint that matches the triple using - the matching operator . - properties: - effect: - description: Effect indicates the taint effect to match. - Empty means match all taint effects. When specified, - allowed values are NoSchedule, PreferNoSchedule and - NoExecute. - type: string - key: - description: Key is the taint key that the toleration - applies to. Empty means match all taint keys. If the - key is empty, operator must be Exists; this combination - means to match all values and all keys. - type: string - operator: - description: Operator represents a key's relationship - to the value. Valid operators are Exists and Equal. - Defaults to Equal. Exists is equivalent to wildcard - for value, so that a pod can tolerate all taints of - a particular category. - type: string - tolerationSeconds: - description: TolerationSeconds represents the period - of time the toleration (which must be of effect NoExecute, - otherwise this field is ignored) tolerates the taint. - By default, it is not set, which means tolerate the - taint forever (do not evict). Zero and negative values - will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: Value is the taint value the toleration - matches to. If the operator is Exists, the value should - be empty, otherwise just a regular string. - type: string - type: object - type: array - type: object - type: object - monitorAccount: - description: The name of the Prometheus service account that needs - read-access to KubeVirt endpoints Defaults to prometheus-k8s - type: string - monitorNamespace: - description: The namespace Prometheus is deployed in Defaults to openshift-monitor - type: string - productName: - description: Designate the apps.kubevirt.io/part-of label for KubeVirt - components. Useful if KubeVirt is included as part of a product. - If ProductName is not specified, the part-of label will be omitted. - type: string - productVersion: - description: Designate the apps.kubevirt.io/version label for KubeVirt - components. Useful if KubeVirt is included as part of a product. - If ProductVersion is not specified, KubeVirt's version will be used. - type: string - uninstallStrategy: - description: Specifies if kubevirt can be deleted if workloads are - still present. This is mainly a precaution to avoid accidental data - loss - type: string - workloadUpdateStrategy: - description: WorkloadUpdateStrategy defines at the cluster level how - to handle automated workload updates - properties: - batchEvictionInterval: - description: "BatchEvictionInterval Represents the interval to - wait before issuing the next batch of shutdowns \n Defaults - to 1 minute" - type: string - batchEvictionSize: - description: "BatchEvictionSize Represents the number of VMIs - that can be forced updated per the BatchShutdownInteral interval - \n Defaults to 10" - type: integer - workloadUpdateMethods: - description: "WorkloadUpdateMethods defines the methods that can - be used to disrupt workloads during automated workload updates. - When multiple methods are present, the least disruptive method - takes precedence over more disruptive methods. For example if - both LiveMigrate and Shutdown methods are listed, only VMs which - are not live migratable will be restarted/shutdown \n An empty - list defaults to no automated workload updating" - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - workloads: - description: selectors and tolerations that should apply to KubeVirt - workloads - properties: - nodePlacement: - description: nodePlacement decsribes scheduling confiuguration - for specific KubeVirt components - properties: - affinity: - description: affinity enables pod affinity/anti-affinity placement - expanding the types of constraints that can be expressed - with nodeSelector. affinity is going to be applied to the - relevant kind of pods in parallel with nodeSelector See - https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity - properties: - nodeAffinity: - description: Describes node affinity scheduling rules - for the pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule - pods to nodes that satisfy the affinity expressions - specified by this field, but it may choose a node - that violates one or more of the expressions. The - node that is most preferred is the one with the - greatest sum of weights, i.e. for each node that - meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, - etc.), compute a sum by iterating through the elements - of this field and adding "weight" to the sum if - the node matches the corresponding matchExpressions; - the node(s) with the highest sum are the most preferred. - items: - description: An empty preferred scheduling term - matches all objects with implicit weight 0 (i.e. - it's a no-op). A null preferred scheduling term - matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated - with the corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that the - selector applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. - If the operator is Gt or Lt, the - values array must have a single - element, which will be interpreted - as an integer. This array is replaced - during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that the - selector applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. - If the operator is Gt or Lt, the - values array must have a single - element, which will be interpreted - as an integer. This array is replaced - during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - description: Weight associated with matching - the corresponding nodeSelectorTerm, in the - range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified - by this field are not met at scheduling time, the - pod will not be scheduled onto the node. If the - affinity requirements specified by this field cease - to be met at some point during pod execution (e.g. - due to an update), the system may or may not try - to eventually evict the pod from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector - terms. The terms are ORed. - items: - description: A null or empty node selector term - matches no objects. The requirements of them - are ANDed. The TopologySelectorTerm type implements - a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that the - selector applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. - If the operator is Gt or Lt, the - values array must have a single - element, which will be interpreted - as an integer. This array is replaced - during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that the - selector applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. - If the operator is Gt or Lt, the - values array must have a single - element, which will be interpreted - as an integer. This array is replaced - during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. - co-locate this pod in the same node, zone, etc. as some - other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule - pods to nodes that satisfy the affinity expressions - specified by this field, but it may choose a node - that violates one or more of the expressions. The - node that is most preferred is the one with the - greatest sum of weights, i.e. for each node that - meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, - etc.), compute a sum by iterating through the elements - of this field and adding "weight" to the sum if - the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum - are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, - associated with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of - resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The - requirements are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: operator represents - a key's relationship to a set - of values. Valid operators are - In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array - of string values. If the operator - is In or NotIn, the values array - must be non-empty. If the operator - is Exists or DoesNotExist, the - values array must be empty. - This array is replaced during - a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of - {key,value} pairs. A single {key,value} - in the matchLabels map is equivalent - to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are - ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which - namespaces the labelSelector applies to - (matches against); null or empty list - means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located - (affinity) or not co-located (anti-affinity) - with the pods matching the labelSelector - in the specified namespaces, where co-located - is defined as running on a node whose - value of the label with key topologyKey - matches that of any node on which any - of the selected pods is running. Empty - topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching - the corresponding podAffinityTerm, in the - range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified - by this field are not met at scheduling time, the - pod will not be scheduled onto the node. If the - affinity requirements specified by this field cease - to be met at some point during pod execution (e.g. - due to a pod label update), the system may or may - not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes - corresponding to each podAffinityTerm are intersected, - i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those - matching the labelSelector relative to the given - namespace(s)) that this pod should be co-located - (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node - whose value of the label with key - matches that of any node on which a pod of the - set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules - (e.g. avoid putting this pod in the same node, zone, - etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule - pods to nodes that satisfy the anti-affinity expressions - specified by this field, but it may choose a node - that violates one or more of the expressions. The - node that is most preferred is the one with the - greatest sum of weights, i.e. for each node that - meets all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity - expressions, etc.), compute a sum by iterating through - the elements of this field and adding "weight" to - the sum if the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum - are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, - associated with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of - resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The - requirements are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: operator represents - a key's relationship to a set - of values. Valid operators are - In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array - of string values. If the operator - is In or NotIn, the values array - must be non-empty. If the operator - is Exists or DoesNotExist, the - values array must be empty. - This array is replaced during - a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of - {key,value} pairs. A single {key,value} - in the matchLabels map is equivalent - to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are - ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which - namespaces the labelSelector applies to - (matches against); null or empty list - means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located - (affinity) or not co-located (anti-affinity) - with the pods matching the labelSelector - in the specified namespaces, where co-located - is defined as running on a node whose - value of the label with key topologyKey - matches that of any node on which any - of the selected pods is running. Empty - topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching - the corresponding podAffinityTerm, in the - range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements specified - by this field are not met at scheduling time, the - pod will not be scheduled onto the node. If the - anti-affinity requirements specified by this field - cease to be met at some point during pod execution - (e.g. due to a pod label update), the system may - or may not try to eventually evict the pod from - its node. When there are multiple elements, the - lists of nodes corresponding to each podAffinityTerm - are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those - matching the labelSelector relative to the given - namespace(s)) that this pod should be co-located - (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node - whose value of the label with key - matches that of any node on which a pod of the - set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - nodeSelector: - additionalProperties: - type: string - description: 'nodeSelector is the node selector applied to - the relevant kind of pods It specifies a map of key-value - pairs: for the pod to be eligible to run on a node, the - node must have each of the indicated key-value pairs as - labels (it can have additional labels as well). See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector' - type: object - tolerations: - description: tolerations is a list of tolerations applied - to the relevant kind of pods See https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ - for more info. These are additional tolerations other than - default ones. - items: - description: The pod this Toleration is attached to tolerates - any taint that matches the triple using - the matching operator . - properties: - effect: - description: Effect indicates the taint effect to match. - Empty means match all taint effects. When specified, - allowed values are NoSchedule, PreferNoSchedule and - NoExecute. - type: string - key: - description: Key is the taint key that the toleration - applies to. Empty means match all taint keys. If the - key is empty, operator must be Exists; this combination - means to match all values and all keys. - type: string - operator: - description: Operator represents a key's relationship - to the value. Valid operators are Exists and Equal. - Defaults to Equal. Exists is equivalent to wildcard - for value, so that a pod can tolerate all taints of - a particular category. - type: string - tolerationSeconds: - description: TolerationSeconds represents the period - of time the toleration (which must be of effect NoExecute, - otherwise this field is ignored) tolerates the taint. - By default, it is not set, which means tolerate the - taint forever (do not evict). Zero and negative values - will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: Value is the taint value the toleration - matches to. If the operator is Exists, the value should - be empty, otherwise just a regular string. - type: string - type: object - type: array - type: object - type: object - type: object - status: - description: KubeVirtStatus represents information pertaining to a KubeVirt - deployment. - properties: - conditions: - items: - description: KubeVirtCondition represents a condition of a KubeVirt - deployment - properties: - lastProbeTime: - format: date-time - nullable: true - type: string - lastTransitionTime: - format: date-time - nullable: true - type: string - message: - type: string - reason: - type: string - status: - type: string - type: - type: string - required: - - status - - type - type: object - type: array - generations: - items: - description: GenerationStatus keeps track of the generation for - a given resource so that decisions about forced updates can be - made. - properties: - group: - description: group is the group of the thing you're tracking - type: string - hash: - description: hash is an optional field set for resources without - generation that are content sensitive like secrets and configmaps - type: string - lastGeneration: - description: lastGeneration is the last generation of the workload - controller involved - format: int64 - type: integer - name: - description: name is the name of the thing you're tracking - type: string - namespace: - description: namespace is where the thing you're tracking is - type: string - resource: - description: resource is the resource type of the thing you're - tracking - type: string - required: - - group - - lastGeneration - - name - - resource - type: object - type: array - x-kubernetes-list-type: atomic - observedDeploymentConfig: - type: string - observedDeploymentID: - type: string - observedKubeVirtRegistry: - type: string - observedKubeVirtVersion: - type: string - operatorVersion: - type: string - outdatedVirtualMachineInstanceWorkloads: - type: integer - phase: - description: KubeVirtPhase is a label for the phase of a KubeVirt - deployment at the current time. - type: string - targetDeploymentConfig: - type: string - targetDeploymentID: - type: string - targetKubeVirtRegistry: - type: string - targetKubeVirtVersion: - type: string - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: kubevirt.io:operator - labels: - operator.kubevirt.io: "" - rbac.authorization.k8s.io/aggregate-to-admin: "true" -rules: - - apiGroups: - - kubevirt.io - resources: - - kubevirts - verbs: - - get - - delete - - create - - update - - patch - - list - - watch - - deletecollection ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - kubevirt.io: "" - name: kubevirt-operator - namespace: kubevirt ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - kubevirt.io: "" - name: kubevirt-operator -rules: - - apiGroups: - - kubevirt.io - resources: - - kubevirts - verbs: - - get - - list - - watch - - patch - - update - - patch - - apiGroups: - - "" - resources: - - serviceaccounts - - services - - endpoints - verbs: - - get - - list - - watch - - create - - update - - delete - - patch - - apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - watch - - create - - delete - - patch - - apiGroups: - - apps - resources: - - deployments - - daemonsets - verbs: - - get - - list - - watch - - create - - delete - - patch - - apiGroups: - - rbac.authorization.k8s.io - resources: - - clusterroles - - clusterrolebindings - - roles - - rolebindings - verbs: - - get - - list - - watch - - create - - delete - - patch - - update - - apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - get - - list - - watch - - create - - delete - - patch - - apiGroups: - - security.openshift.io - resources: - - securitycontextconstraints - verbs: - - get - - list - - watch - - apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - get - - patch - - update - - apiGroups: - - admissionregistration.k8s.io - resources: - - validatingwebhookconfigurations - verbs: - - get - - list - - watch - - create - - delete - - apiGroups: - - admissionregistration.k8s.io - resources: - - validatingwebhookconfigurations - - mutatingwebhookconfigurations - verbs: - - get - - create - - update - - apiGroups: - - apiregistration.k8s.io - resources: - - apiservices - verbs: - - get - - create - - update - - apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - apiGroups: - - "" - resources: - - pods/exec - verbs: - - create - - apiGroups: - - kubevirt.io - resources: - - virtualmachines - - virtualmachineinstances - - virtualmachineinstancemigrations - verbs: - - get - - list - - watch - - patch - - apiGroups: - - kubevirt.io - resources: - - virtualmachineinstancepresets - verbs: - - watch - - list - - apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - limitranges - verbs: - - watch - - list - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - delete - - update - - create - - apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - apiGroups: - - policy - resources: - - poddisruptionbudgets - verbs: - - get - - list - - watch - - delete - - create - - apiGroups: - - "" - resources: - - pods - - configmaps - - endpoints - verbs: - - get - - list - - watch - - delete - - update - - create - - apiGroups: - - "" - resources: - - events - verbs: - - update - - create - - patch - - apiGroups: - - "" - resources: - - pods/finalizers - verbs: - - update - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch - - update - - patch - - apiGroups: - - "" - resources: - - persistentvolumeclaims - verbs: - - get - - list - - watch - - apiGroups: - - kubevirt.io - resources: - - '*' - verbs: - - '*' - - apiGroups: - - cdi.kubevirt.io - resources: - - '*' - verbs: - - '*' - - apiGroups: - - k8s.cni.cncf.io - resources: - - network-attachment-definitions - verbs: - - get - - list - - watch - - apiGroups: - - kubevirt.io - resources: - - virtualmachineinstances - verbs: - - update - - list - - watch - - apiGroups: - - "" - resources: - - secrets - - persistentvolumeclaims - verbs: - - get - - apiGroups: - - "" - resources: - - nodes - verbs: - - patch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - secrets - verbs: - - create - - apiGroups: - - subresources.kubevirt.io - resources: - - version - verbs: - - get - - list - - apiGroups: - - subresources.kubevirt.io - resources: - - virtualmachineinstances/console - - virtualmachineinstances/vnc - verbs: - - get - - apiGroups: - - subresources.kubevirt.io - resources: - - virtualmachines/start - - virtualmachines/stop - - virtualmachines/restart - verbs: - - update - - put - - apiGroups: - - kubevirt.io - resources: - - virtualmachines - - virtualmachineinstances - - virtualmachineinstancepresets - - virtualmachineinstancereplicasets - - virtualmachineinstancemigrations - verbs: - - get - - delete - - create - - update - - patch - - list - - watch - - deletecollection - - apiGroups: - - subresources.kubevirt.io - resources: - - virtualmachineinstances/console - - virtualmachineinstances/vnc - verbs: - - get - - apiGroups: - - subresources.kubevirt.io - resources: - - virtualmachines/start - - virtualmachines/stop - - virtualmachines/restart - verbs: - - update - - put - - apiGroups: - - kubevirt.io - resources: - - virtualmachines - - virtualmachineinstances - - virtualmachineinstancepresets - - virtualmachineinstancereplicasets - - virtualmachineinstancemigrations - verbs: - - get - - delete - - create - - update - - patch - - list - - watch - - apiGroups: - - kubevirt.io - resources: - - virtualmachines - - virtualmachineinstances - - virtualmachineinstancepresets - - virtualmachineinstancereplicasets - - virtualmachineinstancemigrations - verbs: - - get - - list - - watch - - apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create - - apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - kubevirt.io: "" - name: kubevirt-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kubevirt-operator -subjects: - - kind: ServiceAccount - name: kubevirt-operator - namespace: kubevirt - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - kubevirt.io: virt-operator - name: virt-operator - namespace: kubevirt -spec: - replicas: 2 - selector: - matchLabels: - kubevirt.io: virt-operator - strategy: - type: RollingUpdate - template: - metadata: - annotations: - scheduler.alpha.kubernetes.io/critical-pod: "" - scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly","operator":"Exists"}]' - labels: - kubevirt.io: virt-operator - prometheus.kubevirt.io: "" - name: virt-operator - spec: - containers: - - command: - - virt-operator - - --port - - "8443" - - -v - - "2" - env: - - name: OPERATOR_IMAGE - value: index.docker.io/kubevirt/virt-operator:v0.19.0-rc.0 - - name: WATCH_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.annotations['olm.targetNamespaces'] - image: index.docker.io/kubevirt/virt-operator:v0.19.0-rc.0 - imagePullPolicy: IfNotPresent - name: virt-operator - ports: - - containerPort: 8443 - name: metrics - protocol: TCP - readinessProbe: - httpGet: - path: /metrics - port: 8443 - scheme: HTTPS - initialDelaySeconds: 5 - timeoutSeconds: 10 - resources: {} - securityContext: - runAsNonRoot: true - serviceAccountName: kubevirt-operator diff --git a/examples/linode-machinedeployment.yaml b/examples/linode-machinedeployment.yaml deleted file mode 100644 index f00928ac4..000000000 --- a/examples/linode-machinedeployment.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - # If you change the namespace/name, you must also - # adjust the rbac rules - name: machine-controller-linode - namespace: kube-system -type: Opaque -stringData: - token: << LINODE_TOKEN >> ---- -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: linode-machinedeployment - namespace: kube-system -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - foo: bar - template: - metadata: - labels: - foo: bar - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "linode" - cloudProviderSpec: - # If empty, can be set via LINODE_TOKEN env var - token: - secretKeyRef: - namespace: kube-system - name: machine-controller-linode - key: token - region: eu-west - type: g6-standard-2 - backups: false - private_networking: true - tags: - - "machine-controller" - operatingSystem: "ubuntu" - operatingSystemSpec: - disableAutoUpdate: true - versions: - kubelet: 1.28.5 diff --git a/examples/nutanix-machinedeployment.yaml b/examples/nutanix-machinedeployment.yaml deleted file mode 100644 index bb7e863a6..000000000 --- a/examples/nutanix-machinedeployment.yaml +++ /dev/null @@ -1,85 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - # If you change the namespace/name, you must also - # adjust the rbac rules - name: machine-controller-nutanix - namespace: kube-system -type: Opaque -stringData: - password: << NUTANIX_PASSWORD >> ---- -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: nutanix-machinedeployment - namespace: kube-system -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - foo: bar - template: - metadata: - labels: - foo: bar - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "nutanix" - cloudProviderSpec: - # Can also be set via the env var 'NUTANIX_USERNAME' on the machine-controller - username: "<< NUTANIX_USERNAME >>" - # Can also be set via the env var 'NUTANIX_ENDPOINT' on the machine-controller - # example: 'your-nutanix-host' or '10.0.1.5'. No protocol or port should be passed. - endpoint: "<< NUTANIX_ENDPOINT >>" - # Can also be set via the env var 'NUTANIX_PORT' on the machine-controller - # if not set, defaults to 9440 (default Nutanix port) - port: "<< NUTANIX_PORT >>" - # Optional: Allow insecure connections to endpoint if no valid TLS certificate is presented - allowInsecure: true - # Can also be set via the env var 'NUTANIX_PASSWORD' on the machine-controller - password: - secretKeyRef: - namespace: kube-system - name: machine-controller-nutanix - key: password - # Can also be set via the env var 'NUTANIX_CLUSTER_NAME' on the machine-controller - # this refers to a Nutanix cluster, not a Kubernetes cluster - clusterName: nutanix-cluster - # Optional: Sets the project that the VM is deployed into. If none is provided, the VM will be created without a project - projectName: project1 - # Sets the subnet that the VM is connected to. Must exist in the given Nutanix cluster - subnetName: subnet1 - # Optional: Sets multiple additional subnets that the VM is connected to. Must exist in the given Nutanix cluster - # additionalSubnetNames: - # - subnet2 - # - subnet3 - # Provides the image used to create the VM - imageName: ubuntu-20.04 - # Sets the vCPU count for this VM - cpus: 2 - # Sets the CPU cores per vCPUs - cpuCores: 1 - # Memory configuration in MiB - memoryMB: 2048 - # Optional: Enable Nutanix' CPU passthrough feature - #cpuPassthrough: true - # Optional: Set up system disk size in GB. If not set, will be based on image size. - # Cannot be smaller than the image size. - diskSize: 20 - operatingSystem: "ubuntu" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - versions: - kubelet: 1.28.5 diff --git a/examples/opennebula-machinedeployment.yaml b/examples/opennebula-machinedeployment.yaml deleted file mode 100644 index ab4309da7..000000000 --- a/examples/opennebula-machinedeployment.yaml +++ /dev/null @@ -1,70 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - # If you change the namespace/name, you must also - # adjust the rbac rules - name: machine-controller-opennebula - namespace: kube-system -type: Opaque -stringData: - password: << ONE_PASSWORD >> ---- -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: opennebula-machinedeployment - namespace: kube-system -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - foo: bar - template: - metadata: - labels: - foo: bar - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "opennebula" - cloudProviderSpec: - endpoint: "<< ONE_ENDPOINT including '/RPC2' >>" - username: "<< ONE_USERNAME >>" - # If empty, can be set via ONE_PASSWORD env var - password: - secretKeyRef: - namespace: kube-system - name: machine-controller-opennebula - key: password - cpu: 1 - vcpu: 2 - memory: 1024 - - image: "flatcar-stable" - datastore: "<< YOUR_DATASTORE_NAME >>" - diskSize: 51200 # MB - - network: "<< YOUR_NETWORK_NAME >>" - - enableVNC: true - - # if you want to have more control over e.g. placement of the VM you can do this: - #vmTemplateExtra: - # SCHED_REQUIREMENTS: 'RACK="G4"' - operatingSystem: "flatcar" - operatingSystemSpec: - distUpgradeOnBoot: false - - # use cloud-init for flatcar as ignition doesn't know anything about OpenNebula yet - provisioningUtility: "cloud-init" - versions: - kubelet: 1.28.5 diff --git a/examples/operating-system-manager.yaml b/examples/operating-system-manager.yaml index 46aa6ec25..66364fda2 100644 --- a/examples/operating-system-manager.yaml +++ b/examples/operating-system-manager.yaml @@ -199,25 +199,10 @@ spec: description: Name represents the name of the supported cloud provider enum: - aws - - azure - - digitalocean - - edge - - gce - - hetzner - - kubevirt - - linode - - nutanix - openstack - - equinixmetal - - vsphere - fake - - alibaba - - anexia - - scaleway - baremetal - external - - vmware-cloud-director - - opennebula type: string spec: description: @@ -940,25 +925,10 @@ spec: provider enum: - aws - - azure - - digitalocean - - edge - - gce - - hetzner - - kubevirt - - linode - - nutanix - openstack - - equinixmetal - - vsphere - fake - - alibaba - - anexia - - scaleway - baremetal - external - - vmware-cloud-director - - opennebula type: string spec: description: diff --git a/examples/scaleway-machinedeployment.yaml b/examples/scaleway-machinedeployment.yaml deleted file mode 100644 index b996f3f0c..000000000 --- a/examples/scaleway-machinedeployment.yaml +++ /dev/null @@ -1,64 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - # If you change the namespace/name, you must also - # adjust the rbac rules - name: machine-controller-scaleway - namespace: kube-system -type: Opaque -stringData: - access_key: << SCW_ACCESS_KEY >> - secret_key: << SCW_SECRET_KEY >> ---- -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: scaleway-machinedeployment - namespace: kube-system -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - foo: bar - template: - metadata: - labels: - foo: bar - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "scaleway" - cloudProviderSpec: - # If empty, can be set via SCW_ACCESS_KEY env var - accessKey: - secretKeyRef: - namespace: kube-system - name: machine-controller-scaleway - key: access_key - # If empty, can be set via SCW_SECRET_KEY env var - secretKey: - secretKeyRef: - namespace: kube-system - name: machine-controller-scaleway - key: secret_key - projectId: << SCW_DEFAULT_PROJECT_ID >> - zone: fr-par-1 - commercialType: DEV1-M - ipv6: false - tags: - - "machine-controller" - # Can be 'ubuntu' or 'centos' - operatingSystem: "ubuntu" - operatingSystemSpec: - disableAutoUpdate: true - versions: - kubelet: 1.28.5 diff --git a/examples/vmware-cloud-director-machinedeployment.yaml b/examples/vmware-cloud-director-machinedeployment.yaml deleted file mode 100644 index 40d076a8e..000000000 --- a/examples/vmware-cloud-director-machinedeployment.yaml +++ /dev/null @@ -1,92 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - # If you change the namespace/name, you must also - # adjust the rbac rules - name: machine-controller-vmware-cloud-director - namespace: kube-system -type: Opaque -stringData: - password: << VCD_PASSWORD >> ---- -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: vmware-cloud-director-machinedeployment - namespace: kube-system -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - foo: bar - template: - metadata: - labels: - foo: bar - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "vmware-cloud-director" - cloudProviderSpec: - # Can also be set via the env var 'VCD_USER' on the machine-controller - username: "<< VCD_USER >>" - # Can also be set via the env var 'VCD_URL' on the machine-controller - # example: 'https://your-vcloud-director:8443'. '/api' gets appended automatically - url: "<< VCD_URL >>" - # Can also be set via the env var 'VCD_PASSWORD' on the machine-controller - password: - secretKeyRef: - namespace: kube-system - name: machine-controller-vmware-cloud-director - key: password - # Can also be set via the env var 'VCD_API_TOKEN' on the machine-controller - # Either username, password or apiToken should be used for authentication. - apiToken: - secretKeyRef: - namespace: kube-system - name: machine-controller-vmware-cloud-director - key: apiToken - # Can also be set via the env var 'VCD_ORG' on the machine-controller - organization: "<< VCD_ORG >>" - # Can also be set via the env var 'VCD_VDC' on the machine-controller - vdc: "<< VCD_VDC >>" - # Can also be set via the env var 'VCD_ALLOW_UNVERIFIED_SSL' on the machine-controller - allowInsecure: false - # vApp to associate the VM with. This should be created before the machine is created - vapp: "<< VCD_VAPP >>" - # Name of catalog where the VM template is located - catalog: "<< VCD_CATALOG >>" - # Name of OS template to be used for the VM - template: "<< VCD_TEMPLATE >>" - # Direct or routed network that can be used for the VM - network: "<< VCD_NETWORK >>" - ipAllocationMode: "DHCP" - cpus: 2 - cpuCores: 1 - memoryMB: 2048 - # Optional: Resize the root disk to this size. Must be bigger than the existing size - # Default is to leave the disk at the same size as the template - diskSizeGB: 10 - diskBusType: "paravirtual" - diskIOPS: 0 - storageProfile: "*" - # Optional: SizingPolicy is the sizing policy to be used for machines created by this machine deployment. - # If left empty, default sizing policy if specified at OVDC/organization level is used. - sizingPolicy: "" - # Optional: PlacementPolicy is the placement policy to be used for machines created by this machine deployment. - # If left empty, default placement policy if specified at OVDC/organization level is used. - placementPolicy: "" - operatingSystem: "ubuntu" - operatingSystemSpec: - distUpgradeOnBoot: false - versions: - kubelet: 1.28.5 diff --git a/examples/vsphere-datastore-cluster-machinedeployment.yaml b/examples/vsphere-datastore-cluster-machinedeployment.yaml deleted file mode 100644 index 1bae0b162..000000000 --- a/examples/vsphere-datastore-cluster-machinedeployment.yaml +++ /dev/null @@ -1,82 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - # If you change the namespace/name, you must also - # adjust the rbac rules - name: machine-controller-vsphere - namespace: kube-system -type: Opaque -stringData: - password: << VSPHERE_PASSWORD >> ---- -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: vsphere-machinedeployment - namespace: kube-system -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - foo: bar - template: - metadata: - labels: - foo: bar - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "vsphere" - cloudProviderSpec: - # Can also be set via the env var 'VSPHERE_USERNAME' on the machine-controller - username: "<< VSPHERE_USERNAME >>" - # Can also be set via the env var 'VSPHERE_ADDRESS' on the machine-controller - # example: 'https://your-vcenter:8443'. '/sdk' gets appended automatically - vsphereURL: "<< VSPHERE_ADDRESS >>" - # Can also be set via the env var 'VSPHERE_PASSWORD' on the machine-controller - password: - secretKeyRef: - namespace: kube-system - name: machine-controller-vsphere - key: password - datacenter: datacenter1 - templateVMName: ubuntu-template - # Optional. Sets the networks on the VM. If no network is specified, the template default will be used. - networks: - - network1 - # Optional - folder: folder1 - datastoreCluster: datastorecluster1 - # Can also be set via the env var 'VSPHERE_ALLOW_INSECURE' on the machine-controller - allowInsecure: true - # Cluster to configure vm anti affinity rules - cluster: cl-1 - # Automatically create anti affinity rules for machines - vmAntiAffinity: true - cpus: 2 - memoryMB: 2048 - # Optional: Resize the root disk to this size. Must be bigger than the existing size - # Default is to leave the disk at the same size as the template - diskSizeGB: 10 - operatingSystem: "ubuntu" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` - rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" - # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` - rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" - # 'rhsmOfflineToken' if it was provided red hat systems subscriptions will be removed upon machines deletions, and if wasn't - # provided the rhsm will be disabled and any created subscription won't be removed automatically - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" - versions: - kubelet: 1.28.5 diff --git a/examples/vsphere-machinedeployment.yaml b/examples/vsphere-machinedeployment.yaml deleted file mode 100644 index 4e1bb6cef..000000000 --- a/examples/vsphere-machinedeployment.yaml +++ /dev/null @@ -1,82 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - # If you change the namespace/name, you must also - # adjust the rbac rules - name: machine-controller-vsphere - namespace: kube-system -type: Opaque -stringData: - password: << VSPHERE_PASSWORD >> ---- -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: vsphere-machinedeployment - namespace: kube-system -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - foo: bar - template: - metadata: - labels: - foo: bar - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "vsphere" - cloudProviderSpec: - # Can also be set via the env var 'VSPHERE_USERNAME' on the machine-controller - username: "<< VSPHERE_USERNAME >>" - # Can also be set via the env var 'VSPHERE_ADDRESS' on the machine-controller - # example: 'https://your-vcenter:8443'. '/sdk' gets appended automatically - vsphereURL: "<< VSPHERE_ADDRESS >>" - # Can also be set via the env var 'VSPHERE_PASSWORD' on the machine-controller - password: - secretKeyRef: - namespace: kube-system - name: machine-controller-vsphere - key: password - datacenter: datacenter1 - templateVMName: ubuntu-template - # Optional. Sets the networks on the VM. If no network is specified, the template default will be used. - networks: - - network1 - # Optional - folder: folder1 - datastore: datastore1 - # Can also be set via the env var 'VSPHERE_ALLOW_INSECURE' on the machine-controller - allowInsecure: true - # Cluster to configure vm anti affinity rules - cluster: cl-1 - # Automatically create anti affinity rules for machines - vmAntiAffinity: true - cpus: 2 - memoryMB: 2048 - # Optional: Resize the root disk to this size. Must be bigger than the existing size - # Default is to leave the disk at the same size as the template - diskSizeGB: 10 - operatingSystem: "ubuntu" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` - rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" - # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` - rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" - # 'rhsmOfflineToken' if it was provided red hat systems subscriptions will be removed upon machines deletions, and if wasn't - # provided the rhsm will be disabled and any created subscription won't be removed automatically - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" - versions: - kubelet: 1.28.5 diff --git a/examples/vultr-machinedeployment.yaml b/examples/vultr-machinedeployment.yaml deleted file mode 100644 index 03e2b7f07..000000000 --- a/examples/vultr-machinedeployment.yaml +++ /dev/null @@ -1,73 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - # If you change the namespace/name, you must also - # adjust the rbac rules - name: machine-controller-vultr - namespace: kube-system -type: Opaque -stringData: - apiKey: << VULTR_API_KEY >> ---- -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: vultr-machinedeployment - namespace: kube-system -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - foo: bar - template: - metadata: - labels: - foo: bar - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "vultr" - cloudProviderSpec: - # Can also be set via the env var 'VULTR_API_KEY' on the machine-controller - apiKey: - secretKeyRef: - namespace: kube-system - name: machine-controller-vultr - key: apiKey - # Default is false meaning a virtual machine instance is created - # If true, a bare metal instance is created - physicalMachine: false - region: blr - plan: "vhf-8c-32gb" - # This takes precedence over enableVPC - vpcId: - - - # For more reference, see - # https://www.vultr.com/api/#tag/instances/operation/create-instance - enableVPC: false - enableVPC2: true - vpc2Id: - - - # Required: app_id, image_id, os_id, snapshot_id, or iso_id must be provided. Currently only os_id is supported. - # This takes precedence over operatingSystem - osId: 215 - # Optional - tags: - - tag1 - - tag2 - - tag3 - operatingSystem: "ubuntu" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - versions: - kubelet: 1.28.5 diff --git a/hack/ci/run-e2e-tests.sh b/hack/ci/run-e2e-tests.sh index 2b235c06b..c25f7d564 100755 --- a/hack/ci/run-e2e-tests.sh +++ b/hack/ci/run-e2e-tests.sh @@ -56,12 +56,6 @@ echodate "Copying machine-controller plugins..." cp machine-controller-userdata-* /usr/local/bin ls -l /usr/local/bin -# Install genisoimage, this is required for generating user-data for vSphere -if [[ "${JOB_NAME:-}" = *"pull-machine-controller-e2e-vsphere"* ]]; then - echo "Installing genisoimage..." - apt install -y genisoimage -fi - echodate "Creating kind cluster" source hack/ci/setup-kind-cluster.sh diff --git a/pkg/admission/machinedeployments_validation.go b/pkg/admission/machinedeployments_validation.go index dd4a9c1d6..6945ee609 100644 --- a/pkg/admission/machinedeployments_validation.go +++ b/pkg/admission/machinedeployments_validation.go @@ -120,14 +120,6 @@ func mutationsForMachineDeployment(md *v1alpha1.MachineDeployment) error { return fmt.Errorf("failed to read MachineDeployment.Spec.Template.Spec.ProviderSpec: %w", err) } - // Packet has been renamed to Equinix Metal - if providerConfig.CloudProvider == cloudProviderPacket { - err = migrateToEquinixMetal(providerConfig) - if err != nil { - return fmt.Errorf("failed to migrate packet to equinix metal: %w", err) - } - } - // Update value in original object md.Spec.Template.Spec.ProviderSpec.Value.Raw, err = json.Marshal(providerConfig) if err != nil { diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index 914508131..d7532a8a0 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -129,14 +129,6 @@ func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec return fmt.Errorf("failed to read machine.spec.providerSpec: %w", err) } - // Packet has been renamed to Equinix Metal - if providerConfig.CloudProvider == cloudProviderPacket { - err = migrateToEquinixMetal(providerConfig) - if err != nil { - return fmt.Errorf("failed to migrate packet to equinix metal: %w", err) - } - } - skg := providerconfig.NewConfigVarResolver(ctx, ad.workerClient) prov, err := cloudprovider.ForProvider(providerConfig.CloudProvider, skg) if err != nil { diff --git a/pkg/admission/util.go b/pkg/admission/util.go deleted file mode 100644 index 8e95017a4..000000000 --- a/pkg/admission/util.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2021 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package admission - -import ( - "encoding/json" - "fmt" - - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" -) - -const cloudProviderPacket = "packet" - -func migrateToEquinixMetal(providerConfig *providerconfigtypes.Config) (err error) { - providerConfig.CloudProvider = providerconfigtypes.CloudProviderEquinixMetal - - // Field .spec.providerSpec.cloudProviderSpec.apiKey has been replaced with .spec.providerSpec.cloudProviderSpec.token - // We first need to perform in-place replacement for this field - rawConfig := map[string]interface{}{} - if err := json.Unmarshal(providerConfig.CloudProviderSpec.Raw, &rawConfig); err != nil { - return fmt.Errorf("failed to unmarshal providerConfig.CloudProviderSpec.Raw: %w", err) - } - // NB: We have to set the token only if apiKey existed, otherwise, migrated - // machines will not create at all (authentication errors). - apiKey, ok := rawConfig["apiKey"] - if ok { - rawConfig["token"] = apiKey - delete(rawConfig, "apiKey") - } - - // Update original object - providerConfig.CloudProviderSpec.Raw, err = json.Marshal(rawConfig) - if err != nil { - return fmt.Errorf("failed to json marshal providerConfig.CloudProviderSpec.Raw: %w", err) - } - return nil -} diff --git a/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go b/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go index 902a11cd2..5779b4cf0 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go +++ b/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go @@ -27,85 +27,6 @@ import ( "sigs.k8s.io/yaml" ) -func Test_Convert_MachineDeployment_ProviderConfig_To_ProviderSpec(t *testing.T) { - fixtures, err := os.ReadDir("testdata/clusterv1alpha1machineDeploymentWithProviderConfig") - if err != nil { - t.Fatalf("failed to list fixtures: %v", err) - } - - for _, fixture := range fixtures { - fixtureYamlByte, err := os.ReadFile(fmt.Sprintf("testdata/clusterv1alpha1machineDeploymentWithProviderConfig/%s", fixture.Name())) - if err != nil { - t.Errorf("failed to read fixture file %s: %v", fixture.Name(), err) - continue - } - fixtureJSONBytes, err := yaml.YAMLToJSON(fixtureYamlByte) - if err != nil { - t.Errorf("failed to convert yaml to json: %v", err) - continue - } - convertedMachineDeployment, wasConverted, err := Convert_MachineDeployment_ProviderConfig_To_ProviderSpec(fixtureJSONBytes) - if err != nil { - t.Errorf("failed to convert machineDeployment from file %s: %v", fixture.Name(), err) - continue - } - if !wasConverted { - t.Errorf("expected wasConverted to be true but was %t", wasConverted) - } - convertedMachineDeploymentJSONBytes, err := json.Marshal(*convertedMachineDeployment) - if err != nil { - t.Errorf("failed to marshal converted machineDeployment %s: %v", convertedMachineDeployment.Name, err) - continue - } - convertedMachineDeploymentYamlBytes, err := yaml.JSONToYAML(convertedMachineDeploymentJSONBytes) - if err != nil { - t.Errorf("failed to convert json to yaml: %v", err) - continue - } - testhelper.CompareOutput(t, fmt.Sprintf("migrated_clusterv1alpha1machineDeploymentWithProviderConfig/%s", fixture.Name()), string(convertedMachineDeploymentYamlBytes), *update) - } -} - -func Test_Convert_MachineSet_ProviderConfig_To_ProviderSpec(t *testing.T) { - fixtures, err := os.ReadDir("testdata/clusterv1alpha1machineSetWithProviderConfig") - if err != nil { - t.Fatalf("failed to list fixtures: %v", err) - } - - for _, fixture := range fixtures { - fixtureYamlByte, err := os.ReadFile(fmt.Sprintf("testdata/clusterv1alpha1machineSetWithProviderConfig/%s", fixture.Name())) - if err != nil { - t.Errorf("failed to read fixture file %s: %v", fixture.Name(), err) - continue - } - fixtureJSONBytes, err := yaml.YAMLToJSON(fixtureYamlByte) - if err != nil { - t.Errorf("failed to convert yaml to json: %v", err) - continue - } - convertedMachineSet, wasConverted, err := Convert_MachineSet_ProviderConfig_To_ProviderSpec(fixtureJSONBytes) - if err != nil { - t.Errorf("failed to convert machineSet from file %s: %v", fixture.Name(), err) - continue - } - if !wasConverted { - t.Errorf("expected wasConverted to be true but was %t", wasConverted) - } - - convertedMachineSetJSONBytes, err := json.Marshal(*convertedMachineSet) - if err != nil { - t.Errorf("failed to marshal converted machineSet %s: %v", convertedMachineSet.Name, err) - continue - } - convertedMachineSetYamlBytes, err := yaml.JSONToYAML(convertedMachineSetJSONBytes) - if err != nil { - t.Errorf("failed to convert json to yaml: %v", err) - continue - } - testhelper.CompareOutput(t, fmt.Sprintf("migrated_clusterv1alpha1machineSetWithProviderConfig/%s", fixture.Name()), string(convertedMachineSetYamlBytes), *update) - } -} - func Test_Convert_Machine_ProviderConfig_To_ProviderSpec(t *testing.T) { fixtures, err := os.ReadDir("testdata/clusterv1alpha1machineWithProviderConfig") if err != nil { diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineDeploymentWithProviderConfig/hetzner.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineDeploymentWithProviderConfig/hetzner.yaml deleted file mode 100644 index 80d6d0826..000000000 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineDeploymentWithProviderConfig/hetzner.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: cluster.k8s.io/v1alpha1 -kind: MachineDeployment -metadata: - annotations: - machinedeployment.clusters.k8s.io/revision: "1" - generateName: kubermatic-gttbxgswnv- - generation: 1 - name: kubermatic-gttbxgswnv-q5rcj - namespace: kube-system -spec: - minReadySeconds: 0 - progressDeadlineSeconds: 600 - replicas: 3 - revisionHistoryLimit: 1 - selector: - matchLabels: - machine: md-gttbxgswnv-nl5s9qs8ww - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - type: RollingUpdate - template: - metadata: - creationTimestamp: null - labels: - machine: md-gttbxgswnv-nl5s9qs8ww - spec: - metadata: - creationTimestamp: null - providerConfig: - value: - cloudProvider: hetzner - cloudProviderSpec: - datacenter: nbg1-dc3 - location: "" - serverType: cx31 - token: "" - operatingSystem: ubuntu - operatingSystemSpec: - distUpgradeOnBoot: false - sshPublicKeys: [] - versions: - kubelet: 1.11.6 diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineSetWithProviderConfig/hetzner.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineSetWithProviderConfig/hetzner.yaml deleted file mode 100644 index fc1375052..000000000 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineSetWithProviderConfig/hetzner.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: cluster.k8s.io/v1alpha1 -kind: MachineSet -metadata: - annotations: - machinedeployment.clusters.k8s.io/desired-replicas: "3" - machinedeployment.clusters.k8s.io/max-replicas: "4" - machinedeployment.clusters.k8s.io/revision: "1" - creationTimestamp: "2019-01-23T12:59:25Z" - generation: 1 - labels: - machine: md-gttbxgswnv-nl5s9qs8ww - machine-template-hash: "1950051685" - name: kubermatic-gttbxgswnv-q5rcj-5f94495bd9 - namespace: kube-system - ownerReferences: - - apiVersion: cluster.k8s.io/v1alpha1 - blockOwnerDeletion: true - controller: true - kind: MachineDeployment - name: kubermatic-gttbxgswnv-q5rcj - uid: b5fd92a1-1f0e-11e9-9561-b2d5a2b51b30 - resourceVersion: "9387" -spec: - replicas: 3 - selector: - matchLabels: - machine: md-gttbxgswnv-nl5s9qs8ww - machine-template-hash: "1950051685" - template: - metadata: - creationTimestamp: null - labels: - machine: md-gttbxgswnv-nl5s9qs8ww - machine-template-hash: "1950051685" - spec: - metadata: - creationTimestamp: null - providerConfig: - value: - cloudProvider: hetzner - cloudProviderSpec: - datacenter: nbg1-dc3 - location: "" - serverType: cx31 - token: "" - operatingSystem: ubuntu - operatingSystemSpec: - distUpgradeOnBoot: false - sshPublicKeys: [] - versions: - kubelet: 1.11.6 diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/hetzner.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/hetzner.yaml deleted file mode 100644 index 78e599569..000000000 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/hetzner.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: cluster.k8s.io/v1alpha1 -kind: Machine -metadata: - name: hetzner-machine - namespace: kube-system -spec: - providerConfig: - value: - cloudProvider: hetzner - cloudProviderSpec: - datacenter: '' - location: fsn1 - serverType: cx11 - token: << HETZNER_TOKEN >> - operatingSystem: << OS_NAME >> - operatingSystemSpec: - disableAutoUpdate: true - distUpgradeOnBoot: false - sshPublicKeys: - - << YOUR_PUBLIC_KEY >> - versions: - kubelet: << KUBERNETES_VERSION >> diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/azure.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/azure.yaml deleted file mode 100644 index 4fd8dff63..000000000 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/azure.yaml +++ /dev/null @@ -1,47 +0,0 @@ -apiVersion: "machine.k8s.io/v1alpha1" -kind: Machine -metadata: - name: azure -spec: - metadata: - labels: - foo: "bar" - providerConfig: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "azure" - cloudProviderSpec: - tenantID: - secretKeyRef: - namespace: kube-system - name: machine-controller-azure - key: tenantID - clientID: - secretKeyRef: - namespace: kube-system - name: machine-controller-azure - key: clientID - clientSecret: - secretKeyRef: - namespace: kube-system - name: machine-controller-azure - key: clientSecret - subscriptionID: - secretKeyRef: - namespace: kube-system - name: machine-controller-azure - key: subscriptionID - location: "westeurope" - resourceGroup: "<< YOUR_RESOURCE_GROUP >>" - vmSize: "Standard_B1ms" - vnetName: "<< VNET_NAME >>" - subnetName: "<< SUBNET_NAME >>" - routeTableName: "<< ROUTE_TABLE_NAME >>" - assignPublicIP: false - operatingSystem: "flatcar" - operatingSystemSpec: - distUpgradeOnBoot: false - roles: - - "Node" - versions: - kubelet: "v1.10.2" diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/digitalocean.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/digitalocean.yaml deleted file mode 100644 index eebb1d4c7..000000000 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/digitalocean.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: "machine.k8s.io/v1alpha1" -kind: Machine -metadata: - name: digitalocean -spec: - metadata: - name: node1 - providerConfig: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "digitalocean" - cloudProviderSpec: - token: "token" - region: fra1 - size: 2gb - backups: false - ipv6: false - private_networking: true - monitoring: false - tags: - - "machine-controller" - operatingSystem: "ubuntu" - operatingSystemSpec: - disableAutoUpdate: true - roles: - - "Node" - versions: - kubelet: "v1.9.6" diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/hetzner.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/hetzner.yaml deleted file mode 100644 index 62fa47ccc..000000000 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/hetzner.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: "machine.k8s.io/v1alpha1" -kind: Machine -metadata: - name: hetzner -spec: - providerConfig: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "hetzner" - cloudProviderSpec: - token: - secretKeyRef: - namespace: kube-system - name: machine-controller-hetzner - key: token - serverType: "cx11" - datacenter: "" - location: "fsn1" - operatingSystem: "ubuntu" - operatingSystemSpec: - distUpgradeOnBoot: false - roles: - - "Node" - versions: - kubelet: "1.9.6" diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/linode.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/linode.yaml deleted file mode 100644 index 94305c61e..000000000 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/linode.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: "machine.k8s.io/v1alpha1" -kind: Machine -metadata: - name: linode -spec: - metadata: - name: node1 - providerConfig: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "linode" - cloudProviderSpec: - backups: false - private_networking: true - region: eu-west - tags: - - "machine-controller" - token: "token" - type: g6-standard-2 - operatingSystem: "ubuntu" - operatingSystemSpec: - disableAutoUpdate: true - roles: - - "Node" - versions: - kubelet: "v1.9.6" diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/vsphere-static-ip.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/vsphere-static-ip.yaml deleted file mode 100644 index c35f495a3..000000000 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/vsphere-static-ip.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: "machine.k8s.io/v1alpha1" -kind: Machine -metadata: - name: vsphere-static-ip -spec: - metadata: - labels: - foo: "bar" - providerConfig: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "vsphere" - cloudProviderSpec: - templateVMName: '<< OS_NAME >>-template' - username: '<< VSPHERE_USERNAME >>' - vsphereURL: '<< VSPHERE_ADDRESS >>' - datacenter: 'Datacenter' - folder: '/Datacenter/vm/e2e-tests' - password: << VSPHERE_PASSWORD >> - # example: 'https://your-vcenter:8443'. '/sdk' gets appended automatically - cluster: '<< VSPHERE_CLUSTER >>' - datastore: datastore1 - allowInsecure: true - cpus: 2 - MemoryMB: 2048 - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - network: - cidr: "192.168.44.<< IP_OCTET >>/20" - gateway: "192.168.32.1" - dns: - servers: - - "192.168.32.1" - - "8.8.8.8" - roles: - - "Node" - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/vsphere.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/vsphere.yaml deleted file mode 100644 index e552a1f1d..000000000 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/vsphere.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: "machine.k8s.io/v1alpha1" -kind: Machine -metadata: - name: vsphere -spec: - metadata: - labels: - foo: "bar" - providerConfig: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "vsphere" - cloudProviderSpec: - templateVMName: ubuntu-template - # Can also be set via the env var 'VSPHERE_USERNAME' on the machine-controller - username: '<< VSPHERE_USERNAME >>' - # Can also be set via the env var 'VSPHERE_ADDRESS' on the machine-controller - # example: 'https://your-vcenter:8443'. '/sdk' gets appended automatically - vsphereURL: '<< VSPHERE_ADDRESS >>' - datacenter: 'Datacenter' - # Can also be set via the env var 'VSPHERE_PASSWORD' on the machine-controller - password: - secretKeyRef: - namespace: kube-system - name: machine-controller-vsphere - key: password - cluster: "test-cluster" - datastore: datastore1 - # Can also be set via the env var 'VSPHERE_ALLOW_INSECURE' on the machine-controller - allowInsecure: true - cpus: 2 - MemoryMB: 2048 - operatingSystem: "ubuntu" - operatingSystemSpec: - distUpgradeOnBoot: false - roles: - - "Node" - versions: - kubelet: "v1.9.6" diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/azure.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/azure.yaml deleted file mode 100644 index bc7aebe5a..000000000 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/azure.yaml +++ /dev/null @@ -1,48 +0,0 @@ -metadata: - creationTimestamp: null - name: azure - namespace: kube-system -spec: - metadata: - creationTimestamp: null - labels: - foo: bar - providerSpec: - value: - cloudProvider: azure - cloudProviderSpec: - assignPublicIP: false - clientID: - secretKeyRef: - key: clientID - name: machine-controller-azure - namespace: kube-system - clientSecret: - secretKeyRef: - key: clientSecret - name: machine-controller-azure - namespace: kube-system - location: westeurope - resourceGroup: << YOUR_RESOURCE_GROUP >> - routeTableName: << ROUTE_TABLE_NAME >> - subnetName: << SUBNET_NAME >> - subscriptionID: - secretKeyRef: - key: subscriptionID - name: machine-controller-azure - namespace: kube-system - tenantID: - secretKeyRef: - key: tenantID - name: machine-controller-azure - namespace: kube-system - vmSize: Standard_B1ms - vnetName: << VNET_NAME >> - operatingSystem: flatcar - operatingSystemSpec: - distUpgradeOnBoot: false - sshPublicKeys: - - << YOUR_PUBLIC_KEY >> - versions: - kubelet: v1.10.2 -status: {} diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/digitalocean.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/digitalocean.yaml deleted file mode 100644 index ee4aa74c5..000000000 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/digitalocean.yaml +++ /dev/null @@ -1,29 +0,0 @@ -metadata: - creationTimestamp: null - name: digitalocean - namespace: kube-system -spec: - metadata: - creationTimestamp: null - name: node1 - providerSpec: - value: - cloudProvider: digitalocean - cloudProviderSpec: - backups: false - ipv6: false - monitoring: false - private_networking: true - region: fra1 - size: 2gb - tags: - - machine-controller - token: token - operatingSystem: ubuntu - operatingSystemSpec: - disableAutoUpdate: true - sshPublicKeys: - - << YOUR_PUBLIC_KEY >> - versions: - kubelet: v1.9.6 -status: {} diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/hetzner.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/hetzner.yaml deleted file mode 100644 index e5a817e69..000000000 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/hetzner.yaml +++ /dev/null @@ -1,27 +0,0 @@ -metadata: - creationTimestamp: null - name: hetzner - namespace: kube-system -spec: - metadata: - creationTimestamp: null - providerSpec: - value: - cloudProvider: hetzner - cloudProviderSpec: - datacenter: "" - location: fsn1 - serverType: cx11 - token: - secretKeyRef: - key: token - name: machine-controller-hetzner - namespace: kube-system - operatingSystem: ubuntu - operatingSystemSpec: - distUpgradeOnBoot: false - sshPublicKeys: - - << YOUR_PUBLIC_KEY >> - versions: - kubelet: 1.9.6 -status: {} diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/linode.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/linode.yaml deleted file mode 100644 index 1fc163c14..000000000 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/linode.yaml +++ /dev/null @@ -1,27 +0,0 @@ -metadata: - creationTimestamp: null - name: linode - namespace: kube-system -spec: - metadata: - creationTimestamp: null - name: node1 - providerSpec: - value: - cloudProvider: linode - cloudProviderSpec: - backups: false - private_networking: true - region: eu-west - tags: - - machine-controller - token: token - type: g6-standard-2 - operatingSystem: ubuntu - operatingSystemSpec: - disableAutoUpdate: true - sshPublicKeys: - - << YOUR_PUBLIC_KEY >> - versions: - kubelet: v1.9.6 -status: {} diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/vsphere-static-ip.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/vsphere-static-ip.yaml deleted file mode 100644 index bce2d482c..000000000 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/vsphere-static-ip.yaml +++ /dev/null @@ -1,39 +0,0 @@ -metadata: - creationTimestamp: null - name: vsphere-static-ip - namespace: kube-system -spec: - metadata: - creationTimestamp: null - labels: - foo: bar - providerSpec: - value: - cloudProvider: vsphere - cloudProviderSpec: - MemoryMB: 2048 - allowInsecure: true - cluster: << VSPHERE_CLUSTER >> - cpus: 2 - datacenter: Datacenter - datastore: datastore1 - folder: /Datacenter/vm/e2e-tests - password: << VSPHERE_PASSWORD >> - templateVMName: << OS_NAME >>-template - username: << VSPHERE_USERNAME >> - vsphereURL: << VSPHERE_ADDRESS >> - network: - cidr: 192.168.44.<< IP_OCTET >>/20 - dns: - servers: - - 192.168.32.1 - - 8.8.8.8 - gateway: 192.168.32.1 - operatingSystem: << OS_NAME >> - operatingSystemSpec: - distUpgradeOnBoot: false - sshPublicKeys: - - << YOUR_PUBLIC_KEY >> - versions: - kubelet: << KUBERNETES_VERSION >> -status: {} diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/vsphere.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/vsphere.yaml deleted file mode 100644 index 2fcb6c0da..000000000 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/vsphere.yaml +++ /dev/null @@ -1,35 +0,0 @@ -metadata: - creationTimestamp: null - name: vsphere - namespace: kube-system -spec: - metadata: - creationTimestamp: null - labels: - foo: bar - providerSpec: - value: - cloudProvider: vsphere - cloudProviderSpec: - MemoryMB: 2048 - allowInsecure: true - cluster: test-cluster - cpus: 2 - datacenter: Datacenter - datastore: datastore1 - password: - secretKeyRef: - key: password - name: machine-controller-vsphere - namespace: kube-system - templateVMName: ubuntu-template - username: << VSPHERE_USERNAME >> - vsphereURL: << VSPHERE_ADDRESS >> - operatingSystem: ubuntu - operatingSystemSpec: - distUpgradeOnBoot: false - sshPublicKeys: - - << YOUR_PUBLIC_KEY >> - versions: - kubelet: v1.9.6 -status: {} diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineDeploymentWithProviderConfig/hetzner.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineDeploymentWithProviderConfig/hetzner.yaml deleted file mode 100644 index 8ff0bf495..000000000 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineDeploymentWithProviderConfig/hetzner.yaml +++ /dev/null @@ -1,46 +0,0 @@ -apiVersion: cluster.k8s.io/v1alpha1 -kind: MachineDeployment -metadata: - annotations: - machinedeployment.clusters.k8s.io/revision: "1" - creationTimestamp: null - generateName: kubermatic-gttbxgswnv- - generation: 1 - name: kubermatic-gttbxgswnv-q5rcj - namespace: kube-system -spec: - minReadySeconds: 0 - progressDeadlineSeconds: 600 - replicas: 3 - revisionHistoryLimit: 1 - selector: - matchLabels: - machine: md-gttbxgswnv-nl5s9qs8ww - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - type: RollingUpdate - template: - metadata: - creationTimestamp: null - labels: - machine: md-gttbxgswnv-nl5s9qs8ww - spec: - metadata: - creationTimestamp: null - providerSpec: - value: - cloudProvider: hetzner - cloudProviderSpec: - datacenter: nbg1-dc3 - location: "" - serverType: cx31 - token: "" - operatingSystem: ubuntu - operatingSystemSpec: - distUpgradeOnBoot: false - sshPublicKeys: [] - versions: - kubelet: 1.11.6 -status: {} diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineSetWithProviderConfig/hetzner.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineSetWithProviderConfig/hetzner.yaml deleted file mode 100644 index 31a790d15..000000000 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineSetWithProviderConfig/hetzner.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: cluster.k8s.io/v1alpha1 -kind: MachineSet -metadata: - annotations: - machinedeployment.clusters.k8s.io/desired-replicas: "3" - machinedeployment.clusters.k8s.io/max-replicas: "4" - machinedeployment.clusters.k8s.io/revision: "1" - creationTimestamp: "2019-01-23T12:59:25Z" - generation: 1 - labels: - machine: md-gttbxgswnv-nl5s9qs8ww - machine-template-hash: "1950051685" - name: kubermatic-gttbxgswnv-q5rcj-5f94495bd9 - namespace: kube-system - ownerReferences: - - apiVersion: cluster.k8s.io/v1alpha1 - blockOwnerDeletion: true - controller: true - kind: MachineDeployment - name: kubermatic-gttbxgswnv-q5rcj - uid: b5fd92a1-1f0e-11e9-9561-b2d5a2b51b30 - resourceVersion: "9387" -spec: - replicas: 3 - selector: - matchLabels: - machine: md-gttbxgswnv-nl5s9qs8ww - machine-template-hash: "1950051685" - template: - metadata: - creationTimestamp: null - labels: - machine: md-gttbxgswnv-nl5s9qs8ww - machine-template-hash: "1950051685" - spec: - metadata: - creationTimestamp: null - providerSpec: - value: - cloudProvider: hetzner - cloudProviderSpec: - datacenter: nbg1-dc3 - location: "" - serverType: cx31 - token: "" - operatingSystem: ubuntu - operatingSystemSpec: - distUpgradeOnBoot: false - sshPublicKeys: [] - versions: - kubelet: 1.11.6 -status: - replicas: 0 diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/hetzner.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/hetzner.yaml deleted file mode 100644 index e25b81872..000000000 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/hetzner.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: cluster.k8s.io/v1alpha1 -kind: Machine -metadata: - creationTimestamp: null - name: hetzner-machine - namespace: kube-system -spec: - metadata: - creationTimestamp: null - providerSpec: - value: - cloudProvider: hetzner - cloudProviderSpec: - datacenter: "" - location: fsn1 - serverType: cx11 - token: << HETZNER_TOKEN >> - operatingSystem: << OS_NAME >> - operatingSystemSpec: - disableAutoUpdate: true - distUpgradeOnBoot: false - sshPublicKeys: - - << YOUR_PUBLIC_KEY >> - versions: - kubelet: << KUBERNETES_VERSION >> -status: {} diff --git a/pkg/cloudprovider/provider.go b/pkg/cloudprovider/provider.go index 4cd794157..52e4a6aea 100644 --- a/pkg/cloudprovider/provider.go +++ b/pkg/cloudprovider/provider.go @@ -20,26 +20,10 @@ import ( "errors" cloudprovidercache "github.com/kubermatic/machine-controller/pkg/cloudprovider/cache" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/alibaba" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/aws" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/azure" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/digitalocean" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/edge" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/equinixmetal" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/fake" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/gce" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/hetzner" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/kubevirt" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/linode" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/nutanix" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/opennebula" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/openstack" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/scaleway" - vcd "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vmwareclouddirector" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vsphere" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vultr" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" @@ -52,79 +36,19 @@ var ( ErrProviderNotFound = errors.New("cloudprovider not found") providers = map[providerconfigtypes.CloudProvider]func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider{ - providerconfigtypes.CloudProviderDigitalocean: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return digitalocean.New(cvr) - }, providerconfigtypes.CloudProviderAWS: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return aws.New(cvr) }, providerconfigtypes.CloudProviderOpenstack: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return openstack.New(cvr) }, - providerconfigtypes.CloudProviderGoogle: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return gce.New(cvr) - }, - providerconfigtypes.CloudProviderHetzner: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return hetzner.New(cvr) - }, - providerconfigtypes.CloudProviderVsphere: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return vsphere.New(cvr) - }, - providerconfigtypes.CloudProviderAzure: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return azure.New(cvr) - }, - providerconfigtypes.CloudProviderEquinixMetal: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return equinixmetal.New(cvr) - }, - // NB: This is explicitly left to allow old Packet machines to be deleted. - // We can handle those machines in the same way as Equinix Metal machines - // because there are no API changes. - // TODO: Remove this after deprecation period. - providerconfigtypes.CloudProviderPacket: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return equinixmetal.New(cvr) - }, providerconfigtypes.CloudProviderFake: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return fake.New(cvr) }, - providerconfigtypes.CloudProviderEdge: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return edge.New(cvr) - }, - providerconfigtypes.CloudProviderKubeVirt: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return kubevirt.New(cvr) - }, - providerconfigtypes.CloudProviderAlibaba: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return alibaba.New(cvr) - }, - providerconfigtypes.CloudProviderScaleway: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return scaleway.New(cvr) - }, - providerconfigtypes.CloudProviderAnexia: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return anexia.New(cvr) - }, providerconfigtypes.CloudProviderBaremetal: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { // TODO(MQ): add a baremetal driver. return baremetal.New(cvr) }, - providerconfigtypes.CloudProviderNutanix: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return nutanix.New(cvr) - }, - providerconfigtypes.CloudProviderVMwareCloudDirector: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return vcd.New(cvr) - }, - } - - // communityProviders holds a map of cloud providers that have been implemented by community members and - // contributed to machine-controller. They are not end-to-end tested by the machine-controller development team. - communityProviders = map[providerconfigtypes.CloudProvider]func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider{ - providerconfigtypes.CloudProviderLinode: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return linode.New(cvr) - }, - providerconfigtypes.CloudProviderVultr: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return vultr.New(cvr) - }, - providerconfigtypes.CloudProviderOpenNebula: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return opennebula.New(cvr) - }, } ) @@ -133,8 +57,5 @@ func ForProvider(p providerconfigtypes.CloudProvider, cvr *providerconfig.Config if p, found := providers[p]; found { return NewValidationCacheWrappingCloudProvider(p(cvr)), nil } - if p, found := communityProviders[p]; found { - return NewValidationCacheWrappingCloudProvider(p(cvr)), nil - } return nil, ErrProviderNotFound } diff --git a/pkg/cloudprovider/provider/alibaba/provider.go b/pkg/cloudprovider/provider/alibaba/provider.go deleted file mode 100644 index c20965551..000000000 --- a/pkg/cloudprovider/provider/alibaba/provider.go +++ /dev/null @@ -1,486 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package alibaba - -import ( - "context" - "encoding/base64" - "errors" - "fmt" - "net/http" - - "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" - "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - alibabatypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/alibaba/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - kuberneteshelper "github.com/kubermatic/machine-controller/pkg/kubernetes" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" -) - -const ( - machineUIDTag = "machine_uid" - centosImageName = "CentOS 7.9 64 bit" - ubuntuImageName = "Ubuntu 22.04 64 bit" - - finalizerInstance = "kubermatic.io/cleanup-alibaba-instance" -) - -type instanceStatus string - -const ( - stoppedStatus instanceStatus = "Stopped" - runningStatus instanceStatus = "Running" -) - -type provider struct { - configVarResolver *providerconfig.ConfigPointerVarResolver -} - -type Config struct { - AccessKeyID string - AccessKeySecret string - RegionID string - InstanceType string - InstanceID string - VSwitchID string - InternetMaxBandwidthOut string - Labels map[string]string - ZoneID string - DiskType string - DiskSize string -} - -type alibabaInstance struct { - instance *ecs.Instance -} - -func (a *alibabaInstance) Name() string { - return a.instance.InstanceName -} - -func (a *alibabaInstance) ID() string { - return a.instance.InstanceId -} - -// TODO: Implement once we start supporting Alibaba CCM. -func (a *alibabaInstance) ProviderID() string { - return "" -} - -func (a *alibabaInstance) HostID() string { - return "" -} - -func (a *alibabaInstance) Addresses() map[string]v1.NodeAddressType { - primaryIPAddresses := map[string]v1.NodeAddressType{} - for _, networkInterface := range a.instance.NetworkInterfaces.NetworkInterface { - primaryIPAddresses[networkInterface.PrimaryIpAddress] = v1.NodeInternalIP - } - - return primaryIPAddresses -} - -func (a *alibabaInstance) Status() instance.Status { - return instance.Status(a.instance.Status) -} - -// New returns an Alibaba cloud provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return &provider{configVarResolver: &providerconfig.ConfigPointerVarResolver{Cvr: configVarResolver}} -} - -func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { - return spec, nil -} - -func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, machineSpec clusterv1alpha1.MachineSpec) error { - c, pc, err := p.getConfig(machineSpec.ProviderSpec) - if err != nil { - return fmt.Errorf("failed to parse config: %w", err) - } - - if c.AccessKeyID == "" { - return errors.New("accessKeyID is missing") - } - if c.AccessKeySecret == "" { - return errors.New("accessKeySecret is missing") - } - if c.RegionID == "" { - return errors.New("regionID is missing") - } - if c.InstanceType == "" { - return errors.New("instanceType is missing") - } - if c.VSwitchID == "" { - return errors.New("vSwitchID is missing") - } - if c.InternetMaxBandwidthOut == "" { - return errors.New("internetMaxBandwidthOut is missing") - } - if c.ZoneID == "" { - return errors.New("zoneID is missing") - } - _, err = p.getImageIDForOS(machineSpec, pc.OperatingSystem) - if err != nil { - return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, err) - } - if c.DiskType == "" { - return errors.New("DiskType is missing") - } - if c.DiskSize == "" { - return errors.New("DiskSize is missing") - } - - return nil -} - -func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("failed to parse MachineSpec, due to %v", err), - } - } - - client, err := getClient(c.RegionID, c.AccessKeyID, c.AccessKeySecret) - if err != nil { - return nil, fmt.Errorf("failed to get alibaba client: %w", err) - } - - foundInstance, err := getInstance(client, machine.Name, string(machine.UID)) - if err != nil { - return nil, err - } - - switch instanceStatus(foundInstance.Status) { - case stoppedStatus: - startRequest := ecs.CreateStartInstanceRequest() - startRequest.InstanceId = foundInstance.InstanceId - - _, err = client.StartInstance(startRequest) - if err != nil { - return nil, fmt.Errorf("failed to start instance %v: %w", foundInstance.InstanceId, err) - } - return nil, fmt.Errorf("instance %v is in a stopped state", foundInstance.InstanceId) - case runningStatus: - if len(foundInstance.PublicIpAddress.IpAddress) == 0 { - ipAddress := ecs.CreateAllocatePublicIpAddressRequest() - ipAddress.InstanceId = foundInstance.InstanceId - - _, err = client.AllocatePublicIpAddress(ipAddress) - if err != nil { - return nil, fmt.Errorf("failed to allocate ip address for instance %v: %w", foundInstance.InstanceId, err) - } - } - return &alibabaInstance{instance: foundInstance}, nil - } - - return nil, fmt.Errorf("instance %v is not ready", foundInstance.InstanceId) -} - -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return "", "", nil -} - -func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { - c, pc, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("failed to parse MachineSpec, due to %v", err), - } - } - - client, err := getClient(c.RegionID, c.AccessKeyID, c.AccessKeySecret) - if err != nil { - return nil, fmt.Errorf("failed to get alibaba client: %w", err) - } - - createInstanceRequest := ecs.CreateCreateInstanceRequest() - createInstanceRequest.ImageId, err = p.getImageIDForOS(machine.Spec, pc.OperatingSystem) - if err != nil { - return nil, fmt.Errorf("failed to get a valid image for machine : %w", err) - } - createInstanceRequest.InstanceName = machine.Name - createInstanceRequest.InstanceType = c.InstanceType - createInstanceRequest.VSwitchId = c.VSwitchID - createInstanceRequest.InternetMaxBandwidthOut = requests.Integer(c.InternetMaxBandwidthOut) - encodedUserData := base64.StdEncoding.EncodeToString([]byte(userdata)) - createInstanceRequest.UserData = encodedUserData - createInstanceRequest.SystemDiskCategory = c.DiskType - createInstanceRequest.DataDisk = &[]ecs.CreateInstanceDataDisk{ - { - Size: c.DiskSize, - }, - } - createInstanceRequest.SystemDiskSize = requests.Integer(c.DiskSize) - createInstanceRequest.ZoneId = c.ZoneID - tag := ecs.CreateInstanceTag{ - Key: machineUIDTag, - Value: string(machine.UID), - } - createInstanceRequest.Tag = &[]ecs.CreateInstanceTag{tag} - - _, err = client.CreateInstance(createInstanceRequest) - if err != nil { - return nil, fmt.Errorf("failed to create instance at Alibaba cloud: %w", err) - } - - if err = data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { - if !kuberneteshelper.HasFinalizer(updatedMachine, finalizerInstance) { - updatedMachine.Finalizers = append(updatedMachine.Finalizers, finalizerInstance) - } - }); err != nil { - return nil, fmt.Errorf("failed updating machine %v finzaliers: %w", machine.Name, err) - } - - foundInstance, err := getInstance(client, machine.Name, string(machine.UID)) - if err != nil { - return nil, fmt.Errorf("failed to get alibaba instance %v due to %w", machine.Name, err) - } - - return &alibabaInstance{instance: foundInstance}, nil -} - -func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { - foundInstance, err := p.Get(ctx, log, machine, data) - if err != nil { - if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { - if err := data.Update(machine, func(m *clusterv1alpha1.Machine) { - m.Finalizers = kuberneteshelper.RemoveFinalizer(m.Finalizers, finalizerInstance) - }); err != nil { - return false, err - } - } - return false, err - } - - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return false, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - - client, err := getClient(c.RegionID, c.AccessKeyID, c.AccessKeySecret) - if err != nil { - return false, fmt.Errorf("failed to get alibaba client: %w", err) - } - - deleteInstancesRequest := ecs.CreateDeleteInstancesRequest() - deleteInstancesRequest.InstanceId = &[]string{foundInstance.ID()} - - deleteInstancesRequest.Force = requests.Boolean("True") - if _, err = client.DeleteInstances(deleteInstancesRequest); err != nil { - return false, fmt.Errorf("failed to delete instance with instanceID %s, due to %w", foundInstance.ID(), err) - } - - return false, nil -} - -func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { - labels := make(map[string]string) - - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err == nil { - labels["instanceType"] = c.InstanceType - labels["region"] = c.RegionID - } - - return labels, err -} - -func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return fmt.Errorf("failed to decode providerconfig: %w", err) - } - - client, err := getClient(c.RegionID, c.AccessKeyID, c.AccessKeySecret) - if err != nil { - return fmt.Errorf("failed to get alibaba client: %w", err) - } - - foundInstance, err := getInstance(client, machine.Name, string(machine.UID)) - if err != nil { - return fmt.Errorf("failed to get alibaba instance %v due to %w", machine.Name, err) - } - - tag := ecs.AddTagsTag{ - Value: string(newUID), - Key: machineUIDTag, - } - request := ecs.CreateAddTagsRequest() - request.ResourceId = foundInstance.InstanceId - request.ResourceType = "instance" - tags := []ecs.AddTagsTag{tag} - request.Tag = &tags - - if _, err := client.AddTags(request); err != nil { - return fmt.Errorf("failed to create new UID tag: %w", err) - } - - return nil -} - -func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { - return nil -} - -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, errors.New("machine.spec.providerconfig.value is nil") - } - - pconfig, err := providerconfigtypes.GetConfig(provSpec) - if err != nil { - return nil, nil, fmt.Errorf("failed to decode providers config: %w", err) - } - - if pconfig.OperatingSystemSpec.Raw == nil { - return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") - } - - rawConfig, err := alibabatypes.GetConfig(*pconfig) - if err != nil { - return nil, nil, fmt.Errorf("failed to decode alibaba providers config: %w", err) - } - - c := Config{} - c.AccessKeyID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.AccessKeyID, "ALIBABA_ACCESS_KEY_ID") - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"AccessKeyID\" field, error = %w", err) - } - c.AccessKeySecret, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.AccessKeySecret, "ALIBABA_ACCESS_KEY_SECRET") - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"AccessKeySecret\" field, error = %w", err) - } - c.InstanceType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.InstanceType) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"instanceType\" field, error = %w", err) - } - c.RegionID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.RegionID) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"regionID\" field, error = %w", err) - } - c.VSwitchID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VSwitchID) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"vSwitchID\" field, error = %w", err) - } - c.ZoneID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ZoneID) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"zoneID\" field, error = %w", err) - } - c.InternetMaxBandwidthOut, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.InternetMaxBandwidthOut) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"internetMaxBandwidthOut\" field, error = %w", err) - } - c.Labels = rawConfig.Labels - c.DiskType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.DiskType) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"diskType\" field, error = %w", err) - } - c.DiskSize, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.DiskSize) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"diskSize\" field, error = %w", err) - } - - return &c, pconfig, err -} - -func getClient(regionID, accessKeyID, accessKeySecret string) (*ecs.Client, error) { - client, err := ecs.NewClientWithAccessKey(regionID, accessKeyID, accessKeySecret) - if err != nil { - return nil, fmt.Errorf("failed to get Alibaba cloud client: %w", err) - } - return client, nil -} - -func getInstance(client *ecs.Client, instanceName string, uid string) (*ecs.Instance, error) { - describeInstanceRequest := ecs.CreateDescribeInstancesRequest() - describeInstanceRequest.InstanceName = instanceName - tag := []ecs.DescribeInstancesTag{ - { - Key: machineUIDTag, - Value: uid, - }, - } - describeInstanceRequest.Tag = &tag - - response, err := client.DescribeInstances(describeInstanceRequest) - if err != nil { - return nil, fmt.Errorf("failed to describe instance with instanceName: %s: %w", instanceName, err) - } - - if response.Instances.Instance == nil || - len(response.Instances.Instance) == 0 || - response.GetHttpStatus() == http.StatusNotFound { - return nil, cloudprovidererrors.ErrInstanceNotFound - } - - return &response.Instances.Instance[0], nil -} - -func (p *provider) getImageIDForOS(machineSpec clusterv1alpha1.MachineSpec, os providerconfigtypes.OperatingSystem) (string, error) { - c, _, err := p.getConfig(machineSpec.ProviderSpec) - if err != nil { - return "", fmt.Errorf("failed to get alibaba client: %w", err) - } - - client, err := getClient(c.RegionID, c.AccessKeyID, c.AccessKeySecret) - if err != nil { - return "", fmt.Errorf("failed to get alibaba client: %w", err) - } - - request := ecs.CreateDescribeImagesRequest() - request.InstanceType = "ecs.sn1ne.large" - request.OSType = "linux" - request.Architecture = "x86_64" - - response, err := client.DescribeImages(request) - if err != nil { - return "", fmt.Errorf("failed to describe alibaba images: %w", err) - } - - var availableImage = map[providerconfigtypes.OperatingSystem]string{} - for _, image := range response.Images.Image { - switch image.OSNameEn { - case ubuntuImageName: - availableImage[providerconfigtypes.OperatingSystemUbuntu] = image.ImageId - case centosImageName: - availableImage[providerconfigtypes.OperatingSystemCentOS] = image.ImageId - } - } - - if imageID, ok := availableImage[os]; ok { - return imageID, nil - } - - return "", providerconfigtypes.ErrOSNotSupported -} diff --git a/pkg/cloudprovider/provider/alibaba/types/types.go b/pkg/cloudprovider/provider/alibaba/types/types.go deleted file mode 100644 index f1f9a9de4..000000000 --- a/pkg/cloudprovider/provider/alibaba/types/types.go +++ /dev/null @@ -1,42 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" -) - -type RawConfig struct { - AccessKeyID *providerconfigtypes.ConfigVarString `json:"accessKeyID,omitempty"` - AccessKeySecret *providerconfigtypes.ConfigVarString `json:"accessKeySecret,omitempty"` - RegionID *providerconfigtypes.ConfigVarString `json:"regionID,omitempty"` - InstanceName *providerconfigtypes.ConfigVarString `json:"instanceName,omitempty"` - InstanceType *providerconfigtypes.ConfigVarString `json:"instanceType,omitempty"` - VSwitchID *providerconfigtypes.ConfigVarString `json:"vSwitchID,omitempty"` - InternetMaxBandwidthOut *providerconfigtypes.ConfigVarString `json:"internetMaxBandwidthOut,omitempty"` - Labels map[string]string `json:"labels,omitempty"` - ZoneID *providerconfigtypes.ConfigVarString `json:"zoneID,omitempty"` - DiskType *providerconfigtypes.ConfigVarString `json:"diskType,omitempty"` - DiskSize *providerconfigtypes.ConfigVarString `json:"diskSize,omitempty"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/anexia/helper_test.go b/pkg/cloudprovider/provider/anexia/helper_test.go deleted file mode 100644 index ceb1a7f41..000000000 --- a/pkg/cloudprovider/provider/anexia/helper_test.go +++ /dev/null @@ -1,155 +0,0 @@ -/* -Copyright 2022 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package anexia - -import ( - "encoding/json" - "testing" - - "github.com/gophercloud/gophercloud/testhelper" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" -) - -type jsonObject = map[string]interface{} - -type ProvisionVMTestCase struct { - ReconcileContext reconcileContext - AssertJSONBody func(jsonBody jsonObject) -} - -type ConfigTestCase struct { - Config anxtypes.RawConfig - Error error -} - -type ValidateCallTestCase struct { - Spec v1alpha1.MachineSpec - ExpectedError error -} - -func getSpecsForValidationTest(t *testing.T, configCases []ConfigTestCase) []ValidateCallTestCase { - var testCases []ValidateCallTestCase - - for _, configCase := range configCases { - jsonConfig, err := json.Marshal(configCase.Config) - testhelper.AssertNoErr(t, err) - jsonProviderConfig, err := json.Marshal(types.Config{ - CloudProviderSpec: runtime.RawExtension{Raw: jsonConfig}, - OperatingSystemSpec: runtime.RawExtension{Raw: []byte("{}")}, - }) - testhelper.AssertNoErr(t, err) - testCases = append(testCases, ValidateCallTestCase{ - Spec: v1alpha1.MachineSpec{ - ProviderSpec: v1alpha1.ProviderSpec{ - Value: &runtime.RawExtension{Raw: jsonProviderConfig}, - }, - }, - ExpectedError: configCase.Error, - }) - } - return testCases -} - -func newConfigVarString(str string) *types.ConfigVarString { - return &types.ConfigVarString{ - Value: str, - } -} - -// this generates a full config and allows hooking into it to e.g. remove a value. -func hookableConfig(hook func(*anxtypes.RawConfig)) anxtypes.RawConfig { - config := anxtypes.RawConfig{ - CPUs: 1, - - Memory: 2, - - Disks: []anxtypes.RawDisk{ - {Size: 5, PerformanceType: newConfigVarString("ENT6")}, - }, - - Token: newConfigVarString("test-token"), - VlanID: newConfigVarString("test-vlan"), - LocationID: newConfigVarString("test-location"), - TemplateID: newConfigVarString("test-template-id"), - } - - if hook != nil { - hook(&config) - } - - return config -} - -// this generates a full reconcileContext with some default values and allows hooking into it to e.g. remove/overwrite a value. -func hookableReconcileContext(locationID string, templateID string, hook func(*reconcileContext)) reconcileContext { - context := reconcileContext{ - Machine: &v1alpha1.Machine{ - ObjectMeta: metav1.ObjectMeta{Name: "TestMachine"}, - }, - Status: &anxtypes.ProviderStatus{}, - UserData: "", - Config: resolvedConfig{ - VlanID: "VLAN-ID", - LocationID: locationID, - TemplateID: templateID, - Disks: []resolvedDisk{ - { - RawDisk: anxtypes.RawDisk{ - Size: 5, - }, - }, - }, - RawConfig: anxtypes.RawConfig{ - CPUs: 5, - Memory: 5, - }, - }, - ProviderData: &cloudprovidertypes.ProviderData{ - Update: func(m *clusterv1alpha1.Machine, mods ...cloudprovidertypes.MachineModifier) error { - return nil - }, - }, - ProviderConfig: &providerconfigtypes.Config{ - Network: &providerconfigtypes.NetworkConfig{ - DNS: providerconfigtypes.DNSConfig{ - Servers: []string{ - "1.1.1.1", - "", - "192.168.0.1", - "192.168.0.2", - "192.168.0.3", - }, - }, - }, - }, - } - - if hook != nil { - hook(&context) - } - - return context -} diff --git a/pkg/cloudprovider/provider/anexia/instance.go b/pkg/cloudprovider/provider/anexia/instance.go deleted file mode 100644 index 06f1a0dd0..000000000 --- a/pkg/cloudprovider/provider/anexia/instance.go +++ /dev/null @@ -1,110 +0,0 @@ -/* -Copyright 2020 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package anexia - -import ( - "net" - - "go.anx.io/go-anxcloud/pkg/vsphere/info" - - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" - - v1 "k8s.io/api/core/v1" -) - -type anexiaInstance struct { - isCreating bool - isDeleting bool - info *info.Info - reservedAddresses []string -} - -func (ai *anexiaInstance) HostID() string { - return "" -} - -func (ai *anexiaInstance) Name() string { - if ai.info == nil { - return "" - } - - return ai.info.Name -} - -func (ai *anexiaInstance) ID() string { - if ai.info == nil { - return "" - } - - return ai.info.Identifier -} - -func (ai *anexiaInstance) ProviderID() string { - if ai == nil || ai.ID() == "" { - return "" - } - return ai.ID() -} - -func (ai *anexiaInstance) Addresses() map[string]v1.NodeAddressType { - addresses := map[string]v1.NodeAddressType{} - - if ai.reservedAddresses != nil { - for _, reservedIP := range ai.reservedAddresses { - addresses[reservedIP] = v1.NodeExternalIP - } - } - - if ai.info != nil { - for _, network := range ai.info.Network { - for _, ip := range network.IPv4 { - addresses[ip] = v1.NodeExternalIP - } - for _, ip := range network.IPv6 { - addresses[ip] = v1.NodeExternalIP - } - } - } - - for ip := range addresses { - parsed := net.ParseIP(ip) - if parsed.IsPrivate() { - addresses[ip] = v1.NodeInternalIP - } else { - addresses[ip] = v1.NodeExternalIP - } - } - - return addresses -} - -func (ai *anexiaInstance) Status() instance.Status { - if ai.isDeleting { - return instance.StatusDeleting - } - if ai.isCreating { - return instance.StatusCreating - } - - if ai.info != nil { - if ai.info.Status == anxtypes.MachinePoweredOn { - return instance.StatusRunning - } - } - return instance.StatusUnknown -} diff --git a/pkg/cloudprovider/provider/anexia/instance_test.go b/pkg/cloudprovider/provider/anexia/instance_test.go deleted file mode 100644 index 8340752a9..000000000 --- a/pkg/cloudprovider/provider/anexia/instance_test.go +++ /dev/null @@ -1,127 +0,0 @@ -/* -Copyright 2022 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package anexia - -import ( - "testing" - - "github.com/gophercloud/gophercloud/testhelper" - "go.anx.io/go-anxcloud/pkg/vsphere/info" - - v1 "k8s.io/api/core/v1" -) - -func TestAnexiaInstance(t *testing.T) { - addressCheck := func(t *testing.T, testcase string, instance *anexiaInstance, expected map[string]v1.NodeAddressType) { - t.Run(testcase, func(t *testing.T) { - addresses := instance.Addresses() - - testhelper.AssertDeepEquals(t, expected, addresses) - }) - } - - t.Run("empty instance", func(t *testing.T) { - instance := anexiaInstance{} - addressCheck(t, "no addresses", &instance, map[string]v1.NodeAddressType{}) - }) - - t.Run("instance with only reservedAddresses set", func(t *testing.T) { - instance := anexiaInstance{ - reservedAddresses: []string{"10.0.0.2", "fda0:23::2", "8.8.8.8", "2001:db8::2"}, - } - - addressCheck(t, "expected addresses", &instance, map[string]v1.NodeAddressType{ - "10.0.0.2": v1.NodeInternalIP, - "fda0:23::2": v1.NodeInternalIP, - "8.8.8.8": v1.NodeExternalIP, - "2001:db8::2": v1.NodeExternalIP, - }) - }) - - t.Run("instance with only info set", func(t *testing.T) { - instance := anexiaInstance{ - info: &info.Info{ - Network: []info.Network{ - { - IPv4: []string{"10.0.0.2"}, - IPv6: []string{"fda0:23::2"}, - }, - { - IPv4: []string{"8.8.8.8"}, - IPv6: []string{"2001:db8::2"}, - }, - }, - }, - } - - addressCheck(t, "expected addresses", &instance, map[string]v1.NodeAddressType{ - "10.0.0.2": v1.NodeInternalIP, - "fda0:23::2": v1.NodeInternalIP, - "8.8.8.8": v1.NodeExternalIP, - "2001:db8::2": v1.NodeExternalIP, - }) - }) - - t.Run("instance with both reservedAddresses and info set, full overlapping set", func(t *testing.T) { - instance := anexiaInstance{ - reservedAddresses: []string{"10.0.0.2", "fda0:23::2", "8.8.8.8", "2001:db8::2"}, - info: &info.Info{ - Network: []info.Network{ - { - IPv4: []string{"10.0.0.2"}, - IPv6: []string{"fda0:23::2"}, - }, - { - IPv4: []string{"8.8.8.8"}, - IPv6: []string{"2001:db8::2"}, - }, - }, - }, - } - - addressCheck(t, "expected addresses", &instance, map[string]v1.NodeAddressType{ - "10.0.0.2": v1.NodeInternalIP, - "fda0:23::2": v1.NodeInternalIP, - "8.8.8.8": v1.NodeExternalIP, - "2001:db8::2": v1.NodeExternalIP, - }) - }) - - t.Run("instance with both reservedAddresses and info set, some overlap, each adding some", func(t *testing.T) { - instance := anexiaInstance{ - reservedAddresses: []string{"10.0.0.2", "8.8.8.8", "2001:db8::2"}, - info: &info.Info{ - Network: []info.Network{ - { - IPv4: []string{"10.0.0.2"}, - IPv6: []string{"fda0:23::2"}, - }, - { - IPv6: []string{"2001:db8::2"}, - }, - }, - }, - } - - addressCheck(t, "expected addresses", &instance, map[string]v1.NodeAddressType{ - "10.0.0.2": v1.NodeInternalIP, - "fda0:23::2": v1.NodeInternalIP, - "8.8.8.8": v1.NodeExternalIP, - "2001:db8::2": v1.NodeExternalIP, - }) - }) -} diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go deleted file mode 100644 index 23b59da7f..000000000 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ /dev/null @@ -1,687 +0,0 @@ -/* -Copyright 2020 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package anexia - -import ( - "context" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "net/http" - "strings" - "sync" - "time" - - "go.anx.io/go-anxcloud/pkg/api" - corev1 "go.anx.io/go-anxcloud/pkg/apis/core/v1" - vspherev1 "go.anx.io/go-anxcloud/pkg/apis/vsphere/v1" - "go.anx.io/go-anxcloud/pkg/client" - anxclient "go.anx.io/go-anxcloud/pkg/client" - anxaddr "go.anx.io/go-anxcloud/pkg/ipam/address" - "go.anx.io/go-anxcloud/pkg/vsphere" - "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/progress" - anxvm "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/vm" - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/common/ssh" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - - "k8s.io/apimachinery/pkg/api/meta" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - k8stypes "k8s.io/apimachinery/pkg/types" -) - -const ( - ProvisionedType = "Provisioned" -) - -var ( - // ErrConfigDiskSizeAndDisks is returned when the config has both DiskSize and Disks set, which is unsupported. - ErrConfigDiskSizeAndDisks = errors.New("both the deprecated DiskSize and new Disks attribute are set") -) - -type provider struct { - configVarResolver *providerconfig.ConfigPointerVarResolver -} - -// resolvedDisk contains the resolved values from types.RawDisk. -type resolvedDisk struct { - anxtypes.RawDisk - - PerformanceType string -} - -// resolvedConfig contains the resolved values from types.RawConfig. -type resolvedConfig struct { - anxtypes.RawConfig - - Token string - VlanID string - LocationID string - TemplateID string - - Disks []resolvedDisk -} - -func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance instance.Instance, retErr error) { - status := getProviderStatus(log, machine) - log.Debugw("Machine status", "status", status) - - // ensure conditions are present on machine - ensureConditions(&status) - - config, providerCfg, err := p.getConfig(ctx, log, machine.Spec.ProviderSpec) - if err != nil { - return nil, fmt.Errorf("failed to get provider config: %w", err) - } - - ctx = createReconcileContext(ctx, reconcileContext{ - Status: &status, - UserData: userdata, - Config: *config, - ProviderData: data, - ProviderConfig: providerCfg, - Machine: machine, - }) - - _, client, err := getClient(config.Token) - if err != nil { - return nil, err - } - - // make sure status is reflected in Machine Object - defer func() { - // if error occurs during updating the machine object don't override the original error - retErr = anxtypes.NewMultiError(retErr, updateMachineStatus(machine, status, data.Update)) - }() - - // provision machine - err = provisionVM(ctx, log, client) - if err != nil { - return nil, anexiaErrorToTerminalError(err, "failed waiting for vm provisioning") - } - return p.Get(ctx, log, machine, data) -} - -func provisionVM(ctx context.Context, log *zap.SugaredLogger, client anxclient.Client) error { - reconcileContext := getReconcileContext(ctx) - vmAPI := vsphere.NewAPI(client) - - ctx, cancel := context.WithTimeout(ctx, anxtypes.CreateRequestTimeout) - defer cancel() - - status := reconcileContext.Status - if status.ProvisioningID == "" { - log.Info("Machine does not contain a provisioningID yet. Starting to provision") - - config := reconcileContext.Config - reservedIP, err := getIPAddress(ctx, log, client) - if err != nil { - return newError(common.CreateMachineError, "failed to reserve IP: %v", err) - } - networkInterfaces := []anxvm.Network{{ - NICType: anxtypes.VmxNet3NIC, - IPs: []string{reservedIP}, - VLAN: config.VlanID, - }} - - vm := vmAPI.Provisioning().VM().NewDefinition( - config.LocationID, - "templates", - config.TemplateID, - reconcileContext.Machine.Name, - config.CPUs, - config.Memory, - config.Disks[0].Size, - networkInterfaces, - ) - - vm.DiskType = config.Disks[0].PerformanceType - - for _, disk := range config.Disks[1:] { - vm.AdditionalDisks = append(vm.AdditionalDisks, anxvm.AdditionalDisk{ - SizeGBs: disk.Size, - Type: disk.PerformanceType, - }) - } - - vm.Script = base64.StdEncoding.EncodeToString([]byte(reconcileContext.UserData)) - - providerCfg := reconcileContext.ProviderConfig - if providerCfg.Network != nil { - for index, dnsServer := range providerCfg.Network.DNS.Servers { - switch index { - case 0: - vm.DNS1 = dnsServer - case 1: - vm.DNS2 = dnsServer - case 2: - vm.DNS3 = dnsServer - case 3: - vm.DNS4 = dnsServer - } - } - } - - // We generate a fresh SSH key but will never actually use it - we just want a valid public key to disable password authentication for our fresh VM. - sshKey, err := ssh.NewKey() - if err != nil { - return newError(common.CreateMachineError, "failed to generate ssh key: %v", err) - } - vm.SSH = sshKey.PublicKey - - provisionResponse, err := vmAPI.Provisioning().VM().Provision(ctx, vm, false) - meta.SetStatusCondition(&status.Conditions, v1.Condition{ - Type: ProvisionedType, - Status: v1.ConditionFalse, - Reason: "Provisioning", - Message: "provisioning request was sent", - }) - if err != nil { - return newError(common.CreateMachineError, "instance provisioning failed: %v", err) - } - - // we successfully sent a VM provisioning request to the API, we consider the IP as 'Bound' now - status.IPState = anxtypes.IPStateBound - - status.ProvisioningID = provisionResponse.Identifier - err = updateMachineStatus(reconcileContext.Machine, *status, reconcileContext.ProviderData.Update) - if err != nil { - return err - } - } - - log.Info("Using provisionID from machine to await completion") - - meta.SetStatusCondition(&status.Conditions, v1.Condition{ - Type: ProvisionedType, - Status: v1.ConditionTrue, - Reason: "Provisioned", - Message: "Machine has been successfully created", - }) - - return updateMachineStatus(reconcileContext.Machine, *status, reconcileContext.ProviderData.Update) -} - -var _engsup3404mutex sync.Mutex - -func getIPAddress(ctx context.Context, log *zap.SugaredLogger, client anxclient.Client) (string, error) { - reconcileContext := getReconcileContext(ctx) - status := reconcileContext.Status - - // only use IP if it is still unbound - if status.ReservedIP != "" && status.IPState == anxtypes.IPStateUnbound && (!status.IPProvisioningExpires.IsZero() && status.IPProvisioningExpires.After(time.Now())) { - log.Infow("Re-using already provisioned IP", "ip", status.ReservedIP) - return status.ReservedIP, nil - } - - _engsup3404mutex.Lock() - defer _engsup3404mutex.Unlock() - - log.Info("Creating a new IP for machine") - addrAPI := anxaddr.NewAPI(client) - config := reconcileContext.Config - res, err := addrAPI.ReserveRandom(ctx, anxaddr.ReserveRandom{ - LocationID: config.LocationID, - VlanID: config.VlanID, - Count: 1, - }) - if err != nil { - return "", newError(common.InvalidConfigurationMachineError, "failed to reserve an ip address: %v", err) - } - if len(res.Data) < 1 { - return "", newError(common.InsufficientResourcesMachineError, "no ip address is available for this machine") - } - - ip := res.Data[0].Address - status.ReservedIP = ip - status.IPState = anxtypes.IPStateUnbound - status.IPProvisioningExpires = time.Now().Add(anxtypes.IPProvisioningExpires) - - return ip, nil -} - -func isAlreadyProvisioning(ctx context.Context) bool { - status := getReconcileContext(ctx).Status - condition := meta.FindStatusCondition(status.Conditions, ProvisionedType) - lastChange := condition.LastTransitionTime.Time - const reasonInProvisioning = "InProvisioning" - if condition.Reason == reasonInProvisioning && time.Since(lastChange) > 5*time.Minute { - meta.SetStatusCondition(&status.Conditions, v1.Condition{ - Type: ProvisionedType, - Reason: "ReInitialising", - Message: "Could not find ongoing VM provisioning", - Status: v1.ConditionFalse, - }) - } - - return condition.Status == v1.ConditionFalse && condition.Reason == reasonInProvisioning -} - -func ensureConditions(status *anxtypes.ProviderStatus) { - conditions := [...]v1.Condition{ - {Type: ProvisionedType, Message: "", Status: v1.ConditionUnknown, Reason: "Initialising"}, - } - for _, condition := range conditions { - if meta.FindStatusCondition(status.Conditions, condition.Type) == nil { - meta.SetStatusCondition(&status.Conditions, condition) - } - } -} - -func resolveTemplateID(ctx context.Context, a api.API, config anxtypes.RawConfig, configVarResolver *providerconfig.ConfigPointerVarResolver, locationID string) (string, error) { - templateName, err := configVarResolver.GetConfigVarStringValue(config.Template) - if err != nil { - return "", fmt.Errorf("failed to get 'template': %w", err) - } - - templateBuild, err := configVarResolver.GetConfigVarStringValue(config.TemplateBuild) - if err != nil { - return "", fmt.Errorf("failed to get 'templateBuild': %w", err) - } - - template, err := vspherev1.FindNamedTemplate(ctx, a, templateName, templateBuild, corev1.Location{Identifier: locationID}) - if err != nil { - return "", fmt.Errorf("failed to retrieve named template: %w", err) - } - - return template.Identifier, nil -} - -func (p *provider) resolveConfig(ctx context.Context, log *zap.SugaredLogger, config anxtypes.RawConfig) (*resolvedConfig, error) { - var err error - ret := resolvedConfig{ - RawConfig: config, - } - - ret.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(config.Token, anxtypes.AnxTokenEnv) - if err != nil { - return nil, fmt.Errorf("failed to get 'token': %w", err) - } - - ret.LocationID, err = p.configVarResolver.GetConfigVarStringValue(config.LocationID) - if err != nil { - return nil, fmt.Errorf("failed to get 'locationID': %w", err) - } - - ret.TemplateID, err = p.configVarResolver.GetConfigVarStringValue(config.TemplateID) - if err != nil { - return nil, fmt.Errorf("failed to get 'templateID': %w", err) - } - - // when "templateID" is not set, we expect "template" to be - if ret.TemplateID == "" { - a, _, err := getClient(ret.Token) - if err != nil { - return nil, fmt.Errorf("failed initializing API clients: %w", err) - } - - templateID, err := resolveTemplateID(ctx, a, config, p.configVarResolver, ret.LocationID) - if err != nil { - return nil, fmt.Errorf("failed retrieving template id from named template: %w", err) - } - - ret.TemplateID = templateID - } - - ret.VlanID, err = p.configVarResolver.GetConfigVarStringValue(config.VlanID) - if err != nil { - return nil, fmt.Errorf("failed to get 'vlanID': %w", err) - } - - if config.DiskSize != 0 { - if len(config.Disks) != 0 { - return nil, ErrConfigDiskSizeAndDisks - } - - log.Info("Configuration uses the deprecated DiskSize attribute, please migrate to the Disks array instead.") - - config.Disks = []anxtypes.RawDisk{ - { - Size: config.DiskSize, - }, - } - config.DiskSize = 0 - } - - ret.Disks = make([]resolvedDisk, len(config.Disks)) - - for idx, disk := range config.Disks { - ret.Disks[idx].RawDisk = disk - - ret.Disks[idx].PerformanceType, err = p.configVarResolver.GetConfigVarStringValue(disk.PerformanceType) - if err != nil { - return nil, fmt.Errorf("failed to get 'performanceType' of disk %v: %w", idx, err) - } - } - - return &ret, nil -} - -func (p *provider) getConfig(ctx context.Context, log *zap.SugaredLogger, provSpec clusterv1alpha1.ProviderSpec) (*resolvedConfig, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerSpec.value is nil") - } - pconfig, err := providerconfigtypes.GetConfig(provSpec) - if err != nil { - return nil, nil, err - } - - if pconfig.OperatingSystemSpec.Raw == nil { - return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") - } - - rawConfig, err := anxtypes.GetConfig(*pconfig) - if err != nil { - return nil, nil, fmt.Errorf("error parsing provider config: %w", err) - } - - resolvedConfig, err := p.resolveConfig(ctx, log, *rawConfig) - if err != nil { - return nil, nil, fmt.Errorf("error resolving config: %w", err) - } - - return resolvedConfig, pconfig, nil -} - -// New returns an Anexia provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return &provider{configVarResolver: &providerconfig.ConfigPointerVarResolver{Cvr: configVarResolver}} -} - -// AddDefaults adds omitted optional values to the given MachineSpec. -func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { - return spec, nil -} - -// Validate returns success or failure based according to its ProviderSpec. -func (p *provider) Validate(ctx context.Context, log *zap.SugaredLogger, machinespec clusterv1alpha1.MachineSpec) error { - config, _, err := p.getConfig(ctx, log, machinespec.ProviderSpec) - if err != nil { - return fmt.Errorf("failed to parse config: %w", err) - } - - if config.Token == "" { - return errors.New("token not set") - } - - if config.CPUs == 0 { - return errors.New("cpu count is missing") - } - - if len(config.Disks) == 0 { - return errors.New("no disks configured") - } - - for _, disk := range config.Disks { - if disk.Size == 0 { - return errors.New("disk size is missing") - } - } - - if config.Memory == 0 { - return errors.New("memory size is missing") - } - - if config.LocationID == "" { - return errors.New("location id is missing") - } - - if config.TemplateID == "" { - return errors.New("no valid template configured") - } - - if config.VlanID == "" { - return errors.New("vlan id is missing") - } - - return nil -} - -func (p *provider) Get(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, pd *cloudprovidertypes.ProviderData) (instance.Instance, error) { - config, _, err := p.getConfig(ctx, log, machine.Spec.ProviderSpec) - if err != nil { - return nil, newError(common.InvalidConfigurationMachineError, "failed to retrieve config: %v", err) - } - - _, cli, err := getClient(config.Token) - if err != nil { - return nil, newError(common.InvalidConfigurationMachineError, "failed to create Anexia client: %v", err) - } - vsphereAPI := vsphere.NewAPI(cli) - - status := getProviderStatus(log, machine) - if err != nil { - return nil, newError(common.InvalidConfigurationMachineError, "failed to get machine status: %v", err) - } - - if status.InstanceID == "" && status.ProvisioningID == "" { - return nil, cloudprovidererrors.ErrInstanceNotFound - } - - if status.DeprovisioningID != "" { - // info endpoint no longer available for vm -> stop here - return &anexiaInstance{isDeleting: true}, nil - } - - if status.InstanceID == "" { - progress, err := vsphereAPI.Provisioning().Progress().Get(ctx, status.ProvisioningID) - if err != nil { - return nil, anexiaErrorToTerminalError(err, "failed to get provisioning progress") - } - if len(progress.Errors) > 0 { - return nil, fmt.Errorf("vm provisioning had errors: %s", strings.Join(progress.Errors, ",")) - } - if progress.Progress < 100 || progress.VMIdentifier == "" { - return &anexiaInstance{isCreating: true}, nil - } - - status.InstanceID = progress.VMIdentifier - - if err := updateMachineStatus(machine, status, pd.Update); err != nil { - return nil, fmt.Errorf("failed updating machine status: %w", err) - } - } - - instance := anexiaInstance{} - - if status.IPState == anxtypes.IPStateBound && status.ReservedIP != "" { - instance.reservedAddresses = []string{status.ReservedIP} - } - - timeoutCtx, cancel := context.WithTimeout(ctx, anxtypes.GetRequestTimeout) - defer cancel() - - info, err := vsphereAPI.Info().Get(timeoutCtx, status.InstanceID) - if err != nil { - return nil, anexiaErrorToTerminalError(err, "failed getting machine info") - } - instance.info = &info - - return &instance, nil -} - -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (string, string, error) { - return "", "", nil -} - -func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (isDeleted bool, retErr error) { - if inst, err := p.Get(ctx, log, machine, data); err != nil { - if cloudprovidererrors.IsNotFound(err) { - return true, nil - } - - return false, err - } else if inst.Status() == instance.StatusCreating { - log.Error("Failed to cleanup machine: instance is still creating") - return false, nil - } - - status := getProviderStatus(log, machine) - // make sure status is reflected in Machine Object - defer func() { - // if error occurs during updating the machine object don't override the original error - retErr = anxtypes.NewMultiError(retErr, updateMachineStatus(machine, status, data.Update)) - }() - - ensureConditions(&status) - config, _, err := p.getConfig(ctx, log, machine.Spec.ProviderSpec) - if err != nil { - return false, newError(common.InvalidConfigurationMachineError, "failed to parse MachineSpec: %v", err) - } - - _, cli, err := getClient(config.Token) - if err != nil { - return false, newError(common.InvalidConfigurationMachineError, "failed to create Anexia client: %v", err) - } - - vsphereAPI := vsphere.NewAPI(cli) - - deleteCtx, cancel := context.WithTimeout(ctx, anxtypes.DeleteRequestTimeout) - defer cancel() - - // first check whether there is an provisioning ongoing - if status.DeprovisioningID == "" { - response, err := vsphereAPI.Provisioning().VM().Deprovision(deleteCtx, status.InstanceID, false) - if err != nil { - var respErr *anxclient.ResponseError - // Only error if the error was not "not found" - if !(errors.As(err, &respErr) && respErr.ErrorData.Code == http.StatusNotFound) { - return false, newError(common.DeleteMachineError, "failed to delete machine: %v", err) - } - } - status.DeprovisioningID = response.Identifier - } - - return isTaskDone(deleteCtx, cli, status.DeprovisioningID) -} - -func isTaskDone(ctx context.Context, cli anxclient.Client, progressIdentifier string) (bool, error) { - response, err := progress.NewAPI(cli).Get(ctx, progressIdentifier) - if err != nil { - return false, err - } - - if len(response.Errors) != 0 { - taskErrors, _ := json.Marshal(response.Errors) - return true, fmt.Errorf("task failed with: %s", taskErrors) - } - - if response.Progress == 100 { - return true, nil - } - - return false, nil -} - -func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ k8stypes.UID) error { - return nil -} - -func (p *provider) MachineMetricsLabels(_ *clusterv1alpha1.Machine) (map[string]string, error) { - return map[string]string{}, nil -} - -func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { - return nil -} - -func getClient(token string) (api.API, anxclient.Client, error) { - tokenOpt := anxclient.TokenFromString(token) - client := anxclient.HTTPClient(&http.Client{Timeout: 120 * time.Second}) - - a, err := api.NewAPI(api.WithClientOptions(client, tokenOpt)) - if err != nil { - return nil, nil, fmt.Errorf("error creating generic API client: %w", err) - } - - legacyClient, err := anxclient.New(tokenOpt, client) - if err != nil { - return nil, nil, fmt.Errorf("error creating legacy client: %w", err) - } - - return a, legacyClient, nil -} - -func getProviderStatus(log *zap.SugaredLogger, machine *clusterv1alpha1.Machine) anxtypes.ProviderStatus { - var providerStatus anxtypes.ProviderStatus - status := machine.Status.ProviderStatus - if status != nil && status.Raw != nil { - if err := json.Unmarshal(status.Raw, &providerStatus); err != nil { - log.Error("Failed to parse status from machine object; status was discarded for machine") - return anxtypes.ProviderStatus{} - } - } - return providerStatus -} - -// newError creates a terminal error matching to the provider interface. -func newError(reason common.MachineStatusError, msg string, args ...interface{}) error { - return cloudprovidererrors.TerminalError{ - Reason: reason, - Message: fmt.Sprintf(msg, args...), - } -} - -// updateMachineStatus tries to update the machine status by any means -// an error will lead to a panic. -func updateMachineStatus(machine *clusterv1alpha1.Machine, status anxtypes.ProviderStatus, updater cloudprovidertypes.MachineUpdater) error { - rawStatus, err := json.Marshal(status) - if err != nil { - return err - } - err = updater(machine, func(machine *clusterv1alpha1.Machine) { - machine.Status.ProviderStatus = &runtime.RawExtension{ - Raw: rawStatus, - } - }) - - if err != nil { - return err - } - - return nil -} - -func anexiaErrorToTerminalError(err error, msg string) error { - var httpError api.HTTPError - if errors.As(err, &httpError) && (httpError.StatusCode() == http.StatusForbidden || httpError.StatusCode() == http.StatusUnauthorized) { - return cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: "Request was rejected due to invalid credentials", - } - } - - var responseError *client.ResponseError - if errors.As(err, &responseError) && (responseError.ErrorData.Code == http.StatusForbidden || responseError.ErrorData.Code == http.StatusUnauthorized) { - return cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: "Request was rejected due to invalid credentials", - } - } - - return fmt.Errorf("%s: %w", msg, err) -} diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go deleted file mode 100644 index f89d470d8..000000000 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ /dev/null @@ -1,510 +0,0 @@ -/* -Copyright 2022 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package anexia - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "net/http" - "net/http/httptest" - "net/url" - "strings" - "testing" - "time" - - "github.com/gophercloud/gophercloud/testhelper" - "go.anx.io/go-anxcloud/pkg/api" - "go.anx.io/go-anxcloud/pkg/api/mock" - corev1 "go.anx.io/go-anxcloud/pkg/apis/core/v1" - vspherev1 "go.anx.io/go-anxcloud/pkg/apis/vsphere/v1" - "go.anx.io/go-anxcloud/pkg/client" - anxclient "go.anx.io/go-anxcloud/pkg/client" - "go.anx.io/go-anxcloud/pkg/core" - "go.anx.io/go-anxcloud/pkg/ipam/address" - "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/progress" - "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/vm" - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" -) - -const ( - TestIdentifier = "TestIdent" - testTemplateName = "test-template" -) - -func TestAnexiaProvider(t *testing.T) { - testhelper.SetupHTTP() - client, server := anxclient.NewTestClient(nil, testhelper.Mux) - log := zap.NewNop().Sugar() - - a := mock.NewMockAPI() - a.FakeExisting(&vspherev1.Template{Identifier: "TEMPLATE-ID-OLD-BUILD", Name: testTemplateName, Build: "b01"}) - a.FakeExisting(&vspherev1.Template{Identifier: "TEMPLATE-ID", Name: testTemplateName, Build: "b02"}) - a.FakeExisting(&vspherev1.Template{Identifier: "WRONG-TEMPLATE-NAME", Name: "Wrong Template Name", Build: "b02"}) - a.FakeExisting(&vspherev1.Template{Identifier: "TEMPLATE-ID-NO-NETWORK-CONFIG", Name: "no-network-config", Build: "b03"}) - a.FakeExisting(&vspherev1.Template{Identifier: "TEMPLATE-ID-ADDITIONAL-DISKS", Name: "additional-disks", Build: "b03"}) - - t.Cleanup(func() { - testhelper.TeardownHTTP() - server.Close() - }) - - t.Run("Test provision VM", func(t *testing.T) { - t.Parallel() - - testCases := []ProvisionVMTestCase{ - { - // Provision a generic VM with some custom dns entries - ReconcileContext: hookableReconcileContext("LOCATION-ID", "TEMPLATE-ID", func(rc *reconcileContext) { - rc.ProviderConfig = &providerconfigtypes.Config{ - Network: &providerconfigtypes.NetworkConfig{ - DNS: providerconfigtypes.DNSConfig{ - Servers: []string{ - "1.1.1.1", - "", - "192.168.0.1", - "192.168.0.2", - "192.168.0.3", - }, - }, - }, - } - }), - AssertJSONBody: func(jsonBody jsonObject) { - testhelper.AssertEquals(t, jsonBody["cpu_performance_type"], "performance") - testhelper.AssertEquals(t, jsonBody["hostname"], "TestMachine") - testhelper.AssertEquals(t, jsonBody["memory_mb"], json.Number("5")) - - testhelper.AssertEquals(t, jsonBody["dns1"], "1.1.1.1") - _, exists := jsonBody["dns2"] - testhelper.AssertEquals(t, exists, false) - testhelper.AssertEquals(t, jsonBody["dns3"], "192.168.0.1") - testhelper.AssertEquals(t, jsonBody["dns4"], "192.168.0.2") - - networkArray := jsonBody["network"].([]interface{}) - networkObject := networkArray[0].(jsonObject) - testhelper.AssertEquals(t, networkObject["vlan"], "VLAN-ID") - testhelper.AssertEquals(t, networkObject["nic_type"], "vmxnet3") - testhelper.AssertEquals(t, networkObject["ips"].([]interface{})[0], "8.8.8.8") - }, - }, - { - // Provision a VM without any ProviderConfig - ReconcileContext: hookableReconcileContext("LOCATION-ID", "TEMPLATE-ID-NO-NETWORK-CONFIG", func(rc *reconcileContext) { - rc.ProviderConfig = &providerconfigtypes.Config{} - }), - AssertJSONBody: func(jsonBody jsonObject) { - _, exists := jsonBody["dns1"] - testhelper.AssertEquals(t, exists, false) - _, exists = jsonBody["dns2"] - testhelper.AssertEquals(t, exists, false) - _, exists = jsonBody["dns3"] - testhelper.AssertEquals(t, exists, false) - _, exists = jsonBody["dns4"] - testhelper.AssertEquals(t, exists, false) - }, - }, - { - ReconcileContext: hookableReconcileContext("LOCATION-ID", "ADDITIONAL-DISKS", func(rc *reconcileContext) { - rc.Config.Disks = append(rc.Config.Disks, resolvedDisk{ - RawDisk: anxtypes.RawDisk{ - Size: 10, - }, - PerformanceType: "STD1", - }) - }), - - AssertJSONBody: func(jsonBody jsonObject) { - testhelper.AssertEquals(t, json.Number("5"), jsonBody["disk_gb"]) - testhelper.AssertJSONEquals(t, `[{"gb":10,"type":"STD1"}]`, jsonBody["additional_disks"]) - }, - }, - } - - testhelper.Mux.HandleFunc("/api/ipam/v1/address/reserve/ip/count.json", func(writer http.ResponseWriter, request *http.Request) { - err := json.NewEncoder(writer).Encode(address.ReserveRandomSummary{ - Data: []address.ReservedIP{ - { - ID: "IP-ID", - Address: "8.8.8.8", - }, - }, - }) - testhelper.AssertNoErr(t, err) - }) - - for _, testCase := range testCases { - templateID := testCase.ReconcileContext.Config.TemplateID - locationID := testCase.ReconcileContext.Config.LocationID - - testhelper.Mux.HandleFunc(fmt.Sprintf("/api/vsphere/v1/provisioning/vm.json/%s/templates/%s", locationID, templateID), func(writer http.ResponseWriter, request *http.Request) { - testhelper.TestMethod(t, request, http.MethodPost) - var jsonBody jsonObject - decoder := json.NewDecoder(request.Body) - decoder.UseNumber() - testhelper.AssertNoErr(t, decoder.Decode(&jsonBody)) - - testCase.AssertJSONBody(jsonBody) - - err := json.NewEncoder(writer).Encode(vm.ProvisioningResponse{ - Progress: 100, - Errors: nil, - Identifier: templateID, - Queued: false, - }) - testhelper.AssertNoErr(t, err) - }) - - testhelper.Mux.HandleFunc(fmt.Sprintf("/api/vsphere/v1/provisioning/progress.json/%s", templateID), func(writer http.ResponseWriter, request *http.Request) { - testhelper.TestMethod(t, request, http.MethodGet) - - err := json.NewEncoder(writer).Encode(progress.Progress{ - TaskIdentifier: templateID, - Queued: false, - Progress: 100, - VMIdentifier: "VM-IDENTIFIER", - Errors: nil, - }) - testhelper.AssertNoErr(t, err) - }) - - ctx := createReconcileContext(context.Background(), testCase.ReconcileContext) - - err := provisionVM(ctx, log, client) - testhelper.AssertNoErr(t, err) - } - }) - - t.Run("Test resolve template", func(t *testing.T) { - t.Parallel() - - type testCase struct { - config anxtypes.RawConfig - expectedError string - expectedTemplateID string - } - - testCases := []testCase{ - // fail - { - // Template name does not exist - config: hookableConfig(func(c *anxtypes.RawConfig) { - c.Template = newConfigVarString("non-existing-template-name") - }), - expectedError: "failed to retrieve named template", - }, - { - // Template build does not exist - config: hookableConfig(func(c *anxtypes.RawConfig) { - c.Template = newConfigVarString(testTemplateName) - c.TemplateBuild = newConfigVarString("b42") - }), - expectedError: "failed to retrieve named template", - }, - // pass - { - // With named template - config: hookableConfig(func(c *anxtypes.RawConfig) { c.Template = newConfigVarString(testTemplateName); c.TemplateID = newConfigVarString("") }), - expectedTemplateID: "TEMPLATE-ID", - }, - { - // With named template and not latest build - config: hookableConfig(func(c *anxtypes.RawConfig) { - c.Template = newConfigVarString(testTemplateName) - c.TemplateBuild = newConfigVarString("b01") - }), - expectedTemplateID: "TEMPLATE-ID-OLD-BUILD", - }, - } - - provider := New(nil).(*provider) - for _, testCase := range testCases { - templateID, err := resolveTemplateID(context.TODO(), a, testCase.config, provider.configVarResolver, "foo") - if testCase.expectedError != "" { - if err != nil { - testhelper.AssertErr(t, err) - testhelper.AssertEquals(t, true, strings.Contains(err.Error(), testCase.expectedError)) - continue - } - } else { - testhelper.AssertNoErr(t, err) - testhelper.AssertEquals(t, testCase.expectedTemplateID, templateID) - } - } - }) - - t.Run("Test is VM Provisioning", func(t *testing.T) { - t.Parallel() - providerStatus := anxtypes.ProviderStatus{ - Conditions: []metav1.Condition{ - { - Type: ProvisionedType, - Reason: "InProvisioning", - Status: metav1.ConditionFalse, - }, - }, - } - ctx := createReconcileContext(context.Background(), reconcileContext{ - Status: &providerStatus, - UserData: "", - Config: resolvedConfig{}, - ProviderData: nil, - }) - - condition := meta.FindStatusCondition(providerStatus.Conditions, ProvisionedType) - condition.LastTransitionTime = metav1.Time{Time: time.Now().Add(-1 * time.Minute)} - testhelper.AssertEquals(t, true, isAlreadyProvisioning(ctx)) - - condition.Reason = "Provisioned" - condition.Status = metav1.ConditionTrue - testhelper.AssertEquals(t, false, isAlreadyProvisioning(ctx)) - - condition.Reason = "InProvisioning" - condition.Status = metav1.ConditionFalse - condition.LastTransitionTime = metav1.Time{Time: time.Now().Add(-10 * time.Minute)} - testhelper.AssertEquals(t, false, isAlreadyProvisioning(ctx)) - testhelper.AssertEquals(t, condition.Reason, "ReInitialising") - }) - - t.Run("Test getIPAddress", func(t *testing.T) { - t.Parallel() - providerStatus := &anxtypes.ProviderStatus{ - ReservedIP: "", - IPState: "", - } - ctx := createReconcileContext(context.Background(), reconcileContext{Status: providerStatus}) - - t.Run("with unbound reserved IP", func(t *testing.T) { - expectedIP := "8.8.8.8" - providerStatus.ReservedIP = expectedIP - providerStatus.IPState = anxtypes.IPStateUnbound - providerStatus.IPProvisioningExpires = time.Now().Add(anxtypes.IPProvisioningExpires) - reservedIP, err := getIPAddress(ctx, log, client) - testhelper.AssertNoErr(t, err) - testhelper.AssertEquals(t, expectedIP, reservedIP) - }) - }) -} - -func TestValidate(t *testing.T) { - t.Parallel() - - var configCases []ConfigTestCase - configCases = append(configCases, - ConfigTestCase{ - Config: hookableConfig(func(c *anxtypes.RawConfig) { c.Token.Value = "" }), - Error: errors.New("token not set"), - }, - ConfigTestCase{ - Config: hookableConfig(func(c *anxtypes.RawConfig) { c.CPUs = 0 }), - Error: errors.New("cpu count is missing"), - }, - ConfigTestCase{ - Config: hookableConfig(func(c *anxtypes.RawConfig) { c.Disks = []anxtypes.RawDisk{} }), - Error: errors.New("no disks configured"), - }, - ConfigTestCase{ - Config: hookableConfig(func(c *anxtypes.RawConfig) { c.DiskSize = 10 }), - Error: ErrConfigDiskSizeAndDisks, - }, - ConfigTestCase{ - Config: hookableConfig(func(c *anxtypes.RawConfig) { c.Disks[0].Size = 0 }), - Error: errors.New("disk size is missing"), - }, - ConfigTestCase{ - Config: hookableConfig(func(c *anxtypes.RawConfig) { c.Memory = 0 }), - Error: errors.New("memory size is missing"), - }, - ConfigTestCase{ - Config: hookableConfig(func(c *anxtypes.RawConfig) { c.LocationID.Value = "" }), - Error: errors.New("location id is missing"), - }, - ConfigTestCase{ - Config: hookableConfig(func(c *anxtypes.RawConfig) { c.VlanID.Value = "" }), - Error: errors.New("vlan id is missing"), - }, - ConfigTestCase{ - Config: hookableConfig(func(c *anxtypes.RawConfig) { c.DiskSize = 10; c.Disks = []anxtypes.RawDisk{} }), - Error: nil, - }, - ConfigTestCase{ - Config: hookableConfig(nil), - Error: nil, - }, - ) - - provider := New(nil) - for _, testCase := range getSpecsForValidationTest(t, configCases) { - err := provider.Validate(context.Background(), zap.NewNop().Sugar(), testCase.Spec) - if testCase.ExpectedError != nil { - if !errors.Is(err, testCase.ExpectedError) { - testhelper.AssertEquals(t, testCase.ExpectedError.Error(), err.Error()) - } - } else { - testhelper.AssertEquals(t, testCase.ExpectedError, err) - } - } -} - -func TestEnsureConditions(t *testing.T) { - t.Parallel() - status := anxtypes.ProviderStatus{} - - ensureConditions(&status) - - condition := meta.FindStatusCondition(status.Conditions, ProvisionedType) - if condition == nil { - t.Fatal("condition should not be nil") - } - testhelper.AssertEquals(t, metav1.ConditionUnknown, condition.Status) - testhelper.AssertEquals(t, "Initialising", condition.Reason) -} - -func TestGetProviderStatus(t *testing.T) { - t.Parallel() - - machine := &v1alpha1.Machine{} - providerStatus := anxtypes.ProviderStatus{ - InstanceID: "InstanceID", - } - providerStatusJSON, err := json.Marshal(providerStatus) - testhelper.AssertNoErr(t, err) - machine.Status.ProviderStatus = &runtime.RawExtension{Raw: providerStatusJSON} - - returnedStatus := getProviderStatus(zap.NewNop().Sugar(), machine) - - testhelper.AssertEquals(t, "InstanceID", returnedStatus.InstanceID) -} - -func TestUpdateStatus(t *testing.T) { - t.Parallel() - machine := &v1alpha1.Machine{} - providerStatus := anxtypes.ProviderStatus{ - InstanceID: "InstanceID", - } - providerStatusJSON, err := json.Marshal(providerStatus) - testhelper.AssertNoErr(t, err) - machine.Status.ProviderStatus = &runtime.RawExtension{Raw: providerStatusJSON} - - called := false - err = updateMachineStatus(machine, providerStatus, func(paramMachine *v1alpha1.Machine, modifier ...cloudprovidertypes.MachineModifier) error { - called = true - testhelper.AssertEquals(t, machine, paramMachine) - status := getProviderStatus(zap.NewNop().Sugar(), machine) - testhelper.AssertEquals(t, status.InstanceID, providerStatus.InstanceID) - return nil - }) - - testhelper.AssertEquals(t, true, called) - testhelper.AssertNoErr(t, err) -} - -func Test_anexiaErrorToTerminalError(t *testing.T) { - forbiddenMockHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusForbidden) - _, err := w.Write([]byte(`{"error": {"code": 403}}`)) - testhelper.AssertNoErr(t, err) - }) - - unauthorizedMockHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusUnauthorized) - _, err := w.Write([]byte(`{"error": {"code": 401}}`)) - testhelper.AssertNoErr(t, err) - }) - - legacyClientRun := func(url string) error { - client, err := client.New(client.BaseURL(url), client.IgnoreMissingToken(), client.ParseEngineErrors(true)) - testhelper.AssertNoErr(t, err) - _, err = core.NewAPI(client).Location().List(context.TODO(), 1, 1, "", "") - return err - } - - apiClientRun := func(url string) error { - client, err := api.NewAPI(api.WithClientOptions( - client.BaseURL(url), - client.IgnoreMissingToken(), - )) - testhelper.AssertNoErr(t, err) - return client.Get(context.TODO(), &corev1.Location{Identifier: "foo"}) - } - - testCases := []struct { - name string - mockHandler http.HandlerFunc - run func(url string) error - }{ - { - name: "api client returns forbidden", - mockHandler: forbiddenMockHandler, - run: apiClientRun, - }, - { - name: "api client returns unauthorized", - mockHandler: unauthorizedMockHandler, - run: apiClientRun, - }, - { - name: "legacy client returns forbidden", - mockHandler: forbiddenMockHandler, - run: legacyClientRun, - }, - { - name: "legacy client returns unauthorized", - mockHandler: unauthorizedMockHandler, - run: legacyClientRun, - }, - } - - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - srv := httptest.NewServer(testCase.mockHandler) - defer srv.Close() - - err := anexiaErrorToTerminalError(testCase.run(srv.URL), "foo") - if ok, _, _ := cloudprovidererrors.IsTerminalError(err); !ok { - t.Errorf("unexpected error %#v, expected TerminalError", err) - } - }) - } - - t.Run("api client 404 HTTPError shouldn't convert to TerminalError", func(t *testing.T) { - err := api.NewHTTPError(http.StatusNotFound, "GET", &url.URL{}, errors.New("foo")) - err = anexiaErrorToTerminalError(err, "foo") - if ok, _, _ := cloudprovidererrors.IsTerminalError(err); ok { - t.Errorf("unexpected error %#v, expected no TerminalError", err) - } - }) - - t.Run("legacy api client unspecific ResponseError shouldn't convert to TerminalError", func(t *testing.T) { - var err error = &client.ResponseError{} - err = anexiaErrorToTerminalError(err, "foo") - if ok, _, _ := cloudprovidererrors.IsTerminalError(err); ok { - t.Errorf("unexpected error %#v, expected no TerminalError", err) - } - }) -} diff --git a/pkg/cloudprovider/provider/anexia/reconcile_context.go b/pkg/cloudprovider/provider/anexia/reconcile_context.go deleted file mode 100644 index 2a14d8724..000000000 --- a/pkg/cloudprovider/provider/anexia/reconcile_context.go +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright 2022 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package anexia - -import ( - "context" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" -) - -type contextKey byte - -const machineReconcileContextKey contextKey = 0 - -type reconcileContext struct { - Machine *v1alpha1.Machine - Status *anxtypes.ProviderStatus - UserData string - Config resolvedConfig - ProviderData *cloudprovidertypes.ProviderData - ProviderConfig *providerconfigtypes.Config -} - -func createReconcileContext(ctx context.Context, cc reconcileContext) context.Context { - return context.WithValue(ctx, machineReconcileContextKey, cc) -} - -func getReconcileContext(ctx context.Context) reconcileContext { - rawContext := ctx.Value(machineReconcileContextKey) - if recContext, ok := rawContext.(reconcileContext); ok { - return recContext - } - - return reconcileContext{} -} diff --git a/pkg/cloudprovider/provider/anexia/types/errors.go b/pkg/cloudprovider/provider/anexia/types/errors.go deleted file mode 100644 index 65f7ab6d2..000000000 --- a/pkg/cloudprovider/provider/anexia/types/errors.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2022 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "fmt" - "strings" -) - -// MultiError represent multiple errors at the same time. -type MultiErrors []error - -func (r MultiErrors) Error() string { - errString := make([]string, len(r)) - for i, err := range r { - errString[i] = fmt.Sprintf("Error %d: %s", i, err) - } - return fmt.Sprintf("Multiple errors occurred:\n%s", strings.Join(errString, "\n")) -} - -func NewMultiError(errs ...error) error { - var combinedErr []error - for _, err := range errs { - if err == nil { - continue - } - combinedErr = append(combinedErr, err) - } - - if len(combinedErr) > 0 { - return MultiErrors(combinedErr) - } - - return nil -} diff --git a/pkg/cloudprovider/provider/anexia/types/types.go b/pkg/cloudprovider/provider/anexia/types/types.go deleted file mode 100644 index bc692439f..000000000 --- a/pkg/cloudprovider/provider/anexia/types/types.go +++ /dev/null @@ -1,89 +0,0 @@ -/* -Copyright 2020 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "time" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -const ( - AnxTokenEnv = "ANEXIA_TOKEN" - - CreateRequestTimeout = 15 * time.Minute - GetRequestTimeout = 1 * time.Minute - DeleteRequestTimeout = 1 * time.Minute - - IPStateBound = "Bound" - IPStateUnbound = "Unbound" - IPProvisioningExpires = 1800 * time.Second - - VmxNet3NIC = "vmxnet3" - MachinePoweredOn = "poweredOn" -) - -var StatusUpdateFailed = cloudprovidererrors.TerminalError{ - Reason: common.UpdateMachineError, - Message: "Failed to update the machine status", -} - -// RawDisk specifies a single disk, with some values maybe being fetched from secrets. -type RawDisk struct { - Size int `json:"size"` - PerformanceType *providerconfigtypes.ConfigVarString `json:"performanceType"` -} - -// RawConfig contains all the configuration values for VMs to create, with some values maybe being fetched from secrets. -type RawConfig struct { - Token *providerconfigtypes.ConfigVarString `json:"token,omitempty"` - VlanID *providerconfigtypes.ConfigVarString `json:"vlanID"` - LocationID *providerconfigtypes.ConfigVarString `json:"locationID"` - - TemplateID *providerconfigtypes.ConfigVarString `json:"templateID"` - Template *providerconfigtypes.ConfigVarString `json:"template"` - TemplateBuild *providerconfigtypes.ConfigVarString `json:"templateBuild"` - - CPUs int `json:"cpus"` - Memory int `json:"memory"` - - // Deprecated, use Disks instead. - DiskSize int `json:"diskSize"` - - Disks []RawDisk `json:"disks"` -} - -type ProviderStatus struct { - InstanceID string `json:"instanceID"` - ProvisioningID string `json:"provisioningID"` - DeprovisioningID string `json:"deprovisioningID"` - ReservedIP string `json:"reservedIP"` - IPState string `json:"ipState"` - IPProvisioningExpires time.Time `json:"ipProvisioningExpires"` - Conditions []v1.Condition `json:"conditions,omitempty"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/azure/create_delete_resources.go b/pkg/cloudprovider/provider/azure/create_delete_resources.go deleted file mode 100644 index 242fd765f..000000000 --- a/pkg/cloudprovider/provider/azure/create_delete_resources.go +++ /dev/null @@ -1,403 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package azure - -import ( - "context" - "fmt" - - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-05-01/network" - "github.com/Azure/go-autorest/autorest/azure/auth" - "github.com/Azure/go-autorest/autorest/to" - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - - "k8s.io/apimachinery/pkg/types" -) - -// deleteInterfacesByMachineUID will remove all network interfaces tagged with the specific machine's UID. -// The machine has to be deleted or disassociated with the interfaces beforehand, since Azure won't allow -// us to remove interfaces connected to a VM. -func deleteInterfacesByMachineUID(ctx context.Context, c *config, machineUID types.UID) error { - ifClient, err := getInterfacesClient(c) - if err != nil { - return fmt.Errorf("failed to create interfaces client: %w", err) - } - - list, err := ifClient.List(ctx, c.ResourceGroup) - if err != nil { - return fmt.Errorf("failed to list interfaces in resource group %q", c.ResourceGroup) - } - - var allInterfaces []network.Interface - - for list.NotDone() { - allInterfaces = append(allInterfaces, list.Values()...) - if err = list.NextWithContext(ctx); err != nil { - return fmt.Errorf("failed to iterate the result list: %w", err) - } - } - - for _, iface := range allInterfaces { - if iface.Tags != nil && iface.Tags[machineUIDTag] != nil && *iface.Tags[machineUIDTag] == string(machineUID) { - future, err := ifClient.Delete(ctx, c.ResourceGroup, *iface.Name) - if err != nil { - return err - } - - if err = future.WaitForCompletionRef(ctx, ifClient.Client); err != nil { - return err - } - } - } - - return nil -} - -// deleteIPAddressesByMachineUID will remove public IP addresses tagged with the specific machine's UID. -// Their respective network interfaces have to be deleted or disassociated with the IPs beforehand, since -// Azure won't allow us to remove IPs connected to NICs. -func deleteIPAddressesByMachineUID(ctx context.Context, c *config, machineUID types.UID) error { - ipClient, err := getIPClient(c) - if err != nil { - return fmt.Errorf("failed to create IP addresses client: %w", err) - } - - list, err := ipClient.List(ctx, c.ResourceGroup) - if err != nil { - return fmt.Errorf("failed to list public IP addresses in resource group %q", c.ResourceGroup) - } - - var allIPs []network.PublicIPAddress - - for list.NotDone() { - allIPs = append(allIPs, list.Values()...) - if err = list.Next(); err != nil { - return fmt.Errorf("failed to iterate the result list: %w", err) - } - } - - for _, ip := range allIPs { - if ip.Tags != nil && ip.Tags[machineUIDTag] != nil && *ip.Tags[machineUIDTag] == string(machineUID) { - future, err := ipClient.Delete(ctx, c.ResourceGroup, *ip.Name) - if err != nil { - return err - } - - if err = future.WaitForCompletionRef(ctx, ipClient.Client); err != nil { - return err - } - } - } - - return nil -} - -func deleteVMsByMachineUID(ctx context.Context, c *config, machineUID types.UID) error { - vmClient, err := getVMClient(c) - if err != nil { - return err - } - - list, err := vmClient.List(ctx, c.ResourceGroup, "") - - if err != nil { - return err - } - - var allServers []compute.VirtualMachine - - for list.NotDone() { - allServers = append(allServers, list.Values()...) - if err = list.Next(); err != nil { - return fmt.Errorf("failed to iterate the result list: %w", err) - } - } - - for _, vm := range allServers { - if vm.Tags != nil && vm.Tags[machineUIDTag] != nil && *vm.Tags[machineUIDTag] == string(machineUID) { - future, err := vmClient.Delete(ctx, c.ResourceGroup, *vm.Name, nil) - if err != nil { - return err - } - - if err = future.WaitForCompletionRef(ctx, vmClient.Client); err != nil { - return err - } - } - } - - return nil -} - -func deleteDisksByMachineUID(ctx context.Context, c *config, machineUID types.UID) error { - disksClient, err := getDisksClient(c) - if err != nil { - return fmt.Errorf("failed to get disks client: %w", err) - } - - matchingDisks, err := getDisksByMachineUID(ctx, disksClient, machineUID) - if err != nil { - return err - } - - for _, disk := range matchingDisks { - future, err := disksClient.Delete(ctx, c.ResourceGroup, *disk.Name) - if err != nil { - return fmt.Errorf("failed to delete disk %s: %w", *disk.Name, err) - } - - if err = future.WaitForCompletionRef(ctx, disksClient.Client); err != nil { - return fmt.Errorf("failed to wait for deletion of disk %s: %w", *disk.Name, err) - } - } - - return nil -} - -func getDisksByMachineUID(ctx context.Context, disksClient *compute.DisksClient, UID types.UID) ([]compute.Disk, error) { - list, err := disksClient.List(ctx) - if err != nil { - return nil, fmt.Errorf("failed to list disks: %w", err) - } - - var allDisks, matchingDisks []compute.Disk - for list.NotDone() { - allDisks = append(allDisks, list.Values()...) - if err = list.Next(); err != nil { - return nil, fmt.Errorf("failed to iterate the result list: %w", err) - } - } - - for _, disk := range allDisks { - if disk.Tags != nil && disk.Tags[machineUIDTag] != nil && *disk.Tags[machineUIDTag] == string(UID) { - matchingDisks = append(matchingDisks, disk) - } - } - - return matchingDisks, nil -} - -func createOrUpdatePublicIPAddress(ctx context.Context, log *zap.SugaredLogger, ipName string, ipVersion network.IPVersion, sku network.PublicIPAddressSkuName, ipAllocationMethod network.IPAllocationMethod, machineUID types.UID, c *config) (*network.PublicIPAddress, error) { - log.Infow("Creating public IP", "name", ipName) - ipClient, err := getIPClient(c) - if err != nil { - return nil, err - } - - ipParams := network.PublicIPAddress{ - Name: to.StringPtr(ipName), - Location: to.StringPtr(c.Location), - PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ - PublicIPAddressVersion: ipVersion, - PublicIPAllocationMethod: ipAllocationMethod, - }, - Tags: map[string]*string{machineUIDTag: to.StringPtr(string(machineUID))}, - Zones: &c.Zones, - Sku: &network.PublicIPAddressSku{ - Name: sku, - }, - } - - future, err := ipClient.CreateOrUpdate(ctx, c.ResourceGroup, ipName, ipParams) - if err != nil { - return nil, fmt.Errorf("failed to create public IP address: %w", err) - } - - err = future.WaitForCompletionRef(ctx, ipClient.Client) - if err != nil { - return nil, fmt.Errorf("failed to retrieve public IP address creation result: %w", err) - } - - if _, err = future.Result(*ipClient); err != nil { - return nil, fmt.Errorf("failed to create public IP address: %w", err) - } - - log.Infow("Fetching info for IP address", "name", ipName) - ip, err := getPublicIPAddress(ctx, ipName, c.ResourceGroup, ipClient) - if err != nil { - return nil, fmt.Errorf("failed to fetch info about public IP %q: %w", ipName, err) - } - - return ip, nil -} - -func getPublicIPAddress(ctx context.Context, ipName string, resourceGroup string, ipClient *network.PublicIPAddressesClient) (*network.PublicIPAddress, error) { - ip, err := ipClient.Get(ctx, resourceGroup, ipName, "") - if err != nil { - return nil, err - } - - return &ip, nil -} - -func getSubnet(ctx context.Context, c *config) (network.Subnet, error) { - subnetsClient, err := getSubnetsClient(c) - if err != nil { - return network.Subnet{}, fmt.Errorf("failed to create subnets client: %w", err) - } - - return subnetsClient.Get(ctx, c.VNetResourceGroup, c.VNetName, c.SubnetName, "") -} - -func getSKU(ctx context.Context, log *zap.SugaredLogger, c *config) (compute.ResourceSku, error) { - cacheLock.Lock() - defer cacheLock.Unlock() - - cacheKey := fmt.Sprintf("%s-%s", c.Location, c.VMSize) - cacheSku, found := cache.Get(cacheKey) - if found { - log.Debugw("Found SKU in cache", "key", cacheKey, "sku", cacheSku) - return cacheSku.(compute.ResourceSku), nil - } - - skuClient, err := getSKUClient(c) - if err != nil { - return compute.ResourceSku{}, fmt.Errorf("failed to (create) SKU client: %w", err) - } - - skuPages, err := skuClient.List(ctx, fmt.Sprintf("location eq '%s'", c.Location), "false") - if err != nil { - return compute.ResourceSku{}, fmt.Errorf("failed to list available SKUs: %w", err) - } - - var sku *compute.ResourceSku - - for skuPages.NotDone() && sku == nil { - skus := skuPages.Values() - for i, skuResult := range skus { - // skip invalid SKU results so we don't trigger a nil pointer exception - if skuResult.ResourceType == nil || skuResult.Name == nil { - continue - } - - if *skuResult.ResourceType == "virtualMachines" && *skuResult.Name == c.VMSize { - sku = &skus[i] - break - } - } - - // only fetch the next page if we haven't found our SKU yet - if sku == nil { - if err := skuPages.NextWithContext(ctx); err != nil { - return compute.ResourceSku{}, fmt.Errorf("failed to list available SKUs: %w", err) - } - } - } - - if sku == nil { - return compute.ResourceSku{}, fmt.Errorf("no VM SKU '%s' found for subscription '%s'", c.VMSize, c.SubscriptionID) - } - - cache.SetDefault(cacheKey, *sku) - - return *sku, nil -} - -func getVirtualNetwork(ctx context.Context, c *config) (network.VirtualNetwork, error) { - virtualNetworksClient, err := getVirtualNetworksClient(c) - if err != nil { - return network.VirtualNetwork{}, err - } - - return virtualNetworksClient.Get(ctx, c.VNetResourceGroup, c.VNetName, "") -} - -func createOrUpdateNetworkInterface(ctx context.Context, log *zap.SugaredLogger, ifName string, machineUID types.UID, config *config, publicIP, publicIPv6 *network.PublicIPAddress, ipFamily util.IPFamily, enableAcceleratedNetworking *bool) (*network.Interface, error) { - ifClient, err := getInterfacesClient(config) - if err != nil { - return nil, fmt.Errorf("failed to create interfaces client: %w", err) - } - - subnet, err := getSubnet(ctx, config) - if err != nil { - return nil, fmt.Errorf("failed to fetch subnet: %w", err) - } - - ifSpec := network.Interface{ - Name: to.StringPtr(ifName), - Location: &config.Location, - InterfacePropertiesFormat: &network.InterfacePropertiesFormat{ - IPConfigurations: &[]network.InterfaceIPConfiguration{}, - }, - Tags: map[string]*string{machineUIDTag: to.StringPtr(string(machineUID))}, - } - - *ifSpec.InterfacePropertiesFormat.IPConfigurations = append(*ifSpec.InterfacePropertiesFormat.IPConfigurations, network.InterfaceIPConfiguration{ - Name: to.StringPtr("ip-config-1"), - InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ - Subnet: &subnet, - PrivateIPAllocationMethod: network.IPAllocationMethodDynamic, - PublicIPAddress: publicIP, - Primary: to.BoolPtr(true), - }, - }) - - if ipFamily.IsDualstack() { - *ifSpec.InterfacePropertiesFormat.IPConfigurations = append(*ifSpec.InterfacePropertiesFormat.IPConfigurations, network.InterfaceIPConfiguration{ - Name: to.StringPtr("ip-config-2"), - InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ - PrivateIPAllocationMethod: network.IPAllocationMethodDynamic, - Subnet: &subnet, - PublicIPAddress: publicIPv6, - Primary: to.BoolPtr(false), - PrivateIPAddressVersion: network.IPVersionIPv6, - }, - }) - } - - ifSpec.InterfacePropertiesFormat.EnableAcceleratedNetworking = enableAcceleratedNetworking - - if config.SecurityGroupName != "" { - authorizer, err := auth.NewClientCredentialsConfig(config.ClientID, config.ClientSecret, config.TenantID).Authorizer() - if err != nil { - return nil, fmt.Errorf("failed to create authorizer for security groups: %w", err) - } - secGroupClient := network.NewSecurityGroupsClient(config.SubscriptionID) - secGroupClient.Authorizer = authorizer - secGroup, err := secGroupClient.Get(ctx, config.ResourceGroup, config.SecurityGroupName, "") - if err != nil { - return nil, fmt.Errorf("failed to get securityGroup %q: %w", config.SecurityGroupName, err) - } - ifSpec.NetworkSecurityGroup = &secGroup - } - log.Infow("Creating/Updating public network interface", "interface", ifName) - future, err := ifClient.CreateOrUpdate(ctx, config.ResourceGroup, ifName, ifSpec) - if err != nil { - return nil, fmt.Errorf("failed to create interface: %w", err) - } - - err = future.WaitForCompletionRef(ctx, ifClient.Client) - if err != nil { - return nil, fmt.Errorf("failed to get interface creation response: %w", err) - } - - _, err = future.Result(*ifClient) - if err != nil { - return nil, fmt.Errorf("failed to get interface creation result: %w", err) - } - - log.Infow("Fetching info about network interface", "interface", ifName) - iface, err := ifClient.Get(ctx, config.ResourceGroup, ifName, "") - if err != nil { - return nil, fmt.Errorf("failed to fetch info about interface %q: %w", ifName, err) - } - - return &iface, nil -} diff --git a/pkg/cloudprovider/provider/azure/get_client.go b/pkg/cloudprovider/provider/azure/get_client.go deleted file mode 100644 index a4ee34021..000000000 --- a/pkg/cloudprovider/provider/azure/get_client.go +++ /dev/null @@ -1,112 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package azure - -import ( - "fmt" - - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-05-01/network" - "github.com/Azure/go-autorest/autorest/azure/auth" -) - -func getIPClient(c *config) (*network.PublicIPAddressesClient, error) { - var err error - ipClient := network.NewPublicIPAddressesClient(c.SubscriptionID) - ipClient.Authorizer, err = auth.NewClientCredentialsConfig(c.ClientID, c.ClientSecret, c.TenantID).Authorizer() - if err != nil { - return nil, fmt.Errorf("failed to create authorizer: %w", err) - } - - return &ipClient, nil -} - -func getIPConfigClient(c *config) (*network.InterfaceIPConfigurationsClient, error) { - var err error - ipConfigClient := network.NewInterfaceIPConfigurationsClient(c.SubscriptionID) - ipConfigClient.Authorizer, err = auth.NewClientCredentialsConfig(c.ClientID, c.ClientSecret, c.TenantID).Authorizer() - if err != nil { - return nil, fmt.Errorf("failed to create authorizer: %w", err) - } - - return &ipConfigClient, nil -} - -func getSubnetsClient(c *config) (*network.SubnetsClient, error) { - var err error - subnetClient := network.NewSubnetsClient(c.SubscriptionID) - subnetClient.Authorizer, err = auth.NewClientCredentialsConfig(c.ClientID, c.ClientSecret, c.TenantID).Authorizer() - if err != nil { - return nil, fmt.Errorf("failed to create authorizer: %w", err) - } - - return &subnetClient, nil -} - -func getVirtualNetworksClient(c *config) (*network.VirtualNetworksClient, error) { - var err error - virtualNetworksClient := network.NewVirtualNetworksClient(c.SubscriptionID) - virtualNetworksClient.Authorizer, err = auth.NewClientCredentialsConfig(c.ClientID, c.ClientSecret, c.TenantID).Authorizer() - if err != nil { - return nil, fmt.Errorf("failed to create authorizer: %w", err) - } - return &virtualNetworksClient, nil -} - -func getVMClient(c *config) (*compute.VirtualMachinesClient, error) { - var err error - vmClient := compute.NewVirtualMachinesClient(c.SubscriptionID) - vmClient.Authorizer, err = auth.NewClientCredentialsConfig(c.ClientID, c.ClientSecret, c.TenantID).Authorizer() - if err != nil { - return nil, fmt.Errorf("failed to create authorizer: %w", err) - } - - return &vmClient, nil -} - -func getSKUClient(c *config) (*compute.ResourceSkusClient, error) { - var err error - skuClient := compute.NewResourceSkusClient(c.SubscriptionID) - skuClient.Authorizer, err = auth.NewClientCredentialsConfig(c.ClientID, c.ClientSecret, c.TenantID).Authorizer() - if err != nil { - return nil, fmt.Errorf("failed to create authorizer: %w", err) - } - - return &skuClient, nil -} - -func getInterfacesClient(c *config) (*network.InterfacesClient, error) { - var err error - ifClient := network.NewInterfacesClient(c.SubscriptionID) - ifClient.Authorizer, err = auth.NewClientCredentialsConfig(c.ClientID, c.ClientSecret, c.TenantID).Authorizer() - if err != nil { - return nil, fmt.Errorf("failed to create authorizer: %w", err) - } - - return &ifClient, err -} - -func getDisksClient(c *config) (*compute.DisksClient, error) { - var err error - disksClient := compute.NewDisksClient(c.SubscriptionID) - disksClient.Authorizer, err = auth.NewClientCredentialsConfig(c.ClientID, c.ClientSecret, c.TenantID).Authorizer() - if err != nil { - return nil, fmt.Errorf("failed to create authorizer: %w", err) - } - - return &disksClient, err -} diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go deleted file mode 100644 index cdb56bec3..000000000 --- a/pkg/cloudprovider/provider/azure/provider.go +++ /dev/null @@ -1,1336 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package azure - -import ( - "context" - "encoding/base64" - "errors" - "fmt" - "strings" - "sync" - "time" - "unicode" - "unicode/utf8" - - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-05-01/network" - "github.com/Azure/go-autorest/autorest/to" - gocache "github.com/patrickmn/go-cache" - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/common/ssh" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - azuretypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/azure/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - kuberneteshelper "github.com/kubermatic/machine-controller/pkg/kubernetes" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/ptr" -) - -const ( - CapabilityPremiumIO = "PremiumIO" - CapabilityUltraSSD = "UltraSSDAvailable" - CapabilityValueTrue = "True" - capabilityAcceleratedNetworking = "AcceleratedNetworkingEnabled" - - machineUIDTag = "Machine-UID" - - finalizerPublicIP = "kubermatic.io/cleanup-azure-public-ip" - finalizerPublicIPv6 = "kubermatic.io/cleanup-azure-public-ipv6" - finalizerNIC = "kubermatic.io/cleanup-azure-nic" - finalizerDisks = "kubermatic.io/cleanup-azure-disks" - finalizerVM = "kubermatic.io/cleanup-azure-vm" -) - -const ( - envClientID = "AZURE_CLIENT_ID" - envClientSecret = "AZURE_CLIENT_SECRET" - envTenantID = "AZURE_TENANT_ID" - envSubscriptionID = "AZURE_SUBSCRIPTION_ID" -) - -type provider struct { - configVarResolver *providerconfig.ConfigPointerVarResolver -} - -type config struct { - SubscriptionID string - TenantID string - ClientID string - ClientSecret string - - Location string - ResourceGroup string - VNetResourceGroup string - VMSize string - VNetName string - SubnetName string - LoadBalancerSku string - RouteTableName string - AvailabilitySet string - AssignAvailabilitySet *bool - SecurityGroupName string - ImageID string - Zones []string - ImagePlan *compute.Plan - ImageReference *compute.ImageReference - - OSDiskSize int32 - OSDiskSKU *compute.StorageAccountTypes - DataDiskSize int32 - DataDiskSKU *compute.StorageAccountTypes - - AssignPublicIP bool - PublicIPSKU *network.PublicIPAddressSkuName - EnableAcceleratedNetworking *bool - EnableBootDiagnostics bool - Tags map[string]string -} - -type azureVM struct { - vm *compute.VirtualMachine - ipAddresses map[string]v1.NodeAddressType - status instance.Status -} - -func (vm *azureVM) Addresses() map[string]v1.NodeAddressType { - return vm.ipAddresses -} - -func (vm *azureVM) ID() string { - return *vm.vm.ID -} - -func (vm *azureVM) HostID() string { - return "" -} - -func (vm *azureVM) Name() string { - return *vm.vm.Name -} - -func (vm *azureVM) ProviderID() string { - if vm.vm.ID == nil { - return "" - } - - return "azure://" + *vm.vm.ID -} - -func (vm *azureVM) Status() instance.Status { - return vm.status -} - -var imageReferences = map[providerconfigtypes.OperatingSystem]compute.ImageReference{ - providerconfigtypes.OperatingSystemCentOS: { - Publisher: to.StringPtr("OpenLogic"), - Offer: to.StringPtr("CentOS"), - Sku: to.StringPtr("7_9"), // https://docs.microsoft.com/en-us/azure/virtual-machines/linux/using-cloud-init - Version: to.StringPtr("latest"), - }, - providerconfigtypes.OperatingSystemUbuntu: { - Publisher: to.StringPtr("Canonical"), - Offer: to.StringPtr("0001-com-ubuntu-server-jammy"), - Sku: to.StringPtr("22_04-lts"), - Version: to.StringPtr("latest"), - }, - providerconfigtypes.OperatingSystemRHEL: { - Publisher: to.StringPtr("RedHat"), - Offer: to.StringPtr("rhel-byos"), - Sku: to.StringPtr("rhel-lvm85"), - Version: to.StringPtr("8.5.20220316"), - }, - providerconfigtypes.OperatingSystemFlatcar: { - Publisher: to.StringPtr("kinvolk"), - Offer: to.StringPtr("flatcar-container-linux"), - Sku: to.StringPtr("stable"), - Version: to.StringPtr("3374.2.0"), - }, - providerconfigtypes.OperatingSystemRockyLinux: { - Publisher: to.StringPtr("procomputers"), - Offer: to.StringPtr("rocky-linux-8-5"), - Sku: to.StringPtr("rocky-linux-8-5"), - Version: to.StringPtr("8.5.20211118"), - }, -} - -var osPlans = map[providerconfigtypes.OperatingSystem]*compute.Plan{ - providerconfigtypes.OperatingSystemFlatcar: { - Name: ptr.To("stable"), - Publisher: ptr.To("kinvolk"), - Product: ptr.To("flatcar-container-linux"), - }, - providerconfigtypes.OperatingSystemRHEL: { - Name: ptr.To("rhel-lvm85"), - Publisher: ptr.To("redhat"), - Product: ptr.To("rhel-byos"), - }, - providerconfigtypes.OperatingSystemRockyLinux: { - Name: ptr.To("rocky-linux-8-5"), - Publisher: ptr.To("procomputers"), - Product: ptr.To("rocky-linux-8-5"), - }, -} - -var osDiskSKUs = map[compute.StorageAccountTypes]string{ - compute.StorageAccountTypesStandardLRS: "", // Standard_LRS - compute.StorageAccountTypesStandardSSDLRS: "", // StandardSSD_LRS - compute.StorageAccountTypesPremiumLRS: "", // Premium_LRS -} - -var dataDiskSKUs = map[compute.StorageAccountTypes]string{ - compute.StorageAccountTypesStandardLRS: "", // Standard_LRS - compute.StorageAccountTypesStandardSSDLRS: "", // StandardSSD_LRS - compute.StorageAccountTypesPremiumLRS: "", // Premium_LRS - compute.StorageAccountTypesUltraSSDLRS: "", // UltraSSD_LRS -} - -var ( - // cacheLock protects concurrent cache misses against a single key. This usually happens when multiple machines get created simultaneously - // We lock so the first access updates/writes the data to the cache and afterwards everyone reads the cached data. - cacheLock = &sync.Mutex{} - cache = gocache.New(10*time.Minute, 10*time.Minute) -) - -func getOSImageReference(c *config, os providerconfigtypes.OperatingSystem) (*compute.ImageReference, error) { - if c.ImageID != "" { - return &compute.ImageReference{ - ID: to.StringPtr(c.ImageID), - }, nil - } - - if c.ImageReference != nil { - return &compute.ImageReference{ - Version: c.ImageReference.Version, - Sku: c.ImageReference.Sku, - Offer: c.ImageReference.Offer, - Publisher: c.ImageReference.Publisher, - }, nil - } - - ref, supported := imageReferences[os] - if !supported { - return nil, fmt.Errorf("operating system %q not supported", os) - } - - return &ref, nil -} - -// New returns a new azure provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return &provider{configVarResolver: &providerconfig.ConfigPointerVarResolver{Cvr: configVarResolver}} -} - -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - - pconfig, err := providerconfigtypes.GetConfig(provSpec) - if err != nil { - return nil, nil, err - } - - if pconfig.OperatingSystemSpec.Raw == nil { - return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") - } - - rawCfg, err := azuretypes.GetConfig(*pconfig) - if err != nil { - return nil, nil, err - } - - c := config{} - c.SubscriptionID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawCfg.SubscriptionID, envSubscriptionID) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"subscriptionID\" field, error = %w", err) - } - - c.TenantID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawCfg.TenantID, envTenantID) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"tenantID\" field, error = %w", err) - } - - c.ClientID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawCfg.ClientID, envClientID) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"clientID\" field, error = %w", err) - } - - c.ClientSecret, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawCfg.ClientSecret, envClientSecret) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"clientSecret\" field, error = %w", err) - } - - c.ResourceGroup, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.ResourceGroup) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"resourceGroup\" field, error = %w", err) - } - - c.VNetResourceGroup, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.VNetResourceGroup) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"vnetResourceGroup\" field, error = %w", err) - } - - if c.VNetResourceGroup == "" { - c.VNetResourceGroup = c.ResourceGroup - } - - c.Location, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.Location) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"location\" field, error = %w", err) - } - - c.VMSize, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.VMSize) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"vmSize\" field, error = %w", err) - } - - c.VNetName, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.VNetName) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"vnetName\" field, error = %w", err) - } - - c.SubnetName, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.SubnetName) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"subnetName\" field, error = %w", err) - } - - c.LoadBalancerSku, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.LoadBalancerSku) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"loadBalancerSku\" field, error = %w", err) - } - - c.RouteTableName, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.RouteTableName) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"routeTableName\" field, error = %w", err) - } - - c.AssignPublicIP, _, err = p.configVarResolver.GetConfigVarBoolValue(rawCfg.AssignPublicIP) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"assignPublicIP\" field, error = %w", err) - } - - if rawCfg.PublicIPSKU != nil { - c.PublicIPSKU = ipSkuPtr(*rawCfg.PublicIPSKU) - } - - c.AssignAvailabilitySet = rawCfg.AssignAvailabilitySet - c.EnableAcceleratedNetworking = rawCfg.EnableAcceleratedNetworking - - c.AvailabilitySet, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.AvailabilitySet) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"availabilitySet\" field, error = %w", err) - } - - c.SecurityGroupName, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.SecurityGroupName) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"securityGroupName\" field, error = %w", err) - } - - c.Zones = rawCfg.Zones - c.Tags = rawCfg.Tags - c.OSDiskSize = rawCfg.OSDiskSize - c.DataDiskSize = rawCfg.DataDiskSize - - if rawCfg.OSDiskSKU != nil { - c.OSDiskSKU = storageTypePtr(*rawCfg.OSDiskSKU) - } - - if rawCfg.DataDiskSKU != nil { - c.DataDiskSKU = storageTypePtr(*rawCfg.DataDiskSKU) - } - - if rawCfg.ImagePlan != nil && rawCfg.ImagePlan.Name != "" { - c.ImagePlan = &compute.Plan{ - Name: ptr.To(rawCfg.ImagePlan.Name), - Publisher: ptr.To(rawCfg.ImagePlan.Publisher), - Product: ptr.To(rawCfg.ImagePlan.Product), - } - } - - if rawCfg.ImageReference != nil { - c.ImageReference = &compute.ImageReference{ - Publisher: ptr.To(rawCfg.ImageReference.Publisher), - Offer: ptr.To(rawCfg.ImageReference.Offer), - Sku: ptr.To(rawCfg.ImageReference.Sku), - Version: ptr.To(rawCfg.ImageReference.Version), - } - } - - c.ImageID, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.ImageID) - if err != nil { - return nil, nil, fmt.Errorf("failed to get image id: %w", err) - } - - if rawCfg.EnableBootDiagnostics != nil { - c.EnableBootDiagnostics = *rawCfg.EnableBootDiagnostics - } - - return &c, pconfig, nil -} - -func getVMIPAddresses(ctx context.Context, log *zap.SugaredLogger, c *config, vm *compute.VirtualMachine, ipFamily util.IPFamily) (map[string]v1.NodeAddressType, error) { - var ( - ipAddresses = map[string]v1.NodeAddressType{} - err error - ) - - if vm.VirtualMachineProperties == nil { - return nil, fmt.Errorf("machine is missing properties") - } - - if vm.VirtualMachineProperties.NetworkProfile == nil { - return nil, fmt.Errorf("machine has no network profile") - } - - if vm.NetworkProfile.NetworkInterfaces == nil { - return nil, fmt.Errorf("machine has no network interfaces data") - } - - for n, iface := range *vm.NetworkProfile.NetworkInterfaces { - if iface.ID == nil || len(*iface.ID) == 0 { - return nil, fmt.Errorf("interface %d has no ID", n) - } - - splitIfaceID := strings.Split(*iface.ID, "/") - ifaceName := splitIfaceID[len(splitIfaceID)-1] - ipAddresses, err = getNICIPAddresses(ctx, log, c, ipFamily, ifaceName) - if err != nil || vm.NetworkProfile.NetworkInterfaces == nil { - return nil, fmt.Errorf("failed to get addresses for interface %q: %w", ifaceName, err) - } - } - - return ipAddresses, nil -} - -func getNICIPAddresses(ctx context.Context, log *zap.SugaredLogger, c *config, ipFamily util.IPFamily, ifaceName string) (map[string]v1.NodeAddressType, error) { - ifClient, err := getInterfacesClient(c) - if err != nil { - return nil, fmt.Errorf("failed to create interfaces client: %w", err) - } - - netIf, err := ifClient.Get(ctx, c.ResourceGroup, ifaceName, "") - if err != nil { - return nil, fmt.Errorf("failed to get interface %q: %w", ifaceName, err) - } - - ipAddresses := map[string]v1.NodeAddressType{} - - if netIf.IPConfigurations == nil { - return ipAddresses, nil - } - - for _, conf := range *netIf.IPConfigurations { - var name string - if conf.Name != nil { - name = *conf.Name - } else { - log.Infow("IP configuration of NIC was returned with no name, trying to dissect the ID.", "interface", ifaceName) - if conf.ID == nil || len(*conf.ID) == 0 { - return nil, fmt.Errorf("IP configuration of NIC %q was returned with no ID", ifaceName) - } - splitConfID := strings.Split(*conf.ID, "/") - name = splitConfID[len(splitConfID)-1] - } - - if c.AssignPublicIP { - publicIPs, err := getIPAddressStrings(ctx, c, publicIPName(ifaceName)) - if err != nil { - return nil, fmt.Errorf("failed to retrieve IP string for IP %q: %w", name, err) - } - for _, ip := range publicIPs { - ipAddresses[ip] = v1.NodeExternalIP - } - - if ipFamily.HasIPv6() { - publicIP6s, err := getIPAddressStrings(ctx, c, publicIPv6Name(ifaceName)) - if err != nil { - return nil, fmt.Errorf("failed to retrieve IP string for IP %q: %w", name, err) - } - for _, ip := range publicIP6s { - ipAddresses[ip] = v1.NodeExternalIP - } - } - } - - internalIPs, err := getInternalIPAddresses(ctx, c, ifaceName, name) - if err != nil { - return nil, fmt.Errorf("failed to retrieve internal IP string for IP %q: %w", name, err) - } - for _, ip := range internalIPs { - ipAddresses[ip] = v1.NodeInternalIP - } - } - return ipAddresses, nil -} - -func getIPAddressStrings(ctx context.Context, c *config, addrName string) ([]string, error) { - ipClient, err := getIPClient(c) - if err != nil { - return nil, fmt.Errorf("failed to create IP address client: %w", err) - } - - ip, err := ipClient.Get(ctx, c.ResourceGroup, addrName, "") - if err != nil { - return nil, fmt.Errorf("failed to get IP %q: %w", addrName, err) - } - - if ip.IPConfiguration == nil { - return nil, fmt.Errorf("IP %q has nil IPConfiguration", addrName) - } - - var ipAddresses []string - if ip.IPAddress != nil { - ipAddresses = append(ipAddresses, *ip.IPAddress) - } - - return ipAddresses, nil -} - -func getInternalIPAddresses(ctx context.Context, c *config, inetface, ipconfigName string) ([]string, error) { - var ipAddresses []string - ipConfigClient, err := getIPConfigClient(c) - if err != nil { - return nil, fmt.Errorf("failed to create IP config client: %w", err) - } - - internalIP, err := ipConfigClient.Get(ctx, c.ResourceGroup, inetface, ipconfigName) - if err != nil { - return nil, fmt.Errorf("failed to get IP config %q: %w", inetface, err) - } - - if internalIP.ID == nil { - return nil, fmt.Errorf("private IP %q has nil IPConfiguration", inetface) - } - if internalIP.PrivateIPAddress != nil { - ipAddresses = append(ipAddresses, *internalIP.PrivateIPAddress) - } - - return ipAddresses, nil -} - -func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { - return spec, nil -} - -func getStorageProfile(config *config, providerCfg *providerconfigtypes.Config) (*compute.StorageProfile, error) { - osRef, err := getOSImageReference(config, providerCfg.OperatingSystem) - if err != nil { - return nil, fmt.Errorf("failed to get OSImageReference: %w", err) - } - // initial default storage profile, this will use the VMSize default storage profile - sp := &compute.StorageProfile{ - ImageReference: osRef, - } - if config.OSDiskSize != 0 { - sp.OsDisk = &compute.OSDisk{ - DiskSizeGB: ptr.To(config.OSDiskSize), - CreateOption: compute.DiskCreateOptionTypesFromImage, - } - - if config.OSDiskSKU != nil { - sp.OsDisk.ManagedDisk = &compute.ManagedDiskParameters{ - StorageAccountType: *config.OSDiskSKU, - } - } - } - - if config.DataDiskSize != 0 { - sp.DataDisks = &[]compute.DataDisk{ - { - // this should be in range 0-63 and should be unique per datadisk, since we have only one datadisk, this should be fine - Lun: new(int32), - DiskSizeGB: ptr.To(config.DataDiskSize), - CreateOption: compute.DiskCreateOptionTypesEmpty, - }, - } - - if config.DataDiskSKU != nil { - (*sp.DataDisks)[0].ManagedDisk = &compute.ManagedDiskParameters{ - StorageAccountType: *config.DataDiskSKU, - } - } - } - return sp, nil -} - -func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { - config, providerCfg, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("failed to parse MachineSpec, due to %v", err), - } - } - - vmClient, err := getVMClient(config) - if err != nil { - return nil, fmt.Errorf("failed to create VM client: %w", err) - } - - // We genete a random SSH key, since Azure won't let us create a VM without an SSH key or a password - key, err := ssh.NewKey() - if err != nil { - return nil, fmt.Errorf("failed to generate ssh key: %w", err) - } - - ipFamily := providerCfg.Network.GetIPFamily() - sku := network.PublicIPAddressSkuNameBasic - - if config.PublicIPSKU != nil { - sku = *config.PublicIPSKU - } else if ipFamily.IsDualstack() { - // 1. Cannot specify basic sku PublicIp for an IPv6 network interface ipConfiguration. - // 2. Different basic sku and standard sku public Ip resources in availability set is not allowed. - // 1 & 2 means we have to use standard sku in dual-stack configuration. - - // It is not clear from the documentation, but you get the - // errors if you try mixing skus or try to create IPv6 public IP with - // basic sku. - sku = network.PublicIPAddressSkuNameStandard - } - - var publicIP, publicIPv6 *network.PublicIPAddress - if config.AssignPublicIP { - if err = data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { - if !kuberneteshelper.HasFinalizer(updatedMachine, finalizerPublicIP) { - updatedMachine.Finalizers = append(updatedMachine.Finalizers, finalizerPublicIP) - } - }); err != nil { - return nil, err - } - publicIP, err = createOrUpdatePublicIPAddress(ctx, log, publicIPName(ifaceName(machine)), network.IPVersionIPv4, sku, network.IPAllocationMethodStatic, machine.UID, config) - if err != nil { - return nil, fmt.Errorf("failed to create public IP: %w", err) - } - - if ipFamily.IsDualstack() { - publicIPv6, err = createOrUpdatePublicIPAddress(ctx, log, publicIPv6Name(ifaceName(machine)), network.IPVersionIPv6, sku, network.IPAllocationMethodStatic, machine.UID, config) - if err != nil { - return nil, fmt.Errorf("failed to create public IP: %w", err) - } - } - } - - if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { - if !kuberneteshelper.HasFinalizer(updatedMachine, finalizerNIC) { - updatedMachine.Finalizers = append(updatedMachine.Finalizers, finalizerNIC) - } - }); err != nil { - return nil, err - } - - iface, err := createOrUpdateNetworkInterface(ctx, log, ifaceName(machine), machine.UID, config, publicIP, publicIPv6, ipFamily, config.EnableAcceleratedNetworking) - if err != nil { - return nil, fmt.Errorf("failed to generate main network interface: %w", err) - } - - tags := make(map[string]*string, len(config.Tags)+1) - for k, v := range config.Tags { - tags[k] = to.StringPtr(v) - } - tags[machineUIDTag] = to.StringPtr(string(machine.UID)) - - osPlane := osPlans[providerCfg.OperatingSystem] - if config.ImagePlan != nil { - osPlane = config.ImagePlan - } - - adminUserName := getOSUsername(providerCfg.OperatingSystem) - storageProfile, err := getStorageProfile(config, providerCfg) - if err != nil { - return nil, fmt.Errorf("failed to get StorageProfile: %w", err) - } - - vmSpec := compute.VirtualMachine{ - Location: &config.Location, - Plan: osPlane, - VirtualMachineProperties: &compute.VirtualMachineProperties{ - HardwareProfile: &compute.HardwareProfile{VMSize: compute.VirtualMachineSizeTypes(config.VMSize)}, - NetworkProfile: &compute.NetworkProfile{ - NetworkInterfaces: &[]compute.NetworkInterfaceReference{ - { - ID: iface.ID, - NetworkInterfaceReferenceProperties: &compute.NetworkInterfaceReferenceProperties{Primary: to.BoolPtr(true)}, - }, - }, - }, - OsProfile: &compute.OSProfile{ - AdminUsername: to.StringPtr(adminUserName), - ComputerName: &machine.Name, - LinuxConfiguration: &compute.LinuxConfiguration{ - DisablePasswordAuthentication: to.BoolPtr(true), - SSH: &compute.SSHConfiguration{ - PublicKeys: &[]compute.SSHPublicKey{ - { - Path: to.StringPtr(fmt.Sprintf("/home/%s/.ssh/authorized_keys", adminUserName)), - KeyData: &key.PublicKey, - }, - }, - }, - }, - CustomData: to.StringPtr(base64.StdEncoding.EncodeToString([]byte(userdata))), - }, - StorageProfile: storageProfile, - }, - Tags: tags, - Zones: &config.Zones, - } - - if config.AssignAvailabilitySet == nil && config.AvailabilitySet != "" || - config.AssignAvailabilitySet != nil && *config.AssignAvailabilitySet && config.AvailabilitySet != "" { - // Azure expects the full path to the resource - asURI := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/availabilitySets/%s", config.SubscriptionID, config.ResourceGroup, config.AvailabilitySet) - vmSpec.VirtualMachineProperties.AvailabilitySet = &compute.SubResource{ID: to.StringPtr(asURI)} - } - - if config.EnableBootDiagnostics { - vmSpec.DiagnosticsProfile = &compute.DiagnosticsProfile{ - BootDiagnostics: &compute.BootDiagnostics{ - Enabled: ptr.To(config.EnableBootDiagnostics), - }, - } - } - - log.Info("Creating machine") - if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { - if !kuberneteshelper.HasFinalizer(updatedMachine, finalizerDisks) { - updatedMachine.Finalizers = append(updatedMachine.Finalizers, finalizerDisks) - } - if !kuberneteshelper.HasFinalizer(machine, finalizerVM) { - updatedMachine.Finalizers = append(updatedMachine.Finalizers, finalizerVM) - } - }); err != nil { - return nil, err - } - - future, err := vmClient.CreateOrUpdate(ctx, config.ResourceGroup, machine.Name, vmSpec) - if err != nil { - return nil, fmt.Errorf("trying to create a VM: %w", err) - } - - err = future.WaitForCompletionRef(ctx, vmClient.Client) - if err != nil { - return nil, fmt.Errorf("waiting for operation returned: %w", err) - } - - vm, err := future.Result(*vmClient) - if err != nil { - return nil, fmt.Errorf("decoding result: %w", err) - } - - // get the actual VM object filled in with additional data - vm, err = vmClient.Get(ctx, config.ResourceGroup, machine.Name, "") - if err != nil { - return nil, fmt.Errorf("failed to retrieve updated data for VM %q: %w", machine.Name, err) - } - - ipAddresses, err := getVMIPAddresses(ctx, log, config, &vm, ipFamily) - if err != nil { - return nil, fmt.Errorf("failed to retrieve IP addresses for VM %q: %w", machine.Name, err) - } - - status, err := getVMStatus(ctx, log, config, machine.Name) - if err != nil { - return nil, fmt.Errorf("failed to retrieve status for VM %q: %w", machine.Name, err) - } - - return &azureVM{vm: &vm, ipAddresses: ipAddresses, status: status}, nil -} - -func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { - config, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return false, fmt.Errorf("failed to parse MachineSpec: %w", err) - } - - if kuberneteshelper.HasFinalizer(machine, finalizerVM) { - log.Info("Deleting VM") - if err = deleteVMsByMachineUID(ctx, config, machine.UID); err != nil { - return false, fmt.Errorf("failed to delete instance for machine %q: %w", machine.Name, err) - } - - if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { - updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerVM) - }); err != nil { - return false, err - } - } - - if kuberneteshelper.HasFinalizer(machine, finalizerDisks) { - log.Info("Deleting disks") - if err := deleteDisksByMachineUID(ctx, config, machine.UID); err != nil { - return false, fmt.Errorf("failed to remove disks of machine %q: %w", machine.Name, err) - } - if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { - updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerDisks) - }); err != nil { - return false, err - } - } - - if kuberneteshelper.HasFinalizer(machine, finalizerNIC) { - log.Info("Deleting network interfaces") - if err := deleteInterfacesByMachineUID(ctx, config, machine.UID); err != nil { - return false, fmt.Errorf("failed to remove network interfaces of machine %q: %w", machine.Name, err) - } - if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { - updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerNIC) - }); err != nil { - return false, err - } - } - - if kuberneteshelper.HasFinalizer(machine, finalizerPublicIP) { - log.Infof("Deleting public IP addresses") - if err := deleteIPAddressesByMachineUID(ctx, config, machine.UID); err != nil { - return false, fmt.Errorf("failed to remove public IP addresses of machine %q: %w", machine.Name, err) - } - if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { - updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerPublicIP) - }); err != nil { - return false, err - } - } - - return true, nil -} - -func getVMByUID(ctx context.Context, c *config, uid types.UID) (*compute.VirtualMachine, error) { - vmClient, err := getVMClient(c) - if err != nil { - return nil, err - } - - list, err := vmClient.List(ctx, c.ResourceGroup, "") - if err != nil { - return nil, err - } - - var allServers []compute.VirtualMachine - - for list.NotDone() { - allServers = append(allServers, list.Values()...) - if err := list.Next(); err != nil { - return nil, fmt.Errorf("failed to iterate the result list: %w", err) - } - } - - for _, vm := range allServers { - if vm.Tags != nil && vm.Tags[machineUIDTag] != nil && *vm.Tags[machineUIDTag] == string(uid) { - return &vm, nil - } - } - - return nil, cloudprovidererrors.ErrInstanceNotFound -} - -func getVMStatus(ctx context.Context, log *zap.SugaredLogger, c *config, vmName string) (instance.Status, error) { - vmClient, err := getVMClient(c) - if err != nil { - return instance.StatusUnknown, err - } - - iv, err := vmClient.InstanceView(ctx, c.ResourceGroup, vmName) - if err != nil { - return instance.StatusUnknown, fmt.Errorf("failed to get instance view for machine %q: %w", vmName, err) - } - - if iv.Statuses == nil { - return instance.StatusUnknown, nil - } - - // it seems that this field should contain two entries: a provisioning status and a power status - if len(*iv.Statuses) < 2 { - provisioningStatus := (*iv.Statuses)[0] - if provisioningStatus.Code == nil { - log.Info("Azure provisioning status has missing code") - return instance.StatusUnknown, nil - } - - switch *provisioningStatus.Code { - case "": - return instance.StatusUnknown, nil - case "ProvisioningState/deleting": - return instance.StatusDeleting, nil - default: - log.Errorw("Unknown Azure provisioning status", "code", *provisioningStatus.Code, "level", provisioningStatus.Level) - return instance.StatusUnknown, nil - } - } - - // the second field is supposed to be the power status - // https://docs.microsoft.com/en-us/azure/virtual-machines/windows/tutorial-manage-vm#vm-power-states - powerStatus := (*iv.Statuses)[1] - if powerStatus.Code == nil { - log.Info("Azure power status has missing code") - return instance.StatusUnknown, nil - } - - switch *powerStatus.Code { - case "": - return instance.StatusUnknown, nil - case "PowerState/running": - return instance.StatusRunning, nil - case "PowerState/starting": - return instance.StatusCreating, nil - default: - log.Errorw("Unknown Azure power status", "code", *powerStatus.Code, "level", powerStatus.Level) - return instance.StatusUnknown, nil - } -} - -func (p *provider) Get(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - return p.get(ctx, log, machine) -} - -func (p *provider) get(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine) (*azureVM, error) { - config, providerCfg, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, fmt.Errorf("failed to parse MachineSpec: %w", err) - } - - vm, err := getVMByUID(ctx, config, machine.UID) - if err != nil { - if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { - return nil, cloudprovidererrors.ErrInstanceNotFound - } - - return nil, fmt.Errorf("failed to find machine %q by its UID: %w", machine.UID, err) - } - - ipFamily := providerCfg.Network.GetIPFamily() - ipAddresses, err := getVMIPAddresses(ctx, log, config, vm, ipFamily) - if err != nil { - return nil, fmt.Errorf("failed to retrieve IP addresses for VM %v: %w", vm.Name, err) - } - - status, err := getVMStatus(ctx, log, config, machine.Name) - if err != nil { - return nil, fmt.Errorf("failed to retrieve status for VM %v: %w", vm.Name, err) - } - - return &azureVM{vm: vm, ipAddresses: ipAddresses, status: status}, nil -} - -func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { - c, _, err := p.getConfig(spec.ProviderSpec) - if err != nil { - return "", "", fmt.Errorf("failed to parse config: %w", err) - } - - var avSet string - if c.AssignAvailabilitySet == nil && c.AvailabilitySet != "" || - c.AssignAvailabilitySet != nil && *c.AssignAvailabilitySet && c.AvailabilitySet != "" { - avSet = c.AvailabilitySet - } - - cc := &azuretypes.CloudConfig{ - Cloud: "AZUREPUBLICCLOUD", - TenantID: c.TenantID, - SubscriptionID: c.SubscriptionID, - AADClientID: c.ClientID, - AADClientSecret: c.ClientSecret, - ResourceGroup: c.ResourceGroup, - VnetResourceGroup: c.VNetResourceGroup, - Location: c.Location, - VNetName: c.VNetName, - SubnetName: c.SubnetName, - LoadBalancerSku: c.LoadBalancerSku, - RouteTableName: c.RouteTableName, - PrimaryAvailabilitySetName: avSet, - SecurityGroupName: c.SecurityGroupName, - UseInstanceMetadata: true, - } - - s, err := azuretypes.CloudConfigToString(cc) - if err != nil { - return "", "", fmt.Errorf("failed to convert cloud-config to string: %w", err) - } - - return s, "azure", nil -} - -func validateDiskSKUs(_ context.Context, c *config, sku compute.ResourceSku) error { - if c.OSDiskSKU != nil || c.DataDiskSKU != nil { - if c.OSDiskSKU != nil { - if _, ok := osDiskSKUs[*c.OSDiskSKU]; !ok { - return fmt.Errorf("invalid OS disk SKU '%s'", *c.OSDiskSKU) - } - - if err := supportsDiskSKU(sku, *c.OSDiskSKU, c.Zones); err != nil { - return err - } - } - - if c.DataDiskSKU != nil { - if _, ok := dataDiskSKUs[*c.DataDiskSKU]; !ok { - return fmt.Errorf("invalid data disk SKU '%s'", *c.DataDiskSKU) - } - - // Ultra SSDs do not support availability sets, see for reference: - // https://docs.microsoft.com/en-us/azure/virtual-machines/disks-enable-ultra-ssd#ga-scope-and-limitations - if *c.DataDiskSKU == compute.StorageAccountTypesUltraSSDLRS && ((c.AssignAvailabilitySet != nil && *c.AssignAvailabilitySet) || c.AvailabilitySet != "") { - return fmt.Errorf("data disk SKU '%s' does not support availability sets", *c.DataDiskSKU) - } - - if err := supportsDiskSKU(sku, *c.DataDiskSKU, c.Zones); err != nil { - return err - } - } - } - - return nil -} - -func validateSKUCapabilities(_ context.Context, c *config, sku compute.ResourceSku) error { - if c.EnableAcceleratedNetworking != nil && *c.EnableAcceleratedNetworking { - if !SKUHasCapability(sku, capabilityAcceleratedNetworking) { - return fmt.Errorf("VM size %q does not support accelerated networking", c.VMSize) - } - } - return nil -} - -func (p *provider) Validate(ctx context.Context, log *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { - c, providerConfig, err := p.getConfig(spec.ProviderSpec) - if err != nil { - return fmt.Errorf("failed to parse config: %w", err) - } - - if c.SubscriptionID == "" { - return errors.New("subscriptionID is missing") - } - - if c.TenantID == "" { - return errors.New("tenantID is missing") - } - - if c.ClientID == "" { - return errors.New("clientID is missing") - } - - if c.ClientSecret == "" { - return errors.New("clientSecret is missing") - } - - if c.ResourceGroup == "" { - return errors.New("resourceGroup is missing") - } - - if c.VMSize == "" { - return errors.New("vmSize is missing") - } - - if c.VNetName == "" { - return errors.New("vnetName is missing") - } - - if c.SubnetName == "" { - return errors.New("subnetName is missing") - } - - switch f := providerConfig.Network.GetIPFamily(); f { - case util.IPFamilyUnspecified, util.IPFamilyIPv4: - //noop - case util.IPFamilyIPv6: - return fmt.Errorf(util.ErrIPv6OnlyUnsupported) - case util.IPFamilyIPv4IPv6, util.IPFamilyIPv6IPv4: - // validate - default: - return fmt.Errorf(util.ErrUnknownNetworkFamily, f) - } - - if c.PublicIPSKU != nil { - valid := false - for _, sku := range network.PossiblePublicIPAddressSkuNameValues() { - if sku == *c.PublicIPSKU { - valid = true - } - } - - if !valid { - return fmt.Errorf("unknown public IP address SKU: %s", *c.PublicIPSKU) - } - - if providerConfig.Network.GetIPFamily().IsDualstack() && *c.PublicIPSKU == network.PublicIPAddressSkuNameBasic { - return fmt.Errorf("cannot use %s public IP address SKU with dualstack", network.PublicIPAddressSkuNameBasic) - } - } - - vmClient, err := getVMClient(c) - if err != nil { - return fmt.Errorf("failed to (create) vm client: %w", err) - } - - _, err = vmClient.List(ctx, c.ResourceGroup, "") - if err != nil { - return fmt.Errorf("failed to list virtual machines: %w", err) - } - - if _, err := getVirtualNetwork(ctx, c); err != nil { - return fmt.Errorf("failed to get virtual network: %w", err) - } - - if _, err := getSubnet(ctx, c); err != nil { - return fmt.Errorf("failed to get subnet: %w", err) - } - - sku, err := getSKU(ctx, log, c) - if err != nil { - return fmt.Errorf("failed to get VM SKU: %w", err) - } - - if err := validateDiskSKUs(ctx, c, sku); err != nil { - return fmt.Errorf("failed to validate disk SKUs: %w", err) - } - - if err := validateSKUCapabilities(ctx, c, sku); err != nil { - return fmt.Errorf("failed to validate SKU capabilities: %w", err) - } - - _, err = getOSImageReference(c, providerConfig.OperatingSystem) - return err -} - -func ifaceName(machine *clusterv1alpha1.Machine) string { - return machine.Name + "-netiface" -} - -func publicIPName(ifaceName string) string { - return ifaceName + "-pubip" -} - -func publicIPv6Name(ifaceName string) string { - return ifaceName + "-pubipv6" -} - -func (p *provider) MigrateUID(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { - config, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("failed to parse MachineSpec, due to %v", err), - } - } - - vmClient, err := getVMClient(config) - if err != nil { - return fmt.Errorf("failed to create VM client: %w", err) - } - - var publicIP, publicIPv6 *network.PublicIPAddress - sku := network.PublicIPAddressSkuNameBasic - - if kuberneteshelper.HasFinalizer(machine, finalizerPublicIPv6) { - sku = network.PublicIPAddressSkuNameStandard - _, err = createOrUpdatePublicIPAddress(ctx, log, publicIPv6Name(ifaceName(machine)), network.IPVersionIPv6, sku, network.IPAllocationMethodDynamic, newUID, config) - if err != nil { - return fmt.Errorf("failed to update UID on public IP: %w", err) - } - } - - if kuberneteshelper.HasFinalizer(machine, finalizerPublicIP) { - _, err = createOrUpdatePublicIPAddress(ctx, log, publicIPName(ifaceName(machine)), network.IPVersionIPv4, sku, network.IPAllocationMethodStatic, newUID, config) - if err != nil { - return fmt.Errorf("failed to update UID on public IP: %w", err) - } - } - - if kuberneteshelper.HasFinalizer(machine, finalizerNIC) { - _, err = createOrUpdateNetworkInterface(ctx, log, ifaceName(machine), newUID, config, publicIP, publicIPv6, util.IPFamilyUnspecified, config.EnableAcceleratedNetworking) - if err != nil { - return fmt.Errorf("failed to update UID on main network interface: %w", err) - } - } - - if kuberneteshelper.HasFinalizer(machine, finalizerDisks) { - disksClient, err := getDisksClient(config) - if err != nil { - return fmt.Errorf("failed to get disks client: %w", err) - } - - disks, err := getDisksByMachineUID(ctx, disksClient, machine.UID) - if err != nil { - return fmt.Errorf("failed to get disks: %w", err) - } - - for _, disk := range disks { - disk.Tags[machineUIDTag] = to.StringPtr(string(newUID)) - future, err := disksClient.CreateOrUpdate(ctx, config.ResourceGroup, *disk.Name, disk) - if err != nil { - return fmt.Errorf("failed to update UID for disk %s: %w", *disk.Name, err) - } - if err := future.WaitForCompletionRef(ctx, disksClient.Client); err != nil { - return fmt.Errorf("failed waiting for completion of update UID operation for disk %s: %w", *disk.Name, err) - } - } - } - - tags := map[string]*string{} - for k, v := range config.Tags { - tags[k] = to.StringPtr(v) - } - tags[machineUIDTag] = to.StringPtr(string(newUID)) - - vmSpec := compute.VirtualMachine{Location: &config.Location, Tags: tags} - future, err := vmClient.CreateOrUpdate(ctx, config.ResourceGroup, machine.Name, vmSpec) - if err != nil { - return fmt.Errorf("failed to update UID of the instance: %w", err) - } - - if err := future.WaitForCompletionRef(ctx, vmClient.Client); err != nil { - return fmt.Errorf("error waiting for instance to have the updated UID: %w", err) - } - - return nil -} - -func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { - labels := make(map[string]string) - - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err == nil { - labels["size"] = c.VMSize - labels["location"] = c.Location - } - - return labels, err -} - -func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { - return nil -} - -func getOSUsername(os providerconfigtypes.OperatingSystem) string { - switch os { - case providerconfigtypes.OperatingSystemFlatcar: - return "core" - default: - return string(os) - } -} - -func storageTypePtr(storageType string) *compute.StorageAccountTypes { - storage := compute.StorageAccountTypes(storageType) - return &storage -} - -func ipSkuPtr(ipSKU string) *network.PublicIPAddressSkuName { - // the correct Azure API representation is capitalized, so we do that even if the original input was all lowercase - sku := network.PublicIPAddressSkuName(upperFirst(ipSKU)) - return &sku -} - -func upperFirst(str string) string { - if str == "" { - return "" - } - - r, n := utf8.DecodeRuneInString(str) - return string(unicode.ToUpper(r)) + str[n:] -} - -// supportsDiskSKU validates some disk SKU types against the chosen VM SKU / VM type. -func supportsDiskSKU(vmSKU compute.ResourceSku, diskSKU compute.StorageAccountTypes, zones []string) error { - // sanity check to make sure the Azure API did not return something bad - if vmSKU.Name == nil || vmSKU.Capabilities == nil { - return fmt.Errorf("invalid VM SKU object") - } - - switch diskSKU { - case compute.StorageAccountTypesPremiumLRS: - found := false - for _, capability := range *vmSKU.Capabilities { - if *capability.Name == CapabilityPremiumIO && *capability.Value == CapabilityValueTrue { - found = true - break - } - } - - if !found { - return fmt.Errorf("VM SKU '%s' does not support disk SKU '%s'", *vmSKU.Name, diskSKU) - } - - case compute.StorageAccountTypesUltraSSDLRS: - if vmSKU.LocationInfo == nil || len(*vmSKU.LocationInfo) == 0 || (*vmSKU.LocationInfo)[0].Zones == nil || len(*(*vmSKU.LocationInfo)[0].Zones) == 0 { - // no zone information found, let's check for capability - found := false - for _, capability := range *vmSKU.Capabilities { - if *capability.Name == CapabilityUltraSSD && *capability.Value == CapabilityValueTrue { - found = true - break - } - } - - if !found { - return fmt.Errorf("VM SKU '%s' does not support disk SKU '%s'", *vmSKU.Name, diskSKU) - } - } else { - if (*vmSKU.LocationInfo)[0].ZoneDetails != nil { - for _, zone := range zones { - found := false - for _, details := range *(*vmSKU.LocationInfo)[0].ZoneDetails { - matchesZone := false - for _, zoneName := range *details.Name { - if zone == zoneName { - matchesZone = true - break - } - } - - // we only check this zone details for capabilities if it actually includes the zone we're checking for - if matchesZone { - for _, capability := range *details.Capabilities { - if *capability.Name == CapabilityUltraSSD && *capability.Value == CapabilityValueTrue { - found = true - break - } - } - } - } - - if !found { - return fmt.Errorf("VM SKU '%s' does not support disk SKU '%s' in zone '%s'", *vmSKU.Name, diskSKU, zone) - } - } - } - } - } - - return nil -} - -func SKUHasCapability(sku compute.ResourceSku, name string) bool { - if sku.Capabilities != nil { - for _, capability := range *sku.Capabilities { - if capability.Name != nil && *capability.Name == name && *capability.Value == CapabilityValueTrue { - return true - } - } - } - return false -} diff --git a/pkg/cloudprovider/provider/azure/types/cloudconfig.go b/pkg/cloudprovider/provider/azure/types/cloudconfig.go deleted file mode 100644 index 6ddb8b5ca..000000000 --- a/pkg/cloudprovider/provider/azure/types/cloudconfig.go +++ /dev/null @@ -1,50 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "encoding/json" - "fmt" -) - -type CloudConfig struct { - Cloud string `json:"cloud"` - TenantID string `json:"tenantId"` - SubscriptionID string `json:"subscriptionId"` - AADClientID string `json:"aadClientId"` - AADClientSecret string `json:"aadClientSecret"` - - ResourceGroup string `json:"resourceGroup"` - Location string `json:"location"` - VNetName string `json:"vnetName"` - SubnetName string `json:"subnetName"` - RouteTableName string `json:"routeTableName"` - SecurityGroupName string `json:"securityGroupName" yaml:"securityGroupName"` - PrimaryAvailabilitySetName string `json:"primaryAvailabilitySetName"` - VnetResourceGroup string `json:"vnetResourceGroup"` - UseInstanceMetadata bool `json:"useInstanceMetadata"` - LoadBalancerSku string `json:"loadBalancerSku"` -} - -func CloudConfigToString(c *CloudConfig) (string, error) { - b, err := json.Marshal(c) - if err != nil { - return "", fmt.Errorf("failed to unmarshal config: %w", err) - } - - return string(b), nil -} diff --git a/pkg/cloudprovider/provider/azure/types/types.go b/pkg/cloudprovider/provider/azure/types/types.go deleted file mode 100644 index fc1c69e25..000000000 --- a/pkg/cloudprovider/provider/azure/types/types.go +++ /dev/null @@ -1,77 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" -) - -// RawConfig is a direct representation of an Azure machine object's configuration. -type RawConfig struct { - SubscriptionID *providerconfigtypes.ConfigVarString `json:"subscriptionID,omitempty"` - TenantID *providerconfigtypes.ConfigVarString `json:"tenantID,omitempty"` - ClientID *providerconfigtypes.ConfigVarString `json:"clientID,omitempty"` - ClientSecret *providerconfigtypes.ConfigVarString `json:"clientSecret,omitempty"` - - Location *providerconfigtypes.ConfigVarString `json:"location"` - ResourceGroup *providerconfigtypes.ConfigVarString `json:"resourceGroup"` - VNetResourceGroup *providerconfigtypes.ConfigVarString `json:"vnetResourceGroup"` - VMSize *providerconfigtypes.ConfigVarString `json:"vmSize"` - VNetName *providerconfigtypes.ConfigVarString `json:"vnetName"` - SubnetName *providerconfigtypes.ConfigVarString `json:"subnetName"` - LoadBalancerSku *providerconfigtypes.ConfigVarString `json:"loadBalancerSku"` - RouteTableName *providerconfigtypes.ConfigVarString `json:"routeTableName"` - AvailabilitySet *providerconfigtypes.ConfigVarString `json:"availabilitySet"` - AssignAvailabilitySet *bool `json:"assignAvailabilitySet"` - SecurityGroupName *providerconfigtypes.ConfigVarString `json:"securityGroupName"` - Zones []string `json:"zones"` - ImagePlan *ImagePlan `json:"imagePlan,omitempty"` - ImageReference *ImageReference `json:"imageReference,omitempty"` - EnableAcceleratedNetworking *bool `json:"enableAcceleratedNetworking"` - EnableBootDiagnostics *bool `json:"enableBootDiagnostics,omitempty"` - - ImageID *providerconfigtypes.ConfigVarString `json:"imageID"` - OSDiskSize int32 `json:"osDiskSize"` - OSDiskSKU *string `json:"osDiskSKU,omitempty"` - DataDiskSize int32 `json:"dataDiskSize"` - DataDiskSKU *string `json:"dataDiskSKU,omitempty"` - AssignPublicIP *providerconfigtypes.ConfigVarBool `json:"assignPublicIP"` - PublicIPSKU *string `json:"publicIPSKU,omitempty"` - Tags map[string]string `json:"tags,omitempty"` -} - -// ImagePlan contains azure OS Plan fields for the marketplace images. -type ImagePlan struct { - Name string `json:"name,omitempty"` - Publisher string `json:"publisher,omitempty"` - Product string `json:"product,omitempty"` -} - -// ImageReference specifies information about the image to use. -type ImageReference struct { - Publisher string `json:"publisher,omitempty"` - Offer string `json:"offer,omitempty"` - Sku string `json:"sku,omitempty"` - Version string `json:"version,omitempty"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go deleted file mode 100644 index e7552dfb0..000000000 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ /dev/null @@ -1,570 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package digitalocean - -import ( - "context" - "errors" - "fmt" - "net/http" - "strconv" - "time" - - "github.com/digitalocean/godo" - "go.uber.org/zap" - "golang.org/x/oauth2" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/common/ssh" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - digitaloceantypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/digitalocean/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/wait" -) - -type provider struct { - configVarResolver *providerconfig.ConfigPointerVarResolver -} - -// New returns a digitalocean provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return &provider{configVarResolver: &providerconfig.ConfigPointerVarResolver{Cvr: configVarResolver}} -} - -type Config struct { - Token string - Region string - Size string - Backups bool - IPv6 bool - PrivateNetworking bool - Monitoring bool - Tags []string -} - -const ( - createCheckPeriod = 10 * time.Second - createCheckTimeout = 5 * time.Minute - createCheckFailedWaitPeriod = 10 * time.Second -) - -type TokenSource struct { - AccessToken string -} - -func (t *TokenSource) Token() (*oauth2.Token, error) { - token := &oauth2.Token{ - AccessToken: t.AccessToken, - } - return token, nil -} - -func getSlugForOS(os providerconfigtypes.OperatingSystem) (string, error) { - switch os { - case providerconfigtypes.OperatingSystemUbuntu: - return "ubuntu-22-04-x64", nil - case providerconfigtypes.OperatingSystemCentOS: - return "centos-7-x64", nil - case providerconfigtypes.OperatingSystemRockyLinux: - return "rockylinux-8-x64", nil - } - return "", providerconfigtypes.ErrOSNotSupported -} - -func getClient(ctx context.Context, token string) *godo.Client { - tokenSource := &TokenSource{ - AccessToken: token, - } - - oauthClient := oauth2.NewClient(ctx, tokenSource) - return godo.NewClient(oauthClient) -} - -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - - pconfig, err := providerconfigtypes.GetConfig(provSpec) - if err != nil { - return nil, nil, err - } - - if pconfig.OperatingSystemSpec.Raw == nil { - return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") - } - - rawConfig, err := digitaloceantypes.GetConfig(*pconfig) - if err != nil { - return nil, nil, err - } - - c := Config{} - c.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Token, "DO_TOKEN") - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"token\" field, error = %w", err) - } - c.Region, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Region) - if err != nil { - return nil, nil, err - } - c.Size, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Size) - if err != nil { - return nil, nil, err - } - c.Backups, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.Backups) - if err != nil { - return nil, nil, err - } - c.IPv6, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.IPv6) - if err != nil { - return nil, nil, err - } - c.PrivateNetworking, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.PrivateNetworking) - if err != nil { - return nil, nil, err - } - c.Monitoring, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.Monitoring) - if err != nil { - return nil, nil, err - } - for _, tag := range rawConfig.Tags { - tagVal, err := p.configVarResolver.GetConfigVarStringValue(&tag) - if err != nil { - return nil, nil, err - } - c.Tags = append(c.Tags, tagVal) - } - - return &c, pconfig, err -} - -func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { - return spec, nil -} - -func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { - c, pc, err := p.getConfig(spec.ProviderSpec) - if err != nil { - return fmt.Errorf("failed to parse config: %w", err) - } - - if c.Token == "" { - return errors.New("token is missing") - } - - if c.Region == "" { - return errors.New("region is missing") - } - - if c.Size == "" { - return errors.New("size is missing") - } - - _, err = getSlugForOS(pc.OperatingSystem) - if err != nil { - return fmt.Errorf("invalid operating system specified %q: %w", pc.OperatingSystem, err) - } - - switch f := pc.Network.GetIPFamily(); f { - case util.IPFamilyUnspecified, util.IPFamilyIPv4: - // noop - case util.IPFamilyIPv6: - return fmt.Errorf(util.ErrIPv6OnlyUnsupported) - case util.IPFamilyIPv4IPv6, util.IPFamilyIPv6IPv4: - // noop - default: - return fmt.Errorf(util.ErrUnknownNetworkFamily, f) - } - - client := getClient(ctx, c.Token) - - regions, _, err := client.Regions.List(ctx, &godo.ListOptions{PerPage: 1000}) - if err != nil { - return err - } - var foundRegion bool - for _, region := range regions { - if region.Slug == c.Region { - foundRegion = true - break - } - } - if !foundRegion { - return fmt.Errorf("region %q not found", c.Region) - } - - sizes, _, err := client.Sizes.List(ctx, &godo.ListOptions{PerPage: 1000}) - if err != nil { - return err - } - var foundSize bool - for _, size := range sizes { - if size.Slug == c.Size { - if !size.Available { - return fmt.Errorf("size is not available") - } - - var regionAvailable bool - for _, region := range size.Regions { - if region == c.Region { - regionAvailable = true - break - } - } - - if !regionAvailable { - return fmt.Errorf("size %q is not available in region %q", c.Size, c.Region) - } - - foundSize = true - break - } - } - if !foundSize { - return fmt.Errorf("size %q not found", c.Size) - } - - return nil -} - -// uploadRandomSSHPublicKey generates a random key pair and uploads the public part of the key to -// digital ocean because it is not possible to create a droplet without ssh key assigned -// this method returns an error if the key already exists. -func uploadRandomSSHPublicKey(ctx context.Context, service godo.KeysService) (string, error) { - sshkey, err := ssh.NewKey() - if err != nil { - return "", fmt.Errorf("failed to generate ssh key: %w", err) - } - - existingkey, res, err := service.GetByFingerprint(ctx, sshkey.FingerprintMD5) - if err == nil && existingkey != nil && res.StatusCode >= http.StatusOK && res.StatusCode <= http.StatusAccepted { - return "", fmt.Errorf("failed to create ssh public key, the key already exists") - } - - newDoKey, rsp, err := service.Create(ctx, &godo.KeyCreateRequest{ - PublicKey: sshkey.PublicKey, - Name: sshkey.Name, - }) - if err != nil { - return "", doStatusAndErrToTerminalError(rsp.StatusCode, fmt.Errorf("failed to create ssh public key on digitalocean: %w", err)) - } - - return newDoKey.Fingerprint, nil -} - -func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { - c, pc, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - - client := getClient(ctx, c.Token) - - fingerprint, err := uploadRandomSSHPublicKey(ctx, client.Keys) - if err != nil { - return nil, err - } - defer func() { - _, err := client.Keys.DeleteByFingerprint(ctx, fingerprint) - if err != nil { - log.Errorw("Failed to remove a temporary ssh key", "fingerprint", fingerprint, zap.Error(err)) - } - }() - - slug, err := getSlugForOS(pc.OperatingSystem) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, invalid operating system specified %q: %v", pc.OperatingSystem, err), - } - } - createRequest := &godo.DropletCreateRequest{ - Image: godo.DropletCreateImage{Slug: slug}, - Name: machine.Spec.Name, - Region: c.Region, - Size: c.Size, - IPv6: c.IPv6 || pc.Network.GetIPFamily().IsDualstack(), - PrivateNetworking: c.PrivateNetworking, - Backups: c.Backups, - Monitoring: c.Monitoring, - UserData: userdata, - SSHKeys: []godo.DropletCreateSSHKey{{Fingerprint: fingerprint}}, - Tags: append(c.Tags, string(machine.UID)), - } - - droplet, rsp, err := client.Droplets.Create(ctx, createRequest) - if err != nil { - return nil, doStatusAndErrToTerminalError(rsp.StatusCode, err) - } - - dropletLog := log.With("droplet", droplet.ID) - - //We need to wait until the droplet really got created as tags will be only applied when the droplet is running - err = wait.PollUntilContextTimeout(ctx, createCheckPeriod, createCheckTimeout, false, func(ctx context.Context) (bool, error) { - newDroplet, rsp, err := client.Droplets.Get(ctx, droplet.ID) - if err != nil { - tErr := doStatusAndErrToTerminalError(rsp.StatusCode, err) - if isTerminalError, _, _ := cloudprovidererrors.IsTerminalError(tErr); isTerminalError { - return true, tErr - } - //Well just wait 10 sec and hope the droplet got started by then... - time.Sleep(createCheckFailedWaitPeriod) - return false, fmt.Errorf("droplet %q got created but we failed to fetch its status", droplet.ID) - } - if sets.NewString(newDroplet.Tags...).Has(string(machine.UID)) { - dropletLog.Debug("Droplet got fully created") - return true, nil - } - dropletLog.Debug("Waiting until droplet got fully created...") - return false, nil - }) - - return &doInstance{droplet: droplet}, err -} - -func (p *provider) Cleanup(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { - instance, err := p.get(ctx, machine) - if err != nil { - if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { - return true, nil - } - return false, err - } - - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return false, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - client := getClient(ctx, c.Token) - - doID, err := strconv.Atoi(instance.ID()) - if err != nil { - return false, fmt.Errorf("failed to convert instance id %s to int: %w", instance.ID(), err) - } - - rsp, err := client.Droplets.Delete(ctx, doID) - if err != nil { - return false, doStatusAndErrToTerminalError(rsp.StatusCode, err) - } - - return false, nil -} - -func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - return p.get(ctx, machine) -} - -func (p *provider) get(ctx context.Context, machine *clusterv1alpha1.Machine) (*doInstance, error) { - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - - droplets, err := p.listDroplets(ctx, c.Token) - if err != nil { - return nil, err - } - - for i, droplet := range droplets { - if droplet.Name == machine.Spec.Name && sets.NewString(droplet.Tags...).Has(string(machine.UID)) { - return &doInstance{droplet: &droplets[i]}, nil - } - } - - return nil, cloudprovidererrors.ErrInstanceNotFound -} - -func (p *provider) listDroplets(ctx context.Context, token string) ([]godo.Droplet, error) { - client := getClient(ctx, token) - result := make([]godo.Droplet, 0) - - opt := &godo.ListOptions{ - PerPage: 200, - } - - for { - droplets, resp, err := client.Droplets.List(ctx, opt) - if err != nil { - return nil, doStatusAndErrToTerminalError(resp.StatusCode, fmt.Errorf("failed to get droplets: %w", err)) - } - - result = append(result, droplets...) - - if resp.Links == nil || resp.Links.IsLastPage() { - break - } - - page, err := resp.Links.CurrentPage() - if err != nil { - return nil, err - } - - opt.Page = page + 1 - } - - return result, nil -} - -func (p *provider) MigrateUID(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return fmt.Errorf("failed to decode providerconfig: %w", err) - } - client := getClient(ctx, c.Token) - droplets, _, err := client.Droplets.List(ctx, &godo.ListOptions{PerPage: 1000}) - if err != nil { - return fmt.Errorf("failed to list droplets: %w", err) - } - - // The create does not fail if that tag already exists, it even keep responding with a http/201 - _, response, err := client.Tags.Create(ctx, &godo.TagCreateRequest{Name: string(newUID)}) - if err != nil { - return fmt.Errorf("failed to create new UID tag: %w, status code: %v", err, response.StatusCode) - } - - for _, droplet := range droplets { - if droplet.Name == machine.Spec.Name && sets.NewString(droplet.Tags...).Has(string(machine.UID)) { - tagResourceRequest := &godo.TagResourcesRequest{ - Resources: []godo.Resource{{ID: strconv.Itoa(droplet.ID), Type: godo.DropletResourceType}}, - } - _, err = client.Tags.TagResources(ctx, string(newUID), tagResourceRequest) - if err != nil { - return fmt.Errorf("failed to tag droplet with new UID tag: %w", err) - } - untagResourceRequest := &godo.UntagResourcesRequest{ - Resources: []godo.Resource{{ID: strconv.Itoa(droplet.ID), Type: godo.DropletResourceType}}, - } - _, err = client.Tags.UntagResources(ctx, string(machine.UID), untagResourceRequest) - if err != nil { - return fmt.Errorf("failed to remove old UID tag: %w", err) - } - } - } - - return nil -} - -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return "", "", nil -} - -func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { - labels := make(map[string]string) - - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err == nil { - labels["size"] = c.Size - labels["region"] = c.Region - } - - return labels, err -} - -type doInstance struct { - droplet *godo.Droplet -} - -func (d *doInstance) Name() string { - return d.droplet.Name -} - -func (d *doInstance) ID() string { - return strconv.Itoa(d.droplet.ID) -} - -func (d *doInstance) ProviderID() string { - if d.droplet == nil || d.droplet.Name == "" { - return "" - } - return fmt.Sprintf("digitalocean://%d", d.droplet.ID) -} - -func (d *doInstance) HostID() string { - return "" -} - -func (d *doInstance) Addresses() map[string]v1.NodeAddressType { - addresses := map[string]v1.NodeAddressType{} - for _, n := range d.droplet.Networks.V4 { - if n.Type == "public" { - addresses[n.IPAddress] = v1.NodeExternalIP - } else { - addresses[n.IPAddress] = v1.NodeInternalIP - } - } - for _, n := range d.droplet.Networks.V6 { - if n.Type == "public" { - addresses[n.IPAddress] = v1.NodeExternalIP - } else { - addresses[n.IPAddress] = v1.NodeInternalIP - } - } - return addresses -} - -func (d *doInstance) Status() instance.Status { - switch d.droplet.Status { - case "new": - return instance.StatusCreating - case "active": - return instance.StatusRunning - default: - return instance.StatusUnknown - } -} - -// doStatusAndErrToTerminalError judges if the given HTTP status -// can be qualified as a "terminal" error, for more info see v1alpha1.MachineStatus - -// if the given error doesn't qualify the error passed as -// an argument will be returned. -func doStatusAndErrToTerminalError(status int, err error) error { - switch status { - case http.StatusUnauthorized: - // authorization primitives come from MachineSpec - // thus we are setting InvalidConfigurationMachineError - return cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: "A request has been rejected due to invalid credentials which were taken from the MachineSpec", - } - default: - return err - } -} - -func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { - return nil -} diff --git a/pkg/cloudprovider/provider/digitalocean/types/types.go b/pkg/cloudprovider/provider/digitalocean/types/types.go deleted file mode 100644 index 81114fd6e..000000000 --- a/pkg/cloudprovider/provider/digitalocean/types/types.go +++ /dev/null @@ -1,39 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" -) - -type RawConfig struct { - Token *providerconfigtypes.ConfigVarString `json:"token,omitempty"` - Region *providerconfigtypes.ConfigVarString `json:"region"` - Size *providerconfigtypes.ConfigVarString `json:"size"` - Backups *providerconfigtypes.ConfigVarBool `json:"backups"` - IPv6 *providerconfigtypes.ConfigVarBool `json:"ipv6"` - PrivateNetworking *providerconfigtypes.ConfigVarBool `json:"private_networking"` - Monitoring *providerconfigtypes.ConfigVarBool `json:"monitoring"` - Tags []providerconfigtypes.ConfigVarString `json:"tags,omitempty"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/edge/provider.go b/pkg/cloudprovider/provider/edge/provider.go deleted file mode 100644 index e29af0931..000000000 --- a/pkg/cloudprovider/provider/edge/provider.go +++ /dev/null @@ -1,104 +0,0 @@ -/* -Copyright 2024 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package edge - -import ( - "context" - - "go.uber.org/zap" - - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" -) - -type provider struct{} - -type CloudProviderSpec struct{} - -type CloudProviderInstance struct{} - -func (f CloudProviderInstance) Name() string { - return "" -} - -func (f CloudProviderInstance) ID() string { - return "" -} - -func (f CloudProviderInstance) ProviderID() string { - return "" -} - -func (f CloudProviderInstance) HostID() string { - return "" -} - -func (f CloudProviderInstance) Addresses() map[string]corev1.NodeAddressType { - return nil -} - -func (f CloudProviderInstance) Status() instance.Status { - return instance.StatusUnknown -} - -// New returns a edge cloud provider. -func New(_ *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return &provider{} -} - -func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { - return spec, nil -} - -// Validate returns success or failure based according to its EdgeCloudProviderSpec. -func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, _ clusterv1alpha1.MachineSpec) error { - return nil -} - -func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - return CloudProviderInstance{}, nil -} - -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (string, string, error) { - return "", "", nil -} - -// Create creates a cloud instance according to the given machine. -func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, _ string) (instance.Instance, error) { - return CloudProviderInstance{}, nil -} - -func (p *provider) Cleanup(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { - return true, nil -} - -func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ types.UID) error { - return nil -} - -func (p *provider) MachineMetricsLabels(_ *clusterv1alpha1.Machine) (map[string]string, error) { - return map[string]string{}, nil -} - -func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { - return nil -} diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go deleted file mode 100644 index 6bacaa988..000000000 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ /dev/null @@ -1,572 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package equinixmetal - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "reflect" - "strings" - - "github.com/packethost/packngo" - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - equinixmetaltypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/equinixmetal/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" -) - -const ( - machineUIDTag = "kubermatic-machine-controller:machine-uid" - defaultBillingCycle = "hourly" -) - -// New returns a Equinix Metal provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return &provider{configVarResolver: &providerconfig.ConfigPointerVarResolver{Cvr: configVarResolver}} -} - -type Config struct { - Token string - ProjectID string - BillingCycle string - InstanceType string - Metro string - Facilities []string - Tags []string -} - -// because we have both Config and RawConfig, we need to have func for each -// ideally, these would be merged into one. -func (c *Config) populateDefaults() { - if c.BillingCycle == "" { - c.BillingCycle = defaultBillingCycle - } -} - -func populateDefaults(c *equinixmetaltypes.RawConfig) { - if c.BillingCycle.Value == "" { - c.BillingCycle.Value = defaultBillingCycle - } -} - -type provider struct { - configVarResolver *providerconfig.ConfigPointerVarResolver -} - -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *equinixmetaltypes.RawConfig, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - - pconfig, err := providerconfigtypes.GetConfig(provSpec) - if err != nil { - return nil, nil, nil, err - } - - rawConfig, err := equinixmetaltypes.GetConfig(*pconfig) - if err != nil { - return nil, nil, nil, err - } - - if pconfig.OperatingSystemSpec.Raw == nil { - return nil, nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") - } - - c := Config{} - c.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Token, "METAL_AUTH_TOKEN") - if err != nil || len(c.Token) == 0 { - // This retry is temporary and is only required to facilitate migration from Packet to Equinix Metal - // We look for env variable PACKET_API_KEY associated with Packet to ensure that nothing breaks during automated migration for the Machines - // TODO(@ahmedwaleedmalik) Remove this after a release period - c.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Token, "PACKET_API_KEY") - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get the value of \"apiKey\" field, error = %w", err) - } - } - c.ProjectID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ProjectID, "METAL_PROJECT_ID") - if err != nil || len(c.ProjectID) == 0 { - // This retry is temporary and is only required to facilitate migration from Packet to Equinix Metal - // We look for env variable PACKET_PROJECT_ID associated with Packet to ensure that nothing breaks during automated migration for the Machines - // TODO(@ahmedwaleedmalik) Remove this after a release period - c.ProjectID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ProjectID, "PACKET_PROJECT_ID") - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get the value of \"apiKey\" field, error = %w", err) - } - } - c.InstanceType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.InstanceType) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get the value of \"instanceType\" field, error = %w", err) - } - c.BillingCycle, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.BillingCycle) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get the value of \"billingCycle\" field, error = %w", err) - } - for i, tag := range rawConfig.Tags { - tagValue, err := p.configVarResolver.GetConfigVarStringValue(&tag) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to read the value for the Tag at index %d of the \"tags\" field, error = %w", i, err) - } - c.Tags = append(c.Tags, tagValue) - } - for i, facility := range rawConfig.Facilities { - facilityValue, err := p.configVarResolver.GetConfigVarStringValue(&facility) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to read the value for the Tag at index %d of the \"facilities\" field, error = %w", i, err) - } - c.Facilities = append(c.Facilities, facilityValue) - } - c.Metro, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Metro) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get the value of \"metro\" field, error = %w", err) - } - - // ensure we have defaults - c.populateDefaults() - - return &c, rawConfig, pconfig, err -} - -func (p *provider) getMetalDevice(machine *clusterv1alpha1.Machine) (*packngo.Device, *packngo.Client, error) { - c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - - client := getClient(c.Token) - device, err := getDeviceByTag(client, c.ProjectID, generateTag(string(machine.UID))) - if err != nil { - return nil, nil, err - } - return device, client, nil -} - -func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { - c, _, pc, err := p.getConfig(spec.ProviderSpec) - if err != nil { - return fmt.Errorf("failed to parse config: %w", err) - } - - if c.Token == "" { - return errors.New("apiKey is missing") - } - if c.InstanceType == "" { - return errors.New("instanceType is missing") - } - if c.ProjectID == "" { - return errors.New("projectID is missing") - } - - _, err = getNameForOS(pc.OperatingSystem) - if err != nil { - return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, err) - } - - client := getClient(c.Token) - - if c.Metro == "" && (len(c.Facilities) == 0 || c.Facilities[0] == "") { - return fmt.Errorf("must have at least one non-blank facility or a metro") - } - - if c.Facilities != nil && (len(c.Facilities) > 0 || c.Facilities[0] != "") { - // get all valid facilities - facilities, _, err := client.Facilities.List(nil) - if err != nil { - return fmt.Errorf("failed to list facilities: %w", err) - } - // ensure our requested facilities are in those facilities - if missingFacilities := itemsNotInList(facilityProp(facilities, "Code"), c.Facilities); len(missingFacilities) > 0 { - return fmt.Errorf("unknown facilities: %s", strings.Join(missingFacilities, ",")) - } - } - - if c.Metro != "" { - metros, _, err := client.Metros.List(nil) - if err != nil { - return fmt.Errorf("failed to list metros: %w", err) - } - - var metroExists bool - for _, metro := range metros { - if strings.EqualFold(metro.Code, c.Metro) { - metroExists = true - } - } - - if !metroExists { - return fmt.Errorf("unknown metro: %s", c.Metro) - } - } - - // get all valid plans a.k.a. instance types - plans, _, err := client.Plans.List(nil) - if err != nil { - return fmt.Errorf("failed to list instance types / plans: %w", err) - } - // ensure our requested plan is in those plans - validPlanNames := planProp(plans, "Name") - if missingPlans := itemsNotInList(validPlanNames, []string{c.InstanceType}); len(missingPlans) > 0 { - return fmt.Errorf("unknown instance type / plan: %s, acceptable plans: %s", strings.Join(missingPlans, ","), strings.Join(validPlanNames, ",")) - } - - return nil -} - -func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { - c, _, pc, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - - client := getClient(c.Token) - - imageName, err := getNameForOS(pc.OperatingSystem) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Invalid operating system specified %q, details = %v", pc.OperatingSystem, err), - } - } - - serverCreateOpts := &packngo.DeviceCreateRequest{ - Hostname: machine.Spec.Name, - UserData: userdata, - ProjectID: c.ProjectID, - Facility: c.Facilities, - Metro: c.Metro, - BillingCycle: c.BillingCycle, - Plan: c.InstanceType, - OS: imageName, - Tags: []string{ - generateTag(string(machine.UID)), - }, - } - - device, res, err := client.Devices.Create(serverCreateOpts) - if err != nil { - return nil, metalErrorToTerminalError(err, res, "failed to create server") - } - - return &metalDevice{device: device}, nil -} - -func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { - instance, err := p.Get(ctx, log, machine, data) - if err != nil { - if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { - return true, nil - } - return false, err - } - - c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return false, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - - client := getClient(c.Token) - res, err := client.Devices.Delete(instance.(*metalDevice).device.ID, false) - if err != nil { - return false, metalErrorToTerminalError(err, res, "failed to delete the server") - } - - return false, nil -} - -func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { - _, rawConfig, _, err := p.getConfig(spec.ProviderSpec) - if err != nil { - return spec, err - } - populateDefaults(rawConfig) - spec.ProviderSpec.Value, err = setProviderSpec(*rawConfig, spec.ProviderSpec) - if err != nil { - return spec, err - } - return spec, nil -} - -func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - device, _, err := p.getMetalDevice(machine) - if err != nil { - return nil, err - } - if device != nil { - return &metalDevice{device: device}, nil - } - - return nil, cloudprovidererrors.ErrInstanceNotFound -} - -func (p *provider) MigrateUID(_ context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newID types.UID) error { - device, client, err := p.getMetalDevice(machine) - if err != nil { - return err - } - if device == nil { - log.Info("No instance exists for machine") - return nil - } - - // go through existing labels, make sure that no other UID label exists - tags := make([]string, 0) - for _, t := range device.Tags { - // filter out old UID tag(s) - if _, err := getTagUID(t); err != nil { - tags = append(tags, t) - } - } - - // create a new UID label - tags = append(tags, generateTag(string(newID))) - - log.Info("Setting UID label for machine") - dur := &packngo.DeviceUpdateRequest{ - Tags: &tags, - } - _, response, err := client.Devices.Update(device.ID, dur) - if err != nil { - return metalErrorToTerminalError(err, response, "failed to update UID label") - } - log.Info("Successfully set UID label for machine") - - return nil -} - -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return "", "", nil -} - -func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { - labels := make(map[string]string) - - c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err == nil { - labels["size"] = c.InstanceType - labels["facilities"] = strings.Join(c.Facilities, ",") - } - - return labels, err -} - -func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { - return nil -} - -type metalDevice struct { - device *packngo.Device -} - -func (s *metalDevice) Name() string { - return s.device.Hostname -} - -func (s *metalDevice) HostID() string { - return "" -} - -func (s *metalDevice) ID() string { - return s.device.ID -} - -func (s *metalDevice) ProviderID() string { - if s.device == nil || s.device.ID == "" { - return "" - } - return "equinixmetal://" + s.device.ID -} - -func (s *metalDevice) Addresses() map[string]v1.NodeAddressType { - // returns addresses in CIDR format - addresses := map[string]v1.NodeAddressType{} - for _, ip := range s.device.Network { - if ip.Public { - addresses[ip.Address] = v1.NodeExternalIP - continue - } - addresses[ip.Address] = v1.NodeInternalIP - } - - return addresses -} - -func (s *metalDevice) Status() instance.Status { - switch s.device.State { - case "provisioning": - return instance.StatusCreating - case "active": - return instance.StatusRunning - default: - return instance.StatusUnknown - } -} - -// CONVENIENCE INTERNAL FUNCTIONS. -func setProviderSpec(rawConfig equinixmetaltypes.RawConfig, s clusterv1alpha1.ProviderSpec) (*runtime.RawExtension, error) { - if s.Value == nil { - return nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - - pconfig, err := providerconfigtypes.GetConfig(s) - if err != nil { - return nil, err - } - - rawCloudProviderSpec, err := json.Marshal(rawConfig) - if err != nil { - return nil, err - } - - pconfig.CloudProviderSpec = runtime.RawExtension{Raw: rawCloudProviderSpec} - rawPconfig, err := json.Marshal(pconfig) - if err != nil { - return nil, err - } - - return &runtime.RawExtension{Raw: rawPconfig}, nil -} - -func getDeviceByTag(client *packngo.Client, projectID, tag string) (*packngo.Device, error) { - devices, response, err := client.Devices.List(projectID, nil) - if err != nil { - return nil, metalErrorToTerminalError(err, response, "failed to list devices") - } - - for _, device := range devices { - if itemInList(device.Tags, tag) { - return &device, nil - } - } - return nil, nil -} - -// given a defined Kubermatic constant for an operating system, return the canonical slug for Equinix Metal. -func getNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { - switch os { - case providerconfigtypes.OperatingSystemUbuntu: - return "ubuntu_22_04", nil - case providerconfigtypes.OperatingSystemCentOS: - return "centos_7", nil - case providerconfigtypes.OperatingSystemFlatcar: - return "flatcar_stable", nil - case providerconfigtypes.OperatingSystemRockyLinux: - return "rocky_8", nil - } - return "", providerconfigtypes.ErrOSNotSupported -} - -func getClient(apiKey string) *packngo.Client { - return packngo.NewClientWithAuth("kubermatic", apiKey, nil) -} - -func generateTag(ID string) string { - return fmt.Sprintf("%s:%s", machineUIDTag, ID) -} - -func getTagUID(tag string) (string, error) { - parts := strings.Split(tag, ":") - if len(parts) < 2 || parts[0] != machineUIDTag { - return "", fmt.Errorf("not a machine UID tag") - } - return parts[1], nil -} - -// metalErrorToTerminalError judges if the given error -// can be qualified as a "terminal" error, for more info see v1alpha1.MachineStatus -// -// if the given error doesn't qualify the error passed as an argument will be returned. -func metalErrorToTerminalError(err error, response *packngo.Response, msg string) error { - prepareAndReturnError := func() error { - return fmt.Errorf("%s, due to %w", msg, err) - } - - if err != nil { - if response != nil && response.Response != nil && response.Response.StatusCode == 403 { - // authorization primitives come from MachineSpec - // thus we are setting InvalidConfigurationMachineError - return cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: "A request has been rejected due to invalid credentials which were taken from the MachineSpec", - } - } - - return prepareAndReturnError() - } - - return err -} - -func itemInList(list []string, item string) bool { - for _, elm := range list { - if elm == item { - return true - } - } - return false -} - -func itemsNotInList(list, items []string) []string { - listMap := make(map[string]bool) - missing := make([]string, 0) - for _, item := range list { - listMap[item] = true - } - for _, item := range items { - if _, ok := listMap[item]; !ok { - missing = append(missing, item) - } - } - return missing -} - -func facilityProp(vs []packngo.Facility, field string) []string { - vsm := make([]string, len(vs)) - for i, v := range vs { - val := reflect.ValueOf(v) - vsm[i] = val.FieldByName(field).String() - } - return vsm -} - -func planProp(vs []packngo.Plan, field string) []string { - vsm := make([]string, len(vs)) - for i, v := range vs { - val := reflect.ValueOf(v) - vsm[i] = val.FieldByName(field).String() - } - return vsm -} diff --git a/pkg/cloudprovider/provider/equinixmetal/types/types.go b/pkg/cloudprovider/provider/equinixmetal/types/types.go deleted file mode 100644 index ba58a1d99..000000000 --- a/pkg/cloudprovider/provider/equinixmetal/types/types.go +++ /dev/null @@ -1,38 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" -) - -type RawConfig struct { - Token *providerconfigtypes.ConfigVarString `json:"token,omitempty"` - ProjectID *providerconfigtypes.ConfigVarString `json:"projectID,omitempty"` - BillingCycle *providerconfigtypes.ConfigVarString `json:"billingCycle"` - InstanceType *providerconfigtypes.ConfigVarString `json:"instanceType"` - Metro *providerconfigtypes.ConfigVarString `json:"metro,omitempty"` - Facilities []providerconfigtypes.ConfigVarString `json:"facilities"` - Tags []providerconfigtypes.ConfigVarString `json:"tags,omitempty"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go deleted file mode 100644 index 2beb82f2a..000000000 --- a/pkg/cloudprovider/provider/gce/config.go +++ /dev/null @@ -1,325 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// Google Cloud Provider for the Machine Controller -// - -package gce - -import ( - "context" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "strings" - - "golang.org/x/oauth2" - googleoauth "golang.org/x/oauth2/google" - "google.golang.org/api/compute/v1" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - gcetypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/gce/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" -) - -// Environment variables for the configuration of the Google Cloud project access. -const ( - envGoogleServiceAccount = "GOOGLE_SERVICE_ACCOUNT" -) - -// imageProjects maps the OS to the Google Cloud image projects. -var imageProjects = map[providerconfigtypes.OperatingSystem]string{ - providerconfigtypes.OperatingSystemUbuntu: "ubuntu-os-cloud", - providerconfigtypes.OperatingSystemFlatcar: "kinvolk-public", -} - -// imageFamilies maps the OS to the Google Cloud image projects. -var imageFamilies = map[providerconfigtypes.OperatingSystem]string{ - providerconfigtypes.OperatingSystemUbuntu: "ubuntu-2204-lts", - providerconfigtypes.OperatingSystemFlatcar: "flatcar-stable", -} - -// diskTypes are the disk types of the Google Cloud. Map is used for -// validation. -var diskTypes = map[string]bool{ - "pd-standard": true, - "pd-ssd": true, -} - -// Default values for disk type and size (in GB). -const ( - defaultDiskType = "pd-standard" - defaultDiskSize = 25 -) - -// newCloudProviderSpec creates a cloud provider specification out of the -// given ProviderSpec. -func newCloudProviderSpec(provSpec v1alpha1.ProviderSpec) (*gcetypes.CloudProviderSpec, *providerconfigtypes.Config, error) { - // Retrieve provider configuration from machine specification. - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - - pconfig, err := providerconfigtypes.GetConfig(provSpec) - if err != nil { - return nil, nil, fmt.Errorf("cannot unmarshal machine.spec.providerconfig.value: %w", err) - } - - if pconfig.OperatingSystemSpec.Raw == nil { - return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") - } - - // Retrieve cloud provider specification from cloud provider specification. - cpSpec, err := gcetypes.GetConfig(*pconfig) - if err != nil { - return nil, nil, fmt.Errorf("cannot unmarshal cloud provider specification: %w", err) - } - - return cpSpec, pconfig, nil -} - -// config contains the configuration of the Provider. -type config struct { - serviceAccount string - projectID string - zone string - machineType string - diskSize int64 - diskType string - network string - subnetwork string - preemptible bool - automaticRestart *bool - provisioningModel *string - labels map[string]string - tags []string - providerConfig *providerconfigtypes.Config - assignPublicIPAddress bool - multizone bool - regional bool - customImage string - disableMachineServiceAccount bool - enableNestedVirtualization bool - minCPUPlatform string - guestOSFeatures []string - clientConfig *clientConfig -} - -type clientConfig struct { - ClientEmail string - TokenSource oauth2.TokenSource -} - -// newConfig creates a Provider configuration out of the passed resolver and spec. -func newConfig(resolver *providerconfig.ConfigPointerVarResolver, spec v1alpha1.ProviderSpec) (*config, error) { - // Create cloud provider spec. - cpSpec, providerConfig, err := newCloudProviderSpec(spec) - if err != nil { - return nil, err - } - - // Setup configuration. - cfg := &config{ - providerConfig: providerConfig, - labels: cpSpec.Labels, - tags: cpSpec.Tags, - diskSize: cpSpec.DiskSize, - guestOSFeatures: cpSpec.GuestOSFeatures, - } - - cfg.serviceAccount, err = resolver.GetConfigVarStringValueOrEnv(cpSpec.ServiceAccount, envGoogleServiceAccount) - if err != nil { - return nil, fmt.Errorf("cannot retrieve service account: %w", err) - } - - cfg.projectID, err = resolver.GetConfigVarStringValue(&cpSpec.ProjectID) - if err != nil { - return nil, fmt.Errorf("failed to retrieve project id: %w", err) - } - - err = cfg.postprocessServiceAccount() - if err != nil { - return nil, fmt.Errorf("cannot prepare JWT: %w", err) - } - - cfg.zone, err = resolver.GetConfigVarStringValue(cpSpec.Zone) - if err != nil { - return nil, fmt.Errorf("cannot retrieve zone: %w", err) - } - - cfg.machineType, err = resolver.GetConfigVarStringValue(cpSpec.MachineType) - if err != nil { - return nil, fmt.Errorf("cannot retrieve machine type: %w", err) - } - - cfg.diskType, err = resolver.GetConfigVarStringValue(cpSpec.DiskType) - if err != nil { - return nil, fmt.Errorf("cannot retrieve disk type: %w", err) - } - - cfg.network, err = resolver.GetConfigVarStringValue(cpSpec.Network) - if err != nil { - return nil, fmt.Errorf("cannot retrieve network: %w", err) - } - - cfg.subnetwork, err = resolver.GetConfigVarStringValue(cpSpec.Subnetwork) - if err != nil { - return nil, fmt.Errorf("cannot retrieve subnetwork: %w", err) - } - - cfg.preemptible, _, err = resolver.GetConfigVarBoolValue(cpSpec.Preemptible) - if err != nil { - return nil, fmt.Errorf("cannot retrieve preemptible: %w", err) - } - - if cpSpec.AutomaticRestart != nil { - automaticRestart, _, err := resolver.GetConfigVarBoolValue(cpSpec.AutomaticRestart) - if err != nil { - return nil, fmt.Errorf("cannot retrieve automaticRestart: %w", err) - } - cfg.automaticRestart = &automaticRestart - - if *cfg.automaticRestart && cfg.preemptible { - return nil, fmt.Errorf("automatic restart option can only be enabled for standard instances. Preemptible instances cannot be automatically restarted") - } - } - - if cpSpec.ProvisioningModel != nil { - provisioningModel, err := resolver.GetConfigVarStringValue(cpSpec.ProvisioningModel) - if err != nil { - return nil, fmt.Errorf("cannot retrieve provisioningModel: %w", err) - } - cfg.provisioningModel = &provisioningModel - } - - // make it true by default - cfg.assignPublicIPAddress = true - - if cpSpec.AssignPublicIPAddress != nil { - cfg.assignPublicIPAddress, _, err = resolver.GetConfigVarBoolValue(cpSpec.AssignPublicIPAddress) - if err != nil { - return nil, fmt.Errorf("failed to retrieve assignPublicIPAddress: %w", err) - } - } - - cfg.multizone, _, err = resolver.GetConfigVarBoolValue(cpSpec.MultiZone) - if err != nil { - return nil, fmt.Errorf("failed to retrieve multizone: %w", err) - } - - cfg.regional, _, err = resolver.GetConfigVarBoolValue(cpSpec.Regional) - if err != nil { - return nil, fmt.Errorf("failed to retrieve regional: %w", err) - } - - cfg.customImage, err = resolver.GetConfigVarStringValue(cpSpec.CustomImage) - if err != nil { - return nil, fmt.Errorf("failed to retrieve gce custom image: %w", err) - } - - cfg.disableMachineServiceAccount, _, err = resolver.GetConfigVarBoolValue(cpSpec.DisableMachineServiceAccount) - if err != nil { - return nil, fmt.Errorf("failed to retrieve disable machine service account: %w", err) - } - - cfg.enableNestedVirtualization, _, err = resolver.GetConfigVarBoolValue(cpSpec.EnableNestedVirtualization) - if err != nil { - return nil, fmt.Errorf("failed to retrieve enable nested virtualization: %w", err) - } - - cfg.minCPUPlatform, err = resolver.GetConfigVarStringValue(cpSpec.MinCPUPlatform) - if err != nil { - return nil, fmt.Errorf("failed to retrieve min cpu platform: %w", err) - } - - return cfg, nil -} - -// postprocessServiceAccount processes the service account and creates a JWT configuration -// out of it. -func (cfg *config) postprocessServiceAccount() error { - sa := cfg.serviceAccount - - // safely decode the service account, in case we did not read the value - // from a "known-safe" location (like the MachineDeployment), but from - // an environment variable. - decoded, err := base64.StdEncoding.DecodeString(cfg.serviceAccount) - if err == nil { - sa = string(decoded) - } - - creds, err := googleoauth.CredentialsFromJSON(context.TODO(), []byte(sa), compute.ComputeScope) - if err != nil { - return fmt.Errorf("failed to parse credentials from google service account: %w", err) - } - - if cfg.projectID == "" { - cfg.projectID = creds.ProjectID - } - - sam := map[string]string{} - err = json.Unmarshal([]byte(sa), &sam) - if err != nil { - return fmt.Errorf("failed unmarshalling service account: %w", err) - } - - // if the project id is not set in the machine deployment, we fallback to the project id that is embedded in the - // google service account json object. - cfg.clientConfig = &clientConfig{ - ClientEmail: sam["client_email"], - TokenSource: creds.TokenSource, - } - - return nil -} - -// machineTypeDescriptor creates the descriptor out of zone and machine type -// for the machine type of an instance. -func (cfg *config) machineTypeDescriptor() string { - return fmt.Sprintf("zones/%s/machineTypes/%s", cfg.zone, cfg.machineType) -} - -// diskTypeDescriptor creates the descriptor out of zone and disk type -// for the disk type of an instance. -func (cfg *config) diskTypeDescriptor() string { - return fmt.Sprintf("zones/%s/diskTypes/%s", cfg.zone, cfg.diskType) -} - -// sourceImageDescriptor creates the descriptor out of project and family -// for the source image of an instance boot disk. -func (cfg *config) sourceImageDescriptor() (string, error) { - if cfg.customImage != "" { - // If a full image identifier is provided, use it - if strings.HasPrefix("projects/", cfg.customImage) { - return cfg.customImage, nil - } - - // Otherwise, make sure to properly prefix the image identifier - return fmt.Sprintf("global/images/%s", cfg.customImage), nil - } - project, ok := imageProjects[cfg.providerConfig.OperatingSystem] - if !ok { - return "", providerconfigtypes.ErrOSNotSupported - } - family, ok := imageFamilies[cfg.providerConfig.OperatingSystem] - if !ok { - return "", providerconfigtypes.ErrOSNotSupported - } - return fmt.Sprintf("projects/%s/global/images/family/%s", project, family), nil -} diff --git a/pkg/cloudprovider/provider/gce/instance.go b/pkg/cloudprovider/provider/gce/instance.go deleted file mode 100644 index b191a00bc..000000000 --- a/pkg/cloudprovider/provider/gce/instance.go +++ /dev/null @@ -1,131 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// Google Cloud Provider for the Machine Controller -// - -package gce - -import ( - "fmt" - "strconv" - - "google.golang.org/api/compute/v1" - - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - - v1 "k8s.io/api/core/v1" -) - -// Possible instance statuses. -const ( - statusInstanceProvisioning = "PROVISIONING" - statusInstanceRunning = "RUNNING" - statusInstanceStaging = "STAGING" - statusInstanceStopped = "STOPPED" - statusInstanceStopping = "STOPPING" - statusInstanceSuspended = "SUSPENDED" - statusInstanceSuspending = "SUSPENDING" - statusInstanceTerminated = "TERMINATED" -) - -// googleInstance implements instance.Instance for the Google compute instance. -type googleInstance struct { - ci *compute.Instance - projectID string - zone string -} - -// Name implements instance.Instance. -func (gi *googleInstance) Name() string { - return gi.ci.Name -} - -// ID implements instance.Instance. -func (gi *googleInstance) ID() string { - return strconv.FormatUint(gi.ci.Id, 10) -} - -func (gi *googleInstance) ProviderID() string { - if gi.ci == nil || gi.ci.Name == "" { - return "" - } - return fmt.Sprintf("gce://%s/%s/%s", gi.projectID, gi.zone, gi.ci.Name) -} - -func (gi *googleInstance) HostID() string { - return "" -} - -// Addresses implements instance.Instance. -func (gi *googleInstance) Addresses() map[string]v1.NodeAddressType { - addrs := map[string]v1.NodeAddressType{} - for _, ifc := range gi.ci.NetworkInterfaces { - addrs[ifc.NetworkIP] = v1.NodeInternalIP - for _, ac := range ifc.AccessConfigs { - addrs[ac.NatIP] = v1.NodeExternalIP - } - for _, ac := range ifc.Ipv6AccessConfigs { - addrs[ac.ExternalIpv6] = v1.NodeExternalIP - } - } - - // GCE has two types of the internal DNS, so we need to take both - // into the account: - // https://cloud.google.com/compute/docs/internal-dns#instance-fully-qualified-domain-names - // Zonal DNS is present for newer projects and has the following FQDN format: - // [INSTANCE_NAME].[ZONE].c.[PROJECT_ID].internal - zonalDNS := fmt.Sprintf("%s.%s.c.%s.internal", gi.ci.Name, gi.zone, gi.projectID) - addrs[zonalDNS] = v1.NodeInternalDNS - - // Global DNS is present for older projects and has the following FQDN format: - // [INSTANCE_NAME].c.[PROJECT_ID].internal - globalDNS := fmt.Sprintf("%s.c.%s.internal", gi.ci.Name, gi.projectID) - addrs[globalDNS] = v1.NodeInternalDNS - - // GCP provides the search paths to resolve the machine's name, - // so we add is as a DNS name - // https://cloud.google.com/compute/docs/internal-dns#resolv.conf - addrs[gi.ci.Name] = v1.NodeInternalDNS - - return addrs -} - -// Status implements instance.Instance. -// TODO Check status mapping for staging, delet(ed|ing), suspend(ed|ing). -func (gi *googleInstance) Status() instance.Status { - switch gi.ci.Status { - case statusInstanceProvisioning: - return instance.StatusCreating - case statusInstanceRunning: - return instance.StatusRunning - case statusInstanceStaging: - return instance.StatusCreating - case statusInstanceStopped: - return instance.StatusDeleted - case statusInstanceStopping: - return instance.StatusDeleting - case statusInstanceSuspended: - return instance.StatusDeleted - case statusInstanceSuspending: - return instance.StatusDeleting - case statusInstanceTerminated: - return instance.StatusDeleted - } - // Must not happen. - return instance.StatusUnknown -} diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go deleted file mode 100644 index 5c7c5f90e..000000000 --- a/pkg/cloudprovider/provider/gce/provider.go +++ /dev/null @@ -1,412 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// Google Cloud Provider for the Machine Controller -// - -package gce - -import ( - "context" - "errors" - "fmt" - "net/http" - "strconv" - - "cloud.google.com/go/logging" - monitoring "cloud.google.com/go/monitoring/apiv3/v2" - "go.uber.org/zap" - compute "google.golang.org/api/compute/v1" - "google.golang.org/api/googleapi" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - gcetypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/gce/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - - "k8s.io/apimachinery/pkg/types" -) - -// Terminal error messages. -const ( - errMachineSpec = "Failed to parse MachineSpec: %v" - errOperatingSystem = "Invalid or not supported operating system specified %q: %v" - errConnect = "Failed to connect: %v" - errInvalidServiceAccount = "Service account is missing" - errInvalidZone = "Zone is missing" - errInvalidMachineType = "Machine type is missing" - errInvalidDiskSize = "Disk size must be a positive number" - errInvalidDiskType = "Disk type is missing or has wrong type, allowed are 'pd-standard' and 'pd-ssd'" - errRetrieveInstance = "Failed to retrieve instance: %v" - errGotTooManyInstances = "Got more than 1 instance matching the machine UID label" - errCloudConfig = "Failed to convert cloud-config to string: %v" - errInsertInstance = "Failed to insert instance: %v" - errDeleteInstance = "Failed to delete instance: %v" - errSetLabels = "Failed to set the labels for the new machine UID: %v" -) - -// Instance labels. -const ( - labelMachineName = "machine_name" - labelMachineUID = "machine_uid" -) - -// Compile time verification of Provider implementing cloud.Provider. -var _ cloudprovidertypes.Provider = New(nil) - -// Provider implements the cloud.Provider interface for the Google Cloud Platform. -type Provider struct { - resolver *providerconfig.ConfigPointerVarResolver -} - -// New creates a cloud provider instance for the Google Cloud Platform. -func New(configVarResolver *providerconfig.ConfigVarResolver) *Provider { - return &Provider{ - resolver: &providerconfig.ConfigPointerVarResolver{ - Cvr: configVarResolver, - }, - } -} - -// AddDefaults reads the MachineSpec and applies defaults for provider specific fields. -func (p *Provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { - // Read cloud provider spec. - cpSpec, _, err := newCloudProviderSpec(spec.ProviderSpec) - if err != nil { - return spec, newError(common.InvalidConfigurationMachineError, errMachineSpec, err) - } - // Check and set defaults. - if cpSpec.DiskSize == 0 { - cpSpec.DiskSize = defaultDiskSize - } - if cpSpec.DiskType.Value == "" { - cpSpec.DiskType.Value = defaultDiskType - } - spec.ProviderSpec.Value, err = cpSpec.UpdateProviderSpec(spec.ProviderSpec) - return spec, err -} - -// Validate checks the given machine's specification. -func (p *Provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { - // Read configuration. - cfg, err := newConfig(p.resolver, spec.ProviderSpec) - if err != nil { - return newError(common.InvalidConfigurationMachineError, errMachineSpec, err) - } - // Check configured values. - if cfg.serviceAccount == "" { - return newError(common.InvalidConfigurationMachineError, errInvalidServiceAccount) - } - if cfg.zone == "" { - return newError(common.InvalidConfigurationMachineError, errInvalidZone) - } - - switch cfg.providerConfig.Network.GetIPFamily() { - case util.IPFamilyUnspecified, util.IPFamilyIPv4: - // noop - case util.IPFamilyIPv6: - return newError(common.InvalidConfigurationMachineError, util.ErrIPv6OnlyUnsupported) - case util.IPFamilyIPv4IPv6, util.IPFamilyIPv6IPv4: - default: - return newError(common.InvalidConfigurationMachineError, util.ErrUnknownNetworkFamily, cfg.providerConfig.Network.GetIPFamily()) - } - - if cfg.machineType == "" { - return newError(common.InvalidConfigurationMachineError, errInvalidMachineType) - } - if cfg.diskSize < 1 { - return newError(common.InvalidConfigurationMachineError, errInvalidDiskSize) - } - if !diskTypes[cfg.diskType] { - return newError(common.InvalidConfigurationMachineError, errInvalidDiskType) - } - _, err = cfg.sourceImageDescriptor() - if err != nil { - return newError(common.InvalidConfigurationMachineError, errOperatingSystem, cfg.providerConfig.OperatingSystem, err) - } - return nil -} - -// Get retrieves a node instance that is associated with the given machine. -func (p *Provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - return p.get(ctx, machine) -} - -func (p *Provider) get(ctx context.Context, machine *clusterv1alpha1.Machine) (*googleInstance, error) { - // Read configuration. - cfg, err := newConfig(p.resolver, machine.Spec.ProviderSpec) - if err != nil { - return nil, newError(common.InvalidConfigurationMachineError, errMachineSpec, err) - } - // Connect to Google compute. - svc, err := connectComputeService(ctx, cfg) - if err != nil { - return nil, newError(common.InvalidConfigurationMachineError, errConnect, err) - } - // Retrieve instance. - label := fmt.Sprintf("labels.%s=%s", labelMachineUID, machine.UID) - insts, err := svc.Instances.List(cfg.projectID, cfg.zone).Filter(label).Do() - if err != nil { - var gerr *googleapi.Error - if errors.As(err, &gerr) { - if gerr.Code == http.StatusNotFound { - return nil, cloudprovidererrors.ErrInstanceNotFound - } - } - return nil, newError(common.InvalidConfigurationMachineError, errRetrieveInstance, err) - } - if len(insts.Items) == 0 { - return nil, cloudprovidererrors.ErrInstanceNotFound - } - if len(insts.Items) > 1 { - return nil, newError(common.InvalidConfigurationMachineError, errGotTooManyInstances) - } - return &googleInstance{ - ci: insts.Items[0], - projectID: cfg.projectID, - zone: cfg.zone, - }, nil -} - -// GetCloudConfig returns the cloud provider specific cloud-config for the kubelet. -func (p *Provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { - // Read configuration. - cfg, err := newConfig(p.resolver, spec.ProviderSpec) - if err != nil { - return "", "", newError(common.InvalidConfigurationMachineError, errMachineSpec, err) - } - // Init cloud configuration. - cc := &gcetypes.CloudConfig{ - Global: gcetypes.GlobalOpts{ - ProjectID: cfg.projectID, - LocalZone: cfg.zone, - MultiZone: cfg.multizone, - Regional: cfg.regional, - NetworkName: cfg.network, - SubnetworkName: cfg.subnetwork, - NodeTags: cfg.tags, - }, - } - config, err = cc.AsString() - if err != nil { - return "", "", newError(common.InvalidConfigurationMachineError, errCloudConfig, err) - } - return config, "gce", nil -} - -// Create inserts a cloud instance according to the given machine. -func (p *Provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { - // Read configuration. - cfg, err := newConfig(p.resolver, machine.Spec.ProviderSpec) - if err != nil { - return nil, newError(common.InvalidConfigurationMachineError, errMachineSpec, err) - } - // Connect to Google compute. - svc, err := connectComputeService(ctx, cfg) - if err != nil { - return nil, newError(common.InvalidConfigurationMachineError, errConnect, err) - } - // Create Google compute instance spec and insert it. - networkInterfaces, err := svc.networkInterfaces(log, cfg) - if err != nil { - return nil, newError(common.InvalidConfigurationMachineError, errMachineSpec, err) - } - disks, err := svc.attachedDisks(cfg) - if err != nil { - return nil, newError(common.InvalidConfigurationMachineError, errMachineSpec, err) - } - labels := map[string]string{} - for k, v := range cfg.labels { - labels[k] = v - } - labels[labelMachineName] = machine.Spec.Name - labels[labelMachineUID] = string(machine.UID) - inst := &compute.Instance{ - Name: machine.Spec.Name, - MachineType: cfg.machineTypeDescriptor(), - NetworkInterfaces: networkInterfaces, - Disks: disks, - Labels: labels, - Scheduling: &compute.Scheduling{ - Preemptible: cfg.preemptible, - }, - Metadata: &compute.Metadata{ - Items: []*compute.MetadataItems{ - { - Key: "user-data", - Value: &userdata, - }, - }, - }, - Tags: &compute.Tags{ - Items: cfg.tags, - }, - } - - if !cfg.disableMachineServiceAccount { - inst.ServiceAccounts = []*compute.ServiceAccount{ - { - Email: cfg.clientConfig.ClientEmail, - Scopes: append( - monitoring.DefaultAuthScopes(), - compute.ComputeScope, - compute.DevstorageReadOnlyScope, - logging.WriteScope, - ), - }, - } - } - - if cfg.automaticRestart != nil { - inst.Scheduling.AutomaticRestart = cfg.automaticRestart - } - - if cfg.provisioningModel != nil { - inst.Scheduling.ProvisioningModel = *cfg.provisioningModel - } - - if cfg.enableNestedVirtualization { - inst.AdvancedMachineFeatures = &compute.AdvancedMachineFeatures{ - EnableNestedVirtualization: true, - } - } - - if cfg.minCPUPlatform != "" { - inst.MinCpuPlatform = cfg.minCPUPlatform - } - - op, err := svc.Instances.Insert(cfg.projectID, cfg.zone, inst).Do() - if err != nil { - return nil, newError(common.InvalidConfigurationMachineError, errInsertInstance, err) - } - err = svc.waitZoneOperation(ctx, cfg, op.Name) - if err != nil { - return nil, newError(common.InvalidConfigurationMachineError, errInsertInstance, err) - } - // Retrieve it to get a full qualified instance. - return p.Get(ctx, log, machine, data) -} - -// Cleanup deletes the instance associated with the machine and all associated resources. -func (p *Provider) Cleanup(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { - // Read configuration. - cfg, err := newConfig(p.resolver, machine.Spec.ProviderSpec) - if err != nil { - return false, newError(common.InvalidConfigurationMachineError, errMachineSpec, err) - } - // Connect to Google compute. - svc, err := connectComputeService(ctx, cfg) - if err != nil { - return false, newError(common.InvalidConfigurationMachineError, errConnect, err) - } - // Delete instance. - op, err := svc.Instances.Delete(cfg.projectID, cfg.zone, machine.Spec.Name).Do() - if err != nil { - var gerr *googleapi.Error - if errors.As(err, &gerr) { - if gerr.Code == http.StatusNotFound { - return true, nil - } - } - return false, newError(common.InvalidConfigurationMachineError, errDeleteInstance, err) - } - err = svc.waitZoneOperation(ctx, cfg, op.Name) - if err != nil { - return false, newError(common.InvalidConfigurationMachineError, errDeleteInstance, err) - } - - return false, nil -} - -// MachineMetricsLabels returns labels used for the Prometheus metrics about created machines. -func (p *Provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { - // Read configuration. - cfg, err := newConfig(p.resolver, machine.Spec.ProviderSpec) - if err != nil { - return nil, newError(common.InvalidConfigurationMachineError, errMachineSpec, err) - } - // Create labels. - labels := map[string]string{} - - labels["project"] = cfg.projectID - labels["zone"] = cfg.zone - labels["type"] = cfg.machineType - labels["disksize"] = strconv.FormatInt(cfg.diskSize, 10) - labels["disktype"] = cfg.diskType - - return labels, nil -} - -// MigrateUID updates the UID of an instance after the controller migrates types -// and the UID of the machine object changed. -func (p *Provider) MigrateUID(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { - // Read configuration. - cfg, err := newConfig(p.resolver, machine.Spec.ProviderSpec) - if err != nil { - return newError(common.InvalidConfigurationMachineError, errMachineSpec, err) - } - // Connect to Google compute. - svc, err := connectComputeService(ctx, cfg) - if err != nil { - return newError(common.InvalidConfigurationMachineError, errConnect, err) - } - // Retrieve instance. - inst, err := p.get(ctx, machine) - if err != nil { - if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { - return nil - } - return err - } - ci := inst.ci - // Create new labels and set them. - labels := ci.Labels - if labels == nil { - labels = map[string]string{} - } - labels[labelMachineUID] = string(newUID) - req := &compute.InstancesSetLabelsRequest{ - Labels: labels, - LabelFingerprint: ci.LabelFingerprint, - } - op, err := svc.Instances.SetLabels(cfg.projectID, cfg.zone, inst.Name(), req).Do() - if err != nil { - return newError(common.InvalidConfigurationMachineError, errSetLabels, err) - } - err = svc.waitZoneOperation(ctx, cfg, op.Name) - if err != nil { - return newError(common.InvalidConfigurationMachineError, errSetLabels, err) - } - return nil -} - -// SetMetricsForMachines allows providers to provide provider-specific metrics. -func (p *Provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { - return nil -} - -// newError creates a terminal error matching to the provider interface. -func newError(reason common.MachineStatusError, msg string, args ...interface{}) error { - return cloudprovidererrors.TerminalError{ - Reason: reason, - Message: fmt.Sprintf(msg, args...), - } -} diff --git a/pkg/cloudprovider/provider/gce/provider_test.go b/pkg/cloudprovider/provider/gce/provider_test.go deleted file mode 100644 index cf3de6de7..000000000 --- a/pkg/cloudprovider/provider/gce/provider_test.go +++ /dev/null @@ -1,179 +0,0 @@ -/* -Copyright 2022 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gce - -import ( - "context" - "encoding/base64" - "encoding/json" - "os" - "strings" - "testing" - - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - - "k8s.io/apimachinery/pkg/runtime" - fake2 "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func testProviderSpec() map[string]interface{} { - return map[string]interface{}{ - "caPublicKey": "", - "cloudProvider": "gce", - "cloudProviderSpec": map[string]interface{}{ - "assignPublicIPAddress": true, - "customImage": "", - "diskSize": 25, - "diskType": "pd-standard", - "machineType": "e2-highcpu-2", - "multizone": false, - "network": "global/networks/default", - "preemptible": false, - "provisioningModel": "STANDARD", - "regional": false, - "serviceAccount": "", - "subnetwork": "", - "tags": []string{ - "kubernetes-cluster-kdlj8sn58d", - "system-cluster-kdlj8sn58d", - "system-project-sszxpzjcnm", - }, - "zone": "europe-west2-a", - "disableMachineServiceAccount": false, - "enableNestedVirtualization": true, - "minCPUPlatform": "Intel Haswell", - "guestOSFeatures": []string{ - "VIRTIO_SCSI_MULTIQUEUE", - "GVNIC", - }, - }, - "operatingSystem": "ubuntu", - "operatingSystemSpec": map[string]interface{}{ - "distUpgradeOnBoot": false, - }, - "sshPublicKeys": []string{}, - "network": map[string]interface{}{}, - } -} - -func testServiceAccount() string { - return base64.StdEncoding.EncodeToString([]byte(`{ - "type": "service_account", - "project_id": "test-dev", - "private_key_id": "testprivatekeyid", - "private_key": "-----BEGIN PRIVATE KEY-----\n\nMIICXQIBAAKBgQCU6DqY/hqXOOjomOuf3ESiMBxxRCpnn+VTAOPeEOsKaNdAv3zB\n\nmy6KEgOlraAi45cR8Ow8R0UFBNMQaU6Bck99t34BGZSQxjMTFw11W9p0GROKZgqG\n\nobj1WiomRQuwy0D6Q90wRSRhvnawKHqIEDoGnQT+SceV5vb6yoLmZSBoFQIDAQAB\n\nAoGASQoIBBdPz7E4fS7VFJqkh7F1ohE/g4iooagkHT7LK1X1j2rdtNF7aHohk9iw\n\nXayo40H7fi2vKyEMrlYZDeGWH1/XHLIyTGUNo91J3HDRbfs0eJhHKxFsdD/a64yV\n\ndYJviM2nsBQkbCC08O3yVCc/0spB7xKSBlpgFaWTnwDj8AECQQDnjen0Or9C7c9N\n\nOQdkefGoRzD0ltwbJoOHmRz3s49TieRmQpX+XkcbR91BPkIzgbQs8tFatK5YQNDp\n\nDTdp/VoVAkEApKCoEv6hNdj3sjY1qGT2e2sNCKbgsJeXfPrMbotypmv2VgK4w0IE\n\nPA+Tysd6G3EojFooDlzAkG2hXsgie2BWAQJBAN/finnSLsdD63CrGaWgbO+Y3REt\n\npmMtqm94rtQiLAnFwSjJagHEHxWWNqn0ysbHuW7X2WfMVuAG0rTwTUpRZD0CQQCQ\n\nhY0nJ6vkdrV0GIzgaMnNLPxDNSSZQms1x4JCJV8f5DVb6oXCvCi1hUNMR/PVNXDQ\n\nTbFOcnSGFggNCgrjXn4BAkAgoDpFUVa5wLvkWQpTnKXv//xMG4fS3xmlDHi3xE8d\n\nMfEPCgKd8giHPaW0p4XtTAmhk1sdpuR2op4ZfDorCmEC\n\n-----END PRIVATE KEY-----\n", - "client_email": "someguy@some.com", - "client_id": "whateverthisis", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/sometest" -} -`)) -} - -type testMap map[string]interface{} - -// with patches value of m at keypath with val e.g. keypath=x.y val=z then m[x][y] = z. -func (m testMap) with(keypath, val string) testMap { - parts := strings.Split(keypath, ".") - var curr interface{} = m - for _, p := range parts[:len(parts)-1] { - switch m[p].(type) { - case map[string]interface{}, testMap: - curr = m[p] - } - } - - switch x := curr.(type) { - case map[string]interface{}: //, testMap: - x[parts[len(parts)-1]] = val - case testMap: - x[parts[len(parts)-1]] = val - } - return m -} - -func TestValidate(t *testing.T) { - t.Setenv(envGoogleServiceAccount, testServiceAccount()) - defer os.Unsetenv(envGoogleServiceAccount) - - rawBytes := func(m map[string]interface{}) []byte { - data, err := json.Marshal(m) - if err != nil { - t.Fatal(err) - } - return data - } - - p := New(providerconfig.NewConfigVarResolver(context.Background(), fake2.NewClientBuilder().Build())) - tests := []struct { - name string - mspec v1alpha1.MachineSpec - expectErr bool - }{ - { - "without IP family", - v1alpha1.MachineSpec{ - ProviderSpec: v1alpha1.ProviderSpec{ - Value: &runtime.RawExtension{ - Raw: rawBytes(testProviderSpec()), - }, - }, - }, - false, - }, - { - "empty IP family", - v1alpha1.MachineSpec{ - ProviderSpec: v1alpha1.ProviderSpec{ - Value: &runtime.RawExtension{ - Raw: rawBytes(testMap(testProviderSpec()). - with("network.ipFamily", ""), - ), - }, - }, - }, - false, - }, - { - "with IP family", - v1alpha1.MachineSpec{ - ProviderSpec: v1alpha1.ProviderSpec{ - Value: &runtime.RawExtension{ - Raw: rawBytes(testMap(testProviderSpec()). - with("network.ipFamily", "IPv4+IPv6"), - ), - }, - }, - }, - false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - err := p.Validate(context.Background(), zap.NewNop().Sugar(), test.mspec) - if (err != nil) != test.expectErr { - t.Fatalf("expectedErr: %t, got: %v", test.expectErr, err) - } - }) - } -} diff --git a/pkg/cloudprovider/provider/gce/service.go b/pkg/cloudprovider/provider/gce/service.go deleted file mode 100644 index 4c1dd9570..000000000 --- a/pkg/cloudprovider/provider/gce/service.go +++ /dev/null @@ -1,169 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// Google Cloud Provider for the Machine Controller -// - -package gce - -import ( - "context" - "errors" - "fmt" - "time" - - "go.uber.org/zap" - "golang.org/x/oauth2" - "google.golang.org/api/compute/v1" - "google.golang.org/api/option" - - "k8s.io/apimachinery/pkg/util/wait" -) - -const ( - // Interval and timeout for polling. - pollInterval = 5 * time.Second - pollTimeout = 5 * time.Minute -) - -// Google compute operation status. -const ( - statusDone = "DONE" -) - -const ( - defaultNetwork = "global/networks/default" -) - -// service wraps a GCE compute service for the extension with helper methods. -type service struct { - *compute.Service -} - -// connectComputeService establishes a service connection to the Compute Engine. -func connectComputeService(ctx context.Context, cfg *config) (*service, error) { - if cfg.clientConfig != nil && - cfg.clientConfig.TokenSource != nil { - client := oauth2.NewClient(ctx, cfg.clientConfig.TokenSource) - svc, err := compute.NewService(ctx, option.WithHTTPClient(client)) - if err != nil { - return nil, fmt.Errorf("cannot connect to Google Cloud: %w", err) - } - return &service{svc}, nil - } - - return nil, errors.New("gcp token source was not found") -} - -// networkInterfaces returns the configured network interfaces for an instance creation. -func (svc *service) networkInterfaces(log *zap.SugaredLogger, cfg *config) ([]*compute.NetworkInterface, error) { - network := cfg.network - - if cfg.network == "" && cfg.subnetwork == "" { - network = defaultNetwork - } - - ifc := &compute.NetworkInterface{ - Network: network, - Subnetwork: cfg.subnetwork, - } - - log.Infow("Network configuration", "network", cfg.network, "subnetwork", cfg.subnetwork) - - if cfg.assignPublicIPAddress { - ifc.AccessConfigs = []*compute.AccessConfig{ - { - Name: "External NAT", - Type: "ONE_TO_ONE_NAT", - }, - } - } - - // Setup IPv6 - // GCP allocates public IPv6 addr so we only try to setup IPv6 - // if assigning public IP addresses is enabled. - if cfg.assignPublicIPAddress { - // GCP doesn't support IPv6 only stack - if cfg.providerConfig.Network.GetIPFamily().IsDualstack() { - ifc.StackType = "IPV4_IPV6" - - ifc.Ipv6AccessConfigs = []*compute.AccessConfig{ - { - Name: "external-ipv6", - NetworkTier: "PREMIUM", - Type: "DIRECT_IPV6", - }, - } - } else { - log.Infow("IP family doesn't specify dual stack", "family", cfg.providerConfig.Network.GetIPFamily()) - } - } - return []*compute.NetworkInterface{ifc}, nil -} - -// attachedDisks returns the configured attached disks for an instance creation. -func (svc *service) attachedDisks(cfg *config) ([]*compute.AttachedDisk, error) { - sourceImage, err := cfg.sourceImageDescriptor() - if err != nil { - return nil, err - } - bootDisk := &compute.AttachedDisk{ - Boot: true, - AutoDelete: true, - InitializeParams: &compute.AttachedDiskInitializeParams{ - DiskSizeGb: cfg.diskSize, - DiskType: cfg.diskTypeDescriptor(), - SourceImage: sourceImage, - }, - } - for _, v := range cfg.guestOSFeatures { - bootDisk.GuestOsFeatures = append(bootDisk.GuestOsFeatures, &compute.GuestOsFeature{ - Type: v, - }) - } - return []*compute.AttachedDisk{bootDisk}, nil -} - -// waitZoneOperation waits for a GCE operation in a zone to be completed or timed out. -func (svc *service) waitZoneOperation(ctx context.Context, cfg *config, opName string) error { - return svc.waitOperation(ctx, func() (*compute.Operation, error) { - return svc.ZoneOperations.Get(cfg.projectID, cfg.zone, opName).Do() - }) -} - -// waitOperation waits for a GCE operation to be completed or timed out. -func (svc *service) waitOperation(ctx context.Context, refreshOperation func() (*compute.Operation, error)) error { - var op *compute.Operation - var err error - - return wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, false, func(ctx context.Context) (bool, error) { - op, err = refreshOperation() - if err != nil { - return false, err - } - // Check if done (successfully). - if op.Status == statusDone { - if op.Error != nil { - // Operation failed. - return false, fmt.Errorf("GCE operation failed: %v", *op.Error.Errors[0]) - } - return true, nil - } - // Not yet done. - return false, nil - }) -} diff --git a/pkg/cloudprovider/provider/gce/types/cloudconfig.go b/pkg/cloudprovider/provider/gce/types/cloudconfig.go deleted file mode 100644 index 9c4201d4b..000000000 --- a/pkg/cloudprovider/provider/gce/types/cloudconfig.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// Google Cloud Provider for the Machine Controller -// - -package types - -import ( - "bytes" - "fmt" - "text/template" - - "github.com/Masterminds/sprig/v3" - - "github.com/kubermatic/machine-controller/pkg/ini" -) - -// cloudConfigTemplate renders the cloud-config in gcfg format. All -// fields are optional, that's why containing the ifs and the explicit newlines. -const cloudConfigTemplate = "[global]\n" + - "project-id = {{ .Global.ProjectID | iniEscape }}\n" + - "local-zone = {{ .Global.LocalZone | iniEscape }}\n" + - "network-name = {{ .Global.NetworkName | iniEscape }}\n" + - "subnetwork-name = {{ .Global.SubnetworkName | iniEscape }}\n" + - "token-url = {{ .Global.TokenURL | iniEscape }}\n" + - "multizone = {{ .Global.MultiZone }}\n" + - "regional = {{ .Global.Regional }}\n" + - "{{ range .Global.NodeTags }}node-tags = {{ . | iniEscape }}\n{{end}}" - -// GlobalOpts contains the values of the global section of the cloud configuration. -type GlobalOpts struct { - ProjectID string - LocalZone string - NetworkName string - SubnetworkName string - TokenURL string - MultiZone bool - Regional bool - NodeTags []string - RHSMOfflineToken string -} - -// CloudConfig contains only the section global. -type CloudConfig struct { - Global GlobalOpts -} - -// AsString renders the cloud configuration as string. -func (cc *CloudConfig) AsString() (string, error) { - funcMap := sprig.TxtFuncMap() - funcMap["iniEscape"] = ini.Escape - - tmpl, err := template.New("cloud-config").Funcs(funcMap).Parse(cloudConfigTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse the cloud config template: %w", err) - } - - buf := &bytes.Buffer{} - if err := tmpl.Execute(buf, cc); err != nil { - return "", fmt.Errorf("failed to execute cloud config template: %w", err) - } - - return buf.String(), nil -} diff --git a/pkg/cloudprovider/provider/gce/types/cloudconfig_test.go b/pkg/cloudprovider/provider/gce/types/cloudconfig_test.go deleted file mode 100644 index 6b91cefd5..000000000 --- a/pkg/cloudprovider/provider/gce/types/cloudconfig_test.go +++ /dev/null @@ -1,73 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// Google Cloud Provider for the Machine Controller -// -// Unit Tests -// - -package types - -import ( - "testing" -) - -func TestCloudConfigAsString(t *testing.T) { - tests := []struct { - name string - config *CloudConfig - contents string - }{ - { - name: "minimum test", - config: &CloudConfig{ - Global: GlobalOpts{ - ProjectID: "my-project-id", - LocalZone: "my-zone", - NetworkName: "my-cool-network", - SubnetworkName: "my-cool-subnetwork", - TokenURL: "nil", - MultiZone: true, - Regional: true, - NodeTags: []string{"tag1", "tag2"}, - }, - }, - contents: "[global]\n" + - "project-id = \"my-project-id\"\n" + - "local-zone = \"my-zone\"\n" + - "network-name = \"my-cool-network\"\n" + - "subnetwork-name = \"my-cool-subnetwork\"\n" + - "token-url = \"nil\"\n" + - "multizone = true\n" + - "regional = true\n" + - "node-tags = \"tag1\"\n" + - "node-tags = \"tag2\"\n", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - s, err := test.config.AsString() - if err != nil { - t.Fatalf("failed to convert to string: %v", err) - } - if s != test.contents { - t.Fatalf("output is not as expected: %s", s) - } - }) - } -} diff --git a/pkg/cloudprovider/provider/gce/types/types.go b/pkg/cloudprovider/provider/gce/types/types.go deleted file mode 100644 index 027156367..000000000 --- a/pkg/cloudprovider/provider/gce/types/types.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "encoding/json" - "fmt" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - - "k8s.io/apimachinery/pkg/runtime" -) - -// CloudProviderSpec contains the specification of the cloud provider taken -// from the provider configuration. -type CloudProviderSpec struct { - // ServiceAccount must be base64-encoded. - ServiceAccount *providerconfigtypes.ConfigVarString `json:"serviceAccount,omitempty"` - Zone *providerconfigtypes.ConfigVarString `json:"zone"` - MachineType *providerconfigtypes.ConfigVarString `json:"machineType"` - DiskSize int64 `json:"diskSize"` - DiskType *providerconfigtypes.ConfigVarString `json:"diskType"` - Network *providerconfigtypes.ConfigVarString `json:"network"` - Subnetwork *providerconfigtypes.ConfigVarString `json:"subnetwork"` - Preemptible *providerconfigtypes.ConfigVarBool `json:"preemptible"` - AutomaticRestart *providerconfigtypes.ConfigVarBool `json:"automaticRestart,omitempty"` - ProvisioningModel *providerconfigtypes.ConfigVarString `json:"provisioningModel,omitempty"` - Labels map[string]string `json:"labels,omitempty"` - Tags []string `json:"tags,omitempty"` - AssignPublicIPAddress *providerconfigtypes.ConfigVarBool `json:"assignPublicIPAddress,omitempty"` - MultiZone *providerconfigtypes.ConfigVarBool `json:"multizone"` - Regional *providerconfigtypes.ConfigVarBool `json:"regional"` - CustomImage *providerconfigtypes.ConfigVarString `json:"customImage,omitempty"` - DisableMachineServiceAccount *providerconfigtypes.ConfigVarBool `json:"disableMachineServiceAccount,omitempty"` - EnableNestedVirtualization *providerconfigtypes.ConfigVarBool `json:"enableNestedVirtualization,omitempty"` - MinCPUPlatform *providerconfigtypes.ConfigVarString `json:"minCPUPlatform,omitempty"` - GuestOSFeatures []string `json:"guestOSFeatures,omitempty"` - ProjectID providerconfigtypes.ConfigVarString `json:"projectID,omitempty"` -} - -// UpdateProviderSpec updates the given provider spec with changed -// configuration values. -func (cpSpec *CloudProviderSpec) UpdateProviderSpec(spec v1alpha1.ProviderSpec) (*runtime.RawExtension, error) { - if spec.Value == nil { - return nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - providerConfig := providerconfigtypes.Config{} - err := json.Unmarshal(spec.Value.Raw, &providerConfig) - if err != nil { - return nil, err - } - rawCPSpec, err := json.Marshal(cpSpec) - if err != nil { - return nil, err - } - providerConfig.CloudProviderSpec = runtime.RawExtension{Raw: rawCPSpec} - rawProviderConfig, err := json.Marshal(providerConfig) - if err != nil { - return nil, err - } - return &runtime.RawExtension{Raw: rawProviderConfig}, nil -} - -type RawConfig = CloudProviderSpec - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go deleted file mode 100644 index ab893737f..000000000 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ /dev/null @@ -1,646 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package hetzner - -import ( - "context" - "errors" - "fmt" - "net/http" - "strconv" - "strings" - - "github.com/hetznercloud/hcloud-go/hcloud" - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/common/ssh" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - hetznertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/hetzner/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/rand" -) - -const ( - machineUIDLabelKey = "machine-uid" -) - -type provider struct { - configVarResolver *providerconfig.ConfigPointerVarResolver -} - -// New returns a Hetzner provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return &provider{configVarResolver: &providerconfig.ConfigPointerVarResolver{Cvr: configVarResolver}} -} - -type Config struct { - Token string - ServerType string - Datacenter string - Image string - Location string - PlacementGroupPrefix string - Networks []string - Firewalls []string - Labels map[string]string - AssignIPv4 bool - AssignIPv6 bool -} - -func getNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { - switch os { - case providerconfigtypes.OperatingSystemUbuntu: - return "ubuntu-22.04", nil - case providerconfigtypes.OperatingSystemCentOS: - return "centos-7", nil - case providerconfigtypes.OperatingSystemRockyLinux: - return "rocky-8", nil - } - return "", providerconfigtypes.ErrOSNotSupported -} - -func getClient(token string) *hcloud.Client { - return hcloud.NewClient(hcloud.WithToken(token)) -} - -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - - pconfig, err := providerconfigtypes.GetConfig(provSpec) - if err != nil { - return nil, nil, err - } - - if pconfig.OperatingSystemSpec.Raw == nil { - return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") - } - - rawConfig, err := hetznertypes.GetConfig(*pconfig) - if err != nil { - return nil, nil, err - } - - c := Config{} - c.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Token, "HZ_TOKEN") - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"token\" field, error = %w", err) - } - - c.ServerType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ServerType) - if err != nil { - return nil, nil, err - } - - c.Datacenter, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Datacenter) - if err != nil { - return nil, nil, err - } - - c.Image, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Image) - if err != nil { - return nil, nil, err - } - - c.Location, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Location) - if err != nil { - return nil, nil, err - } - - c.PlacementGroupPrefix, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.PlacementGroupPrefix) - if err != nil { - return nil, nil, err - } - - for _, network := range rawConfig.Networks { - networkValue, err := p.configVarResolver.GetConfigVarStringValue(&network) - if err != nil { - return nil, nil, err - } - c.Networks = append(c.Networks, networkValue) - } - - for _, firewall := range rawConfig.Firewalls { - firewallValue, err := p.configVarResolver.GetConfigVarStringValue(&firewall) - if err != nil { - return nil, nil, err - } - c.Firewalls = append(c.Firewalls, firewallValue) - } - - ipv4, ipv6, err := p.publicIPsAssignment(rawConfig) - if err != nil { - return nil, nil, err - } - - c.AssignIPv4 = ipv4 - c.AssignIPv6 = ipv6 - - c.Labels = rawConfig.Labels - - return &c, pconfig, err -} - -func (p *provider) getServerPlacementGroup(ctx context.Context, client *hcloud.Client, c *Config) (*hcloud.PlacementGroup, error) { - placementGroups, _, err := client.PlacementGroup.List(ctx, hcloud.PlacementGroupListOpts{Type: hcloud.PlacementGroupTypeSpread}) - if err != nil { - return nil, hzErrorToTerminalError(err, "failed to get placement groups of type spread") - } - for _, pg := range placementGroups { - if !strings.HasPrefix(pg.Name, c.PlacementGroupPrefix) { - continue - } - if len(pg.Servers) < 10 { - return pg, nil - } - } - pgLabels := map[string]string{} - for k, v := range c.Labels { - if k != machineUIDLabelKey { - pgLabels[k] = v - } - } - createdPg, _, err := client.PlacementGroup.Create(ctx, hcloud.PlacementGroupCreateOpts{ - Name: fmt.Sprintf("%s-%s", c.PlacementGroupPrefix, rand.SafeEncodeString(rand.String(5))), - Labels: pgLabels, - Type: hcloud.PlacementGroupTypeSpread, - }) - if err != nil { - return nil, hzErrorToTerminalError(err, "failed to create placement group") - } - return createdPg.PlacementGroup, nil -} - -func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { - c, pc, err := p.getConfig(spec.ProviderSpec) - if err != nil { - return fmt.Errorf("failed to parse config: %w", err) - } - - if c.Token == "" { - return errors.New("token is missing") - } - - _, err = getNameForOS(pc.OperatingSystem) - if err != nil { - return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, err) - } - - client := getClient(c.Token) - - if c.Location != "" && c.Datacenter != "" { - return fmt.Errorf("location and datacenter must not be set at the same time") - } - - if c.Location != "" { - if _, _, err = client.Location.Get(ctx, c.Location); err != nil { - return fmt.Errorf("failed to get location: %w", err) - } - } - - if c.Datacenter != "" { - if _, _, err = client.Datacenter.Get(ctx, c.Datacenter); err != nil { - return fmt.Errorf("failed to get datacenter: %w", err) - } - } - - if c.Image != "" { - if _, _, err = client.Image.Get(ctx, c.Image); err != nil { - return fmt.Errorf("failed to get image: %w", err) - } - } - - for _, network := range c.Networks { - if _, _, err = client.Network.Get(ctx, network); err != nil { - return fmt.Errorf("failed to get network %q: %w", network, err) - } - } - - for _, firewall := range c.Firewalls { - f, _, err := client.Firewall.Get(ctx, firewall) - if err != nil { - return fmt.Errorf("failed to get firewall %q: %w", firewall, err) - } - if f == nil { - return fmt.Errorf("firewall %q does not exist", firewall) - } - } - - if !c.AssignIPv4 && !c.AssignIPv6 && len(c.Networks) < 1 { - return errors.New("server should have either a public ipv4, ipv6 or dedicated network") - } - - if _, _, err = client.ServerType.Get(ctx, c.ServerType); err != nil { - return fmt.Errorf("failed to get server type: %w", err) - } - - return nil -} - -func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { - c, pc, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - - client := getClient(c.Token) - - if c.Image == "" { - imageName, err := getNameForOS(pc.OperatingSystem) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Invalid operating system specified %q, details = %v", pc.OperatingSystem, err), - } - } - c.Image = imageName - } - - if c.Labels == nil { - c.Labels = map[string]string{} - } - - c.Labels[machineUIDLabelKey] = string(machine.UID) - - serverCreateOpts := hcloud.ServerCreateOpts{ - Name: machine.Spec.Name, - UserData: userdata, - Labels: c.Labels, - PublicNet: &hcloud.ServerCreatePublicNet{ - EnableIPv4: c.AssignIPv4, - EnableIPv6: c.AssignIPv6, - }, - } - - if c.Datacenter != "" { - dc, _, err := client.Datacenter.Get(ctx, c.Datacenter) - if err != nil { - return nil, hzErrorToTerminalError(err, "failed to get datacenter") - } - if dc == nil { - return nil, fmt.Errorf("datacenter %q does not exist", c.Datacenter) - } - serverCreateOpts.Datacenter = dc - } - - if c.Location != "" { - location, _, err := client.Location.Get(ctx, c.Location) - if err != nil { - return nil, hzErrorToTerminalError(err, "failed to get location") - } - if location == nil { - return nil, fmt.Errorf("location %q does not exist", c.Location) - } - serverCreateOpts.Location = location - } - - if c.PlacementGroupPrefix != "" { - selectedPg, err := p.getServerPlacementGroup(ctx, client, c) - if err != nil { - return nil, err - } - serverCreateOpts.PlacementGroup = selectedPg - } - - for _, network := range c.Networks { - n, _, err := client.Network.Get(ctx, network) - if err != nil { - return nil, hzErrorToTerminalError(err, "failed to get network") - } - if n == nil { - return nil, fmt.Errorf("network %q does not exist", network) - } - serverCreateOpts.Networks = append(serverCreateOpts.Networks, n) - } - - for _, firewall := range c.Firewalls { - n, _, err := client.Firewall.Get(ctx, firewall) - if err != nil { - return nil, hzErrorToTerminalError(err, "failed to get firewall") - } - if n == nil { - return nil, fmt.Errorf("firewall %q does not exist", firewall) - } - serverCreateOpts.Firewalls = append(serverCreateOpts.Firewalls, &hcloud.ServerCreateFirewall{Firewall: *n}) - } - - image, _, err := client.Image.Get(ctx, c.Image) - if err != nil { - return nil, hzErrorToTerminalError(err, "failed to get image") - } - if image == nil { - return nil, fmt.Errorf("image %q does not exist", c.Image) - } - serverCreateOpts.Image = image - - serverType, _, err := client.ServerType.Get(ctx, c.ServerType) - if err != nil { - return nil, hzErrorToTerminalError(err, "failed to get server type") - } - if serverType == nil { - return nil, fmt.Errorf("server type %q does not exist", c.ServerType) - } - serverCreateOpts.ServerType = serverType - - // We generate a temporary SSH key here, because otherwise Hetzner creates - // a password and sends it via E-Mail to the account owner, which can be quite - // spammy. No one will ever get access to the private key. - sshkey, err := ssh.NewKey() - if err != nil { - return nil, fmt.Errorf("failed to generate ssh key: %w", err) - } - - hkey, res, err := client.SSHKey.Create(ctx, hcloud.SSHKeyCreateOpts{ - Name: sshkey.Name, - PublicKey: sshkey.PublicKey, - }) - if err != nil { - return nil, fmt.Errorf("creating temporary ssh key failed with error %w", err) - } - if res.StatusCode != http.StatusCreated { - return nil, fmt.Errorf("got invalid http status code when creating ssh key: expected=%d, god=%d", http.StatusCreated, res.StatusCode) - } - defer func() { - _, err := client.SSHKey.Delete(ctx, hkey) - if err != nil { - log.Errorw("Failed to delete temporary ssh key", zap.Error(err)) - } - }() - serverCreateOpts.SSHKeys = []*hcloud.SSHKey{hkey} - - serverCreateRes, res, err := client.Server.Create(ctx, serverCreateOpts) - if err != nil { - return nil, hzErrorToTerminalError(err, "failed to create server") - } - if res.StatusCode != http.StatusCreated { - return nil, fmt.Errorf("failed to create server invalid status code returned. expected=%d got %d", http.StatusCreated, res.StatusCode) - } - - return &hetznerServer{server: serverCreateRes.Server}, nil -} - -func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { - instance, err := p.Get(ctx, log, machine, data) - if err != nil { - if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { - return true, nil - } - return false, err - } - - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return false, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - - client := getClient(c.Token) - hzServer := instance.(*hetznerServer).server - - _, res, err := client.Server.DeleteWithResult(ctx, hzServer) - if err != nil { - return false, hzErrorToTerminalError(err, "failed to delete the server") - } - if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusNotFound { - return false, fmt.Errorf("invalid status code returned. expected=%d got=%d", http.StatusOK, res.StatusCode) - } - - if hzServer.PlacementGroup != nil { - pgHzServer, _, err := client.PlacementGroup.Get(ctx, hzServer.PlacementGroup.Name) - if err != nil { - return false, hzErrorToTerminalError(err, "failed to get placement group") - } - count := 0 - for _, s := range pgHzServer.Servers { - if s != hzServer.ID { - count++ - } - } - if count == 0 { - _, err := client.PlacementGroup.Delete(ctx, pgHzServer) - if err != nil { - return false, hzErrorToTerminalError(err, "failed to delete empty placement group") - } - } - } - - return false, nil -} - -func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { - return spec, nil -} - -func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - - client := getClient(c.Token) - - servers, _, err := client.Server.List(ctx, hcloud.ServerListOpts{ListOpts: hcloud.ListOpts{ - LabelSelector: machineUIDLabelKey + "==" + string(machine.UID), - }}) - if err != nil { - return nil, hzErrorToTerminalError(err, "failed to list servers") - } - - for _, server := range servers { - if server.Labels[machineUIDLabelKey] == string(machine.UID) { - return &hetznerServer{server: server}, nil - } - } - - return nil, cloudprovidererrors.ErrInstanceNotFound -} - -func (p *provider) MigrateUID(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - client := getClient(c.Token) - - // We didn't use the UID for Hetzner before - server, _, err := client.Server.Get(ctx, machine.Spec.Name) - if err != nil { - return fmt.Errorf("failed to get server: %w", err) - } - if server == nil { - log.Info("No instance exists for machine") - return nil - } - - log.Info("Setting UID label for machine") - _, response, err := client.Server.Update(ctx, server, hcloud.ServerUpdateOpts{ - Labels: map[string]string{machineUIDLabelKey: string(newUID)}, - }) - if err != nil { - return fmt.Errorf("failed to update UID label: %w", err) - } - if response.Response.StatusCode != http.StatusOK { - return fmt.Errorf("got unexpected response code %v, expected %v", response.Response.Status, http.StatusOK) - } - // This succeeds, but does not result in a label on the server, seems to be a bug - // on Hetzner side - log.Info("Successfully set UID label for machine") - - return nil -} - -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return "", "", nil -} - -func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { - labels := make(map[string]string) - - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err == nil { - labels["size"] = c.ServerType - labels["dc"] = c.Datacenter - labels["location"] = c.Location - } - - return labels, err -} - -type hetznerServer struct { - server *hcloud.Server -} - -func (s *hetznerServer) Name() string { - return s.server.Name -} - -func (s *hetznerServer) ID() string { - return strconv.Itoa(s.server.ID) -} - -func (s *hetznerServer) ProviderID() string { - if s.server == nil || s.server.ID == 0 { - return "" - } - return fmt.Sprintf("hcloud://%d", s.server.ID) -} - -func (s *hetznerServer) HostID() string { - return "" -} - -func (s *hetznerServer) Addresses() map[string]v1.NodeAddressType { - addresses := map[string]v1.NodeAddressType{} - for _, fips := range s.server.PublicNet.FloatingIPs { - addresses[fips.IP.String()] = v1.NodeExternalIP - } - for _, privateNetwork := range s.server.PrivateNet { - addresses[privateNetwork.IP.String()] = v1.NodeInternalIP - } - addresses[s.server.PublicNet.IPv4.IP.String()] = v1.NodeExternalIP - // For a given IPv6 network of 2001:db8:1234::/64, the instance address is 2001:db8:1234::1 - // Reference: https://github.com/hetznercloud/hcloud-cloud-controller-manager/blob/v1.12.1/hcloud/instances.go#L165-167 - if s.server.PublicNet.IPv6.IP != nil && !s.server.PublicNet.IPv6.IP.IsUnspecified() { - s.server.PublicNet.IPv6.IP[len(s.server.PublicNet.IPv6.IP)-1] |= 0x01 - addresses[s.server.PublicNet.IPv6.IP.String()] = v1.NodeExternalIP - } - return addresses -} - -func (s *hetznerServer) Status() instance.Status { - switch s.server.Status { - case hcloud.ServerStatusInitializing: - return instance.StatusCreating - case hcloud.ServerStatusRunning: - return instance.StatusRunning - default: - return instance.StatusUnknown - } -} - -// hzErrorToTerminalError judges if the given error -// can be qualified as a "terminal" error, for more info see v1alpha1.MachineStatus -// -// if the given error doesn't qualify the error passed as an argument will be returned. -func hzErrorToTerminalError(err error, msg string) error { - prepareAndReturnError := func() error { - return fmt.Errorf("%s, due to %w", msg, err) - } - - if err != nil { - if hcloud.IsError(err, hcloud.ErrorCode("unauthorized")) { - // authorization primitives come from MachineSpec - // thus we are setting InvalidConfigurationMachineError - return cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: "A request has been rejected due to invalid credentials which were taken from the MachineSpec", - } - } - - return prepareAndReturnError() - } - - return err -} - -func (p *provider) publicIPsAssignment(rawConfig *hetznertypes.RawConfig) (bool, bool, error) { - assignIPv4, ipv4Set, err := p.configVarResolver.GetConfigVarBoolValue(&rawConfig.AssignPublicIPv4) - if err != nil { - return false, false, err - } - - assignIPv6, ipv6Set, err := p.configVarResolver.GetConfigVarBoolValue(&rawConfig.AssignPublicIPv6) - if err != nil { - return false, false, err - } - - // hetzner default behaviour assigns public ips when users don't set them explicitly for the server. In order to - // retain this behaviour, if the field AssignPublicIPv4/AssignPublicIPv6 in MachineDeployment is not set, machine controller - // default them to true. - if !ipv4Set { - assignIPv4 = true - } - if !ipv6Set { - assignIPv6 = true - } - - return assignIPv4, assignIPv6, nil -} - -func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { - return nil -} diff --git a/pkg/cloudprovider/provider/hetzner/types/types.go b/pkg/cloudprovider/provider/hetzner/types/types.go deleted file mode 100644 index ce1cbacb5..000000000 --- a/pkg/cloudprovider/provider/hetzner/types/types.go +++ /dev/null @@ -1,42 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" -) - -type RawConfig struct { - Token *providerconfigtypes.ConfigVarString `json:"token,omitempty"` - ServerType *providerconfigtypes.ConfigVarString `json:"serverType"` - Datacenter *providerconfigtypes.ConfigVarString `json:"datacenter"` - Image *providerconfigtypes.ConfigVarString `json:"image"` - Location *providerconfigtypes.ConfigVarString `json:"location"` - PlacementGroupPrefix *providerconfigtypes.ConfigVarString `json:"placementGroupPrefix"` - Networks []providerconfigtypes.ConfigVarString `json:"networks"` - Firewalls []providerconfigtypes.ConfigVarString `json:"firewalls"` - Labels map[string]string `json:"labels,omitempty"` - AssignPublicIPv4 providerconfigtypes.ConfigVarBool `json:"assignPublicIPv4,omitempty"` - AssignPublicIPv6 providerconfigtypes.ConfigVarBool `json:"assignPublicIPv6,omitempty"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go deleted file mode 100644 index 27da10688..000000000 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ /dev/null @@ -1,961 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubevirt - -import ( - "context" - "encoding/base64" - "errors" - "fmt" - "net/url" - "os" - "strconv" - "strings" - "time" - - "go.uber.org/zap" - kubevirtv1 "kubevirt.io/api/core/v1" - cdiv1beta1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - kubevirttypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/kubevirt/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - netutil "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - controllerutil "github.com/kubermatic/machine-controller/pkg/controller/util" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - - corev1 "k8s.io/api/core/v1" - kerrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -func init() { - if err := kubevirtv1.AddToScheme(scheme.Scheme); err != nil { - panic(fmt.Sprintf("failed to add kubevirtv1 to scheme: %v", err)) - } - if err := cdiv1beta1.AddToScheme(scheme.Scheme); err != nil { - panic(fmt.Sprintf("failed to add cdiv1beta1 to scheme: %v", err)) - } -} - -type imageSource string - -const ( - // topologyKeyHostname defines the topology key for the node hostname. - topologyKeyHostname = "kubernetes.io/hostname" - // machineDeploymentLabelKey defines the label key used to contains as value the MachineDeployment name - // which machine comes from. - machineDeploymentLabelKey = "md" - // httpSource defines the http source type for VM Disk Image. - httpSource imageSource = "http" - // registrySource defines the OCI registry source type for VM Disk Image. - registrySource imageSource = "registry" - // pvcSource defines the pvc source type for VM Disk Image. - pvcSource imageSource = "pvc" -) - -type provider struct { - configVarResolver *providerconfig.ConfigPointerVarResolver -} - -// New returns a Kubevirt provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return &provider{configVarResolver: &providerconfig.ConfigPointerVarResolver{Cvr: configVarResolver}} -} - -type Config struct { - Kubeconfig string - ClusterName string - RestConfig *rest.Config - DNSConfig *corev1.PodDNSConfig - DNSPolicy corev1.DNSPolicy - CPUs string - Memory string - Namespace string - OSImageSource *cdiv1beta1.DataVolumeSource - StorageClassName string - StorageAccessType corev1.PersistentVolumeAccessMode - PVCSize resource.Quantity - Instancetype *kubevirtv1.InstancetypeMatcher - Preference *kubevirtv1.PreferenceMatcher - SecondaryDisks []SecondaryDisks - NodeAffinityPreset NodeAffinityPreset - TopologySpreadConstraints []corev1.TopologySpreadConstraint -} - -type AffinityType string - -const ( - // Facade for podAffinity, podAntiAffinity, nodeAffinity, nodeAntiAffinity - // HardAffinityType: affinity will include requiredDuringSchedulingIgnoredDuringExecution. - hardAffinityType = "hard" - // SoftAffinityType: affinity will include preferredDuringSchedulingIgnoredDuringExecution. - softAffinityType = "soft" - // NoAffinityType: affinity section will not be preset. - noAffinityType = "" -) - -func (p *provider) affinityType(affinityType *providerconfigtypes.ConfigVarString) (AffinityType, error) { - podAffinityPresetString, err := p.configVarResolver.GetConfigVarStringValue(affinityType) - if err != nil { - return "", fmt.Errorf(`failed to parse "podAffinityPreset" field: %w`, err) - } - switch strings.ToLower(podAffinityPresetString) { - case string(hardAffinityType): - return hardAffinityType, nil - case string(softAffinityType): - return softAffinityType, nil - case string(noAffinityType): - return noAffinityType, nil - } - - return "", fmt.Errorf("unknown affinityType: %s", affinityType) -} - -// NodeAffinityPreset. -type NodeAffinityPreset struct { - Type AffinityType - Key string - Values []string -} - -type SecondaryDisks struct { - Name string - Size resource.Quantity - StorageClassName string - StorageAccessType corev1.PersistentVolumeAccessMode -} - -type kubeVirtServer struct { - vmi kubevirtv1.VirtualMachineInstance -} - -func (k *kubeVirtServer) Name() string { - return k.vmi.Name -} - -func (k *kubeVirtServer) ID() string { - return string(k.vmi.UID) -} - -func (k *kubeVirtServer) ProviderID() string { - if k.vmi.Name == "" { - return "" - } - return "kubevirt://" + k.vmi.Name -} - -func (k *kubeVirtServer) HostID() string { - return "" -} - -func (k *kubeVirtServer) Addresses() map[string]corev1.NodeAddressType { - addresses := map[string]corev1.NodeAddressType{} - for _, kvInterface := range k.vmi.Status.Interfaces { - if address := strings.Split(kvInterface.IP, "/")[0]; address != "" { - addresses[address] = corev1.NodeInternalIP - } - } - return addresses -} - -func (k *kubeVirtServer) Status() instance.Status { - if k.vmi.Status.Phase == kubevirtv1.Running { - return instance.StatusRunning - } - return instance.StatusUnknown -} - -var _ instance.Instance = &kubeVirtServer{} - -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - - pconfig, err := providerconfigtypes.GetConfig(provSpec) - if err != nil { - return nil, nil, err - } - - if pconfig.OperatingSystemSpec.Raw == nil { - return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") - } - - rawConfig, err := kubevirttypes.GetConfig(*pconfig) - if err != nil { - return nil, nil, err - } - - config := Config{} - - // Kubeconfig was specified directly in the Machine/MachineDeployment CR. In this case we need to ensure that the value is base64 encoded. - if rawConfig.Auth.Kubeconfig.Value != "" { - val, err := base64.StdEncoding.DecodeString(rawConfig.Auth.Kubeconfig.Value) - if err != nil { - // An error here means that this is not a valid base64 string - // We can be more explicit here with the error for visibility. Webhook will return this error if we hit this scenario. - return nil, nil, fmt.Errorf("failed to decode base64 encoded kubeconfig. Expected value is a base64 encoded Kubeconfig in JSON or YAML format: %w", err) - } - config.Kubeconfig = string(val) - } else { - // Environment variable or secret reference was used for providing the value of kubeconfig - // We have to be lenient in this case and allow unencoded values as well. - config.Kubeconfig, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Auth.Kubeconfig, "KUBEVIRT_KUBECONFIG") - if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of "kubeconfig" field: %w`, err) - } - val, err := base64.StdEncoding.DecodeString(config.Kubeconfig) - // We intentionally ignore errors here with an assumption that an unencoded YAML or JSON must have been passed on - // in this case. - if err == nil { - config.Kubeconfig = string(val) - } - } - - config.ClusterName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ClusterName) - if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of "clusterName" field: %w`, err) - } - - config.RestConfig, err = clientcmd.RESTConfigFromKubeConfig([]byte(config.Kubeconfig)) - if err != nil { - return nil, nil, fmt.Errorf("failed to decode kubeconfig: %w", err) - } - - config.CPUs, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.CPUs) - if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of "cpus" field: %w`, err) - } - config.Memory, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.Memory) - if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of "memory" field: %w`, err) - } - config.Namespace = getNamespace() - - config.OSImageSource, err = p.parseOSImageSource(rawConfig.VirtualMachine.Template.PrimaryDisk, config.Namespace) - if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of "osImageSource" field: %w`, err) - } - - pvcSize, err := p.configVarResolver.GetConfigVarStringValue(&rawConfig.VirtualMachine.Template.PrimaryDisk.Size) - if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of "pvcSize" field: %w`, err) - } - if config.PVCSize, err = resource.ParseQuantity(pvcSize); err != nil { - return nil, nil, fmt.Errorf(`failed to parse value of "pvcSize" field: %w`, err) - } - config.StorageClassName, err = p.configVarResolver.GetConfigVarStringValue(&rawConfig.VirtualMachine.Template.PrimaryDisk.StorageClassName) - if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of "storageClassName" field: %w`, err) - } - - // Instancetype and Preference - config.Instancetype = rawConfig.VirtualMachine.Instancetype - config.Preference = rawConfig.VirtualMachine.Preference - - dnsPolicyString, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.DNSPolicy) - if err != nil { - return nil, nil, fmt.Errorf(`failed to parse "dnsPolicy" field: %w`, err) - } - if dnsPolicyString != "" { - config.DNSPolicy, err = dnsPolicy(dnsPolicyString) - if err != nil { - return nil, nil, fmt.Errorf("failed to get dns policy: %w", err) - } - } - if rawConfig.VirtualMachine.DNSConfig != nil { - config.DNSConfig = rawConfig.VirtualMachine.DNSConfig - } - config.SecondaryDisks = make([]SecondaryDisks, 0, len(rawConfig.VirtualMachine.Template.SecondaryDisks)) - for i, sd := range rawConfig.VirtualMachine.Template.SecondaryDisks { - sdSizeString, err := p.configVarResolver.GetConfigVarStringValue(&sd.Size) - if err != nil { - return nil, nil, fmt.Errorf(`failed to parse "secondaryDisks.size" field: %w`, err) - } - pvc, err := resource.ParseQuantity(sdSizeString) - if err != nil { - return nil, nil, fmt.Errorf(`failed to parse value of "secondaryDisks.size" field: %w`, err) - } - - scString, err := p.configVarResolver.GetConfigVarStringValue(&sd.StorageClassName) - if err != nil { - return nil, nil, fmt.Errorf(`failed to parse value of "secondaryDisks.storageClass" field: %w`, err) - } - config.SecondaryDisks = append(config.SecondaryDisks, SecondaryDisks{ - Name: fmt.Sprintf("secondarydisk%d", i), - Size: pvc, - StorageClassName: scString, - StorageAccessType: p.getStorageAccessType(sd.StorageAccessType), - }) - } - config.StorageAccessType = p.getStorageAccessType(rawConfig.VirtualMachine.Template.PrimaryDisk.StorageAccessType) - - config.NodeAffinityPreset, err = p.parseNodeAffinityPreset(rawConfig.Affinity.NodeAffinityPreset) - if err != nil { - return nil, nil, fmt.Errorf(`failed to parse "nodeAffinityPreset" field: %w`, err) - } - config.TopologySpreadConstraints, err = p.parseTopologySpreadConstraint(rawConfig.TopologySpreadConstraints) - if err != nil { - return nil, nil, fmt.Errorf(`failed to parse "topologySpreadConstraints" field: %w`, err) - } - - return &config, pconfig, nil -} - -func (p *provider) getStorageAccessType(accessType providerconfigtypes.ConfigVarString) corev1.PersistentVolumeAccessMode { - at, _ := p.configVarResolver.GetConfigVarStringValue(&accessType) - if at == "" { - return corev1.ReadWriteOnce - } - return corev1.PersistentVolumeAccessMode(at) -} - -func (p *provider) parseNodeAffinityPreset(nodeAffinityPreset kubevirttypes.NodeAffinityPreset) (NodeAffinityPreset, error) { - nodeAffinity := NodeAffinityPreset{} - var err error - nodeAffinity.Type, err = p.affinityType(nodeAffinityPreset.Type) - if err != nil { - return nodeAffinity, fmt.Errorf(`failed to parse "nodeAffinity.type" field: %w`, err) - } - nodeAffinity.Key, err = p.configVarResolver.GetConfigVarStringValue(nodeAffinityPreset.Key) - if err != nil { - return nodeAffinity, fmt.Errorf(`failed to parse "nodeAffinity.key" field: %w`, err) - } - nodeAffinity.Values = make([]string, 0, len(nodeAffinityPreset.Values)) - for _, v := range nodeAffinityPreset.Values { - valueString, err := p.configVarResolver.GetConfigVarStringValue(&v) - if err != nil { - return nodeAffinity, fmt.Errorf(`failed to parse "nodeAffinity.value" field: %w`, err) - } - nodeAffinity.Values = append(nodeAffinity.Values, valueString) - } - return nodeAffinity, nil -} - -func (p *provider) parseTopologySpreadConstraint(topologyConstraints []kubevirttypes.TopologySpreadConstraint) ([]corev1.TopologySpreadConstraint, error) { - parsedTopologyConstraints := make([]corev1.TopologySpreadConstraint, 0, len(topologyConstraints)) - for _, constraint := range topologyConstraints { - maxSkewString, err := p.configVarResolver.GetConfigVarStringValue(constraint.MaxSkew) - if err != nil { - return nil, fmt.Errorf(`failed to parse "topologySpreadConstraint.maxSkew" field: %w`, err) - } - maxSkew, err := strconv.ParseInt(maxSkewString, 10, 32) - if err != nil { - return nil, fmt.Errorf(`failed to parse "topologySpreadConstraint.maxSkew" field: %w`, err) - } - topologyKey, err := p.configVarResolver.GetConfigVarStringValue(constraint.TopologyKey) - if err != nil { - return nil, fmt.Errorf(`failed to parse "topologySpreadConstraint.topologyKey" field: %w`, err) - } - whenUnsatisfiable, err := p.configVarResolver.GetConfigVarStringValue(constraint.WhenUnsatisfiable) - if err != nil { - return nil, fmt.Errorf(`failed to parse "topologySpreadConstraint.whenUnsatisfiable" field: %w`, err) - } - parsedTopologyConstraints = append(parsedTopologyConstraints, corev1.TopologySpreadConstraint{ - MaxSkew: int32(maxSkew), - TopologyKey: topologyKey, - WhenUnsatisfiable: corev1.UnsatisfiableConstraintAction(whenUnsatisfiable), - }) - } - return parsedTopologyConstraints, nil -} - -func (p *provider) parseOSImageSource(primaryDisk kubevirttypes.PrimaryDisk, namespace string) (*cdiv1beta1.DataVolumeSource, error) { - osImage, err := p.configVarResolver.GetConfigVarStringValue(primaryDisk.OsImage) - if err != nil { - return nil, fmt.Errorf(`failed to get value of "primaryDisk.osImage" field: %w`, err) - } - osImageSource, err := p.configVarResolver.GetConfigVarStringValue(&primaryDisk.Source) - if err != nil { - return nil, fmt.Errorf(`failed to get value of "primaryDisk.source" field: %w`, err) - } - pullMethod, err := p.getPullMethod(primaryDisk.PullMethod) - if err != nil { - return nil, fmt.Errorf(`failed to get value of "primaryDisk.pullMethod" field: %w`, err) - } - switch imageSource(osImageSource) { - case httpSource: - return &cdiv1beta1.DataVolumeSource{HTTP: &cdiv1beta1.DataVolumeSourceHTTP{URL: osImage}}, nil - case registrySource: - return registryDataVolume(osImage, pullMethod), nil - case pvcSource: - if namespaceAndName := strings.Split(osImage, "/"); len(namespaceAndName) >= 2 { - return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: namespaceAndName[1], Namespace: namespaceAndName[0]}}, nil - } - return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: osImage, Namespace: namespace}}, nil - default: - // handle old API for backward compatibility. - if srcURL, err := url.ParseRequestURI(osImage); err == nil { - if srcURL.Scheme == cdiv1beta1.RegistrySchemeDocker || srcURL.Scheme == cdiv1beta1.RegistrySchemeOci { - return registryDataVolume(osImage, pullMethod), nil - } - return &cdiv1beta1.DataVolumeSource{HTTP: &cdiv1beta1.DataVolumeSourceHTTP{URL: osImage}}, nil - } - if namespaceAndName := strings.Split(osImage, "/"); len(namespaceAndName) >= 2 { - return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: namespaceAndName[1], Namespace: namespaceAndName[0]}}, nil - } - return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: osImage, Namespace: namespace}}, nil - } -} - -// getNamespace returns the namespace where the VM is created. -// VM is created in a dedicated namespace -// which is the namespace where the machine-controller pod is running. -// Defaults to `kube-system`. -func getNamespace() string { - ns := os.Getenv("POD_NAMESPACE") - if ns == "" { - // Useful especially for ci tests. - ns = metav1.NamespaceSystem - } - return ns -} - -func (p *provider) getPullMethod(pullMethod providerconfigtypes.ConfigVarString) (cdiv1beta1.RegistryPullMethod, error) { - resolvedPM, err := p.configVarResolver.GetConfigVarStringValue(&pullMethod) - if err != nil { - return "", err - } - switch pm := cdiv1beta1.RegistryPullMethod(resolvedPM); pm { - case cdiv1beta1.RegistryPullNode, cdiv1beta1.RegistryPullPod: - return pm, nil - case "": - return cdiv1beta1.RegistryPullNode, nil - default: - return "", fmt.Errorf("unsupported value: %v", resolvedPM) - } -} - -func registryDataVolume(imageURL string, pullMethod cdiv1beta1.RegistryPullMethod) *cdiv1beta1.DataVolumeSource { - return &cdiv1beta1.DataVolumeSource{ - Registry: &cdiv1beta1.DataVolumeSourceRegistry{ - URL: &imageURL, - PullMethod: &pullMethod, - }, - } -} - -func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - sigClient, err := client.New(c.RestConfig, client.Options{}) - if err != nil { - return nil, fmt.Errorf("failed to get kubevirt client: %w", err) - } - - virtualMachine := &kubevirtv1.VirtualMachine{} - if err := sigClient.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: machine.Name}, virtualMachine); err != nil { - if !kerrors.IsNotFound(err) { - return nil, fmt.Errorf("failed to get VirtualMachine %s: %w", machine.Name, err) - } - return nil, cloudprovidererrors.ErrInstanceNotFound - } - - virtualMachineInstance := &kubevirtv1.VirtualMachineInstance{} - if err := sigClient.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: machine.Name}, virtualMachineInstance); err != nil { - if kerrors.IsNotFound(err) { - return &kubeVirtServer{}, nil - } - - return nil, err - } - - // Deletion takes some time, so consider the VMI as deleted as soon as it has a DeletionTimestamp - // because once the node got into status not ready its informers won't fire again - // With the current approach we may run into a conflict when creating the VMI again, however this - // results in the machine being reqeued - if virtualMachineInstance.DeletionTimestamp != nil { - return nil, cloudprovidererrors.ErrInstanceNotFound - } - - return &kubeVirtServer{vmi: *virtualMachineInstance}, nil -} - -// We don't use the UID for kubevirt because the name of a VMI must stay stable -// in order for the node name to stay stable. The operator is responsible for ensuring -// there are no conflicts, e.G. by using one Namespace per Kubevirt user cluster. -func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ types.UID) error { - return nil -} - -func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { - c, pc, err := p.getConfig(spec.ProviderSpec) - if err != nil { - return fmt.Errorf("failed to parse config: %w", err) - } - // If instancetype is specified, skip CPU and Memory validation. - // Values will come from instancetype. - if c.Instancetype == nil { - if _, err := parseResources(c.CPUs, c.Memory); err != nil { - return err - } - } - - sigClient, err := client.New(c.RestConfig, client.Options{}) - if err != nil { - return fmt.Errorf("failed to get kubevirt client: %w", err) - } - if _, ok := kubevirttypes.SupportedOS[pc.OperatingSystem]; !ok { - return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, providerconfigtypes.ErrOSNotSupported) - } - if c.DNSPolicy == corev1.DNSNone { - if c.DNSConfig == nil || len(c.DNSConfig.Nameservers) == 0 { - return fmt.Errorf("dns config must be specified when dns policy is None") - } - } - // Check if we can reach the API of the target cluster. - vmi := &kubevirtv1.VirtualMachineInstance{} - if err := sigClient.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: "not-expected-to-exist"}, vmi); err != nil && !kerrors.IsNotFound(err) { - return fmt.Errorf("failed to request VirtualMachineInstances: %w", err) - } - - return nil -} - -func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { - return spec, nil -} - -func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { - c, _, err := p.getConfig(spec.ProviderSpec) - if err != nil { - return "", "", fmt.Errorf("failed to parse config: %w", err) - } - - cc := kubevirttypes.CloudConfig{ - Namespace: c.Namespace, - } - ccs, err := cc.String() - - return ccs, string(providerconfigtypes.CloudProviderExternal), err -} - -func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { - labels := make(map[string]string) - - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err == nil { - labels["cpus"] = c.CPUs - labels["memoryMIB"] = c.Memory - if c.OSImageSource.HTTP != nil { - labels["osImage"] = c.OSImageSource.HTTP.URL - } else if c.OSImageSource.PVC != nil { - labels["osImage"] = c.OSImageSource.PVC.Name - } - } - - return labels, err -} - -type machineDeploymentNameGetter func() (string, error) - -func machineDeploymentNameAndRevisionForMachineGetter(ctx context.Context, machine *clusterv1alpha1.Machine, c client.Client) machineDeploymentNameGetter { - mdName, _, err := controllerutil.GetMachineDeploymentNameAndRevisionForMachine(ctx, machine, c) - return func() (string, error) { - return mdName, err - } -} - -func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { - c, pc, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - - sigClient, err := client.New(c.RestConfig, client.Options{}) - if err != nil { - return nil, fmt.Errorf("failed to get kubevirt client: %w", err) - } - - userDataSecretName := fmt.Sprintf("userdata-%s-%s", machine.Name, strconv.Itoa(int(time.Now().Unix()))) - - virtualMachine, err := p.newVirtualMachine(ctx, c, pc, machine, userDataSecretName, userdata, - machineDeploymentNameAndRevisionForMachineGetter(ctx, machine, data.Client), randomMacAddressGetter) - if err != nil { - return nil, fmt.Errorf("could not create a VirtualMachine manifest %w", err) - } - - if err := sigClient.Create(ctx, virtualMachine); err != nil { - return nil, fmt.Errorf("failed to create vmi: %w", err) - } - - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: userDataSecretName, - Namespace: virtualMachine.Namespace, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(virtualMachine, kubevirtv1.VirtualMachineGroupVersionKind)}, - }, - Data: map[string][]byte{"userdata": []byte(userdata)}, - } - if err := sigClient.Create(ctx, secret); err != nil { - return nil, fmt.Errorf("failed to create secret for userdata: %w", err) - } - return &kubeVirtServer{}, nil -} - -func (p *provider) newVirtualMachine(_ context.Context, c *Config, pc *providerconfigtypes.Config, machine *clusterv1alpha1.Machine, - userdataSecretName, userdata string, mdNameGetter machineDeploymentNameGetter, macAddressGetter macAddressGetter) (*kubevirtv1.VirtualMachine, error) { - // We add the timestamp because the secret name must be different when we recreate the VMI - // because its pod got deleted - // The secret has an ownerRef on the VMI so garbace collection will take care of cleaning up. - terminationGracePeriodSeconds := int64(30) - - evictionStrategy := kubevirtv1.EvictionStrategyExternal - - resourceRequirements := kubevirtv1.ResourceRequirements{} - labels := map[string]string{"kubevirt.io/vm": machine.Name} - //Add a common label to all VirtualMachines spawned by the same MachineDeployment (= MachineDeployment name). - if mdName, err := mdNameGetter(); err == nil { - labels[machineDeploymentLabelKey] = mdName - } - - // if no instancetype, resources are from config. - if c.Instancetype == nil { - requestsAndLimits, err := parseResources(c.CPUs, c.Memory) - if err != nil { - return nil, err - } - resourceRequirements.Requests = *requestsAndLimits - resourceRequirements.Limits = *requestsAndLimits - } - - // Add cluster labels - labels["cluster.x-k8s.io/cluster-name"] = c.ClusterName - labels["cluster.x-k8s.io/role"] = "worker" - - var ( - dataVolumeName = machine.Name - annotations map[string]string - ) - // Add machineName as prefix to secondaryDisks. - addPrefixToSecondaryDisk(c.SecondaryDisks, dataVolumeName) - - if pc.OperatingSystem == providerconfigtypes.OperatingSystemFlatcar { - annotations = map[string]string{ - "kubevirt.io/ignitiondata": userdata, - } - } - - defaultBridgeNetwork, err := defaultBridgeNetwork(macAddressGetter) - if err != nil { - return nil, fmt.Errorf("could not compute a random MAC address") - } - - runStrategyOnce := kubevirtv1.RunStrategyOnce - - virtualMachine := &kubevirtv1.VirtualMachine{ - ObjectMeta: metav1.ObjectMeta{ - Name: machine.Name, - Namespace: c.Namespace, - Labels: labels, - }, - Spec: kubevirtv1.VirtualMachineSpec{ - RunStrategy: &runStrategyOnce, - Instancetype: c.Instancetype, - Preference: c.Preference, - Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: annotations, - Labels: labels, - }, - Spec: kubevirtv1.VirtualMachineInstanceSpec{ - EvictionStrategy: &evictionStrategy, - Networks: []kubevirtv1.Network{ - *kubevirtv1.DefaultPodNetwork(), - }, - Domain: kubevirtv1.DomainSpec{ - Devices: kubevirtv1.Devices{ - Disks: getVMDisks(c), - Interfaces: []kubevirtv1.Interface{*defaultBridgeNetwork}, - }, - Resources: resourceRequirements, - }, - Affinity: getAffinity(c), - TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, - Volumes: getVMVolumes(c, dataVolumeName, userdataSecretName), - DNSPolicy: c.DNSPolicy, - DNSConfig: c.DNSConfig, - TopologySpreadConstraints: getTopologySpreadConstraints(c, map[string]string{machineDeploymentLabelKey: labels[machineDeploymentLabelKey]}), - }, - }, - DataVolumeTemplates: getDataVolumeTemplates(c, dataVolumeName), - }, - } - return virtualMachine, nil -} - -func (p *provider) Cleanup(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return false, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - sigClient, err := client.New(c.RestConfig, client.Options{}) - if err != nil { - return false, fmt.Errorf("failed to get kubevirt client: %w", err) - } - - vm := &kubevirtv1.VirtualMachine{} - if err := sigClient.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: machine.Name}, vm); err != nil { - if !kerrors.IsNotFound(err) { - return false, fmt.Errorf("failed to get VirtualMachineInstance %s: %w", machine.Name, err) - } - return true, nil - } - - return false, sigClient.Delete(ctx, vm) -} - -func parseResources(cpus, memory string) (*corev1.ResourceList, error) { - memoryResource, err := resource.ParseQuantity(memory) - if err != nil { - return nil, fmt.Errorf("failed to parse memory requests: %w", err) - } - cpuResource, err := resource.ParseQuantity(cpus) - if err != nil { - return nil, fmt.Errorf("failed to parse cpu request: %w", err) - } - return &corev1.ResourceList{ - corev1.ResourceMemory: memoryResource, - corev1.ResourceCPU: cpuResource, - }, nil -} - -func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { - return nil -} - -func dnsPolicy(policy string) (corev1.DNSPolicy, error) { - switch policy { - case string(corev1.DNSClusterFirstWithHostNet): - return corev1.DNSClusterFirstWithHostNet, nil - case string(corev1.DNSClusterFirst): - return corev1.DNSClusterFirst, nil - case string(corev1.DNSDefault): - return corev1.DNSDefault, nil - case string(corev1.DNSNone): - return corev1.DNSNone, nil - } - - return "", fmt.Errorf("unknown dns policy: %s", policy) -} - -func getVMDisks(config *Config) []kubevirtv1.Disk { - disks := []kubevirtv1.Disk{ - { - Name: "datavolumedisk", - DiskDevice: kubevirtv1.DiskDevice{Disk: &kubevirtv1.DiskTarget{Bus: "virtio"}}, - }, - { - Name: "cloudinitdisk", - DiskDevice: kubevirtv1.DiskDevice{Disk: &kubevirtv1.DiskTarget{Bus: "virtio"}}, - }, - } - for _, sd := range config.SecondaryDisks { - disks = append(disks, kubevirtv1.Disk{ - Name: sd.Name, - DiskDevice: kubevirtv1.DiskDevice{Disk: &kubevirtv1.DiskTarget{Bus: "virtio"}}, - }) - } - return disks -} - -type macAddressGetter func() (string, error) - -func randomMacAddressGetter() (string, error) { - mac, err := netutil.GenerateRandMAC() - if err != nil { - return "", err - } - return mac.String(), nil -} - -func defaultBridgeNetwork(macAddressGetter macAddressGetter) (*kubevirtv1.Interface, error) { - defaultBridgeNetwork := kubevirtv1.DefaultBridgeNetworkInterface() - mac, err := macAddressGetter() - if err != nil { - return nil, err - } - defaultBridgeNetwork.MacAddress = mac - return defaultBridgeNetwork, nil -} - -func getVMVolumes(config *Config, dataVolumeName string, userDataSecretName string) []kubevirtv1.Volume { - volumes := []kubevirtv1.Volume{ - { - Name: "datavolumedisk", - VolumeSource: kubevirtv1.VolumeSource{ - DataVolume: &kubevirtv1.DataVolumeSource{ - Name: dataVolumeName, - }, - }, - }, - { - Name: "cloudinitdisk", - VolumeSource: kubevirtv1.VolumeSource{ - CloudInitNoCloud: &kubevirtv1.CloudInitNoCloudSource{ - UserDataSecretRef: &corev1.LocalObjectReference{ - Name: userDataSecretName, - }, - }, - }, - }, - } - for _, sd := range config.SecondaryDisks { - volumes = append(volumes, kubevirtv1.Volume{ - Name: sd.Name, - VolumeSource: kubevirtv1.VolumeSource{ - DataVolume: &kubevirtv1.DataVolumeSource{ - Name: sd.Name, - }}, - }) - } - return volumes -} - -func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1.DataVolumeTemplateSpec { - pvcRequest := corev1.ResourceList{corev1.ResourceStorage: config.PVCSize} - dataVolumeTemplates := []kubevirtv1.DataVolumeTemplateSpec{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: dataVolumeName, - }, - Spec: cdiv1beta1.DataVolumeSpec{ - PVC: &corev1.PersistentVolumeClaimSpec{ - StorageClassName: ptr.To(config.StorageClassName), - AccessModes: []corev1.PersistentVolumeAccessMode{ - config.StorageAccessType, - }, - Resources: corev1.VolumeResourceRequirements{ - Requests: pvcRequest, - }, - }, - Source: config.OSImageSource, - }, - }, - } - for _, sd := range config.SecondaryDisks { - dataVolumeTemplates = append(dataVolumeTemplates, kubevirtv1.DataVolumeTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: sd.Name, - }, - Spec: cdiv1beta1.DataVolumeSpec{ - PVC: &corev1.PersistentVolumeClaimSpec{ - StorageClassName: ptr.To(sd.StorageClassName), - AccessModes: []corev1.PersistentVolumeAccessMode{ - config.StorageAccessType, - }, - Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceStorage: sd.Size}, - }, - }, - Source: config.OSImageSource, - }, - }) - } - return dataVolumeTemplates -} - -func getAffinity(config *Config) *corev1.Affinity { - affinity := &corev1.Affinity{} - - expressions := []corev1.NodeSelectorRequirement{ - { - Key: config.NodeAffinityPreset.Key, - Operator: corev1.NodeSelectorOperator(metav1.LabelSelectorOpExists), - }, - } - - // change the operator if any values were passed for node affinity matching - if len(config.NodeAffinityPreset.Values) > 0 { - expressions[0].Operator = corev1.NodeSelectorOperator(metav1.LabelSelectorOpIn) - expressions[0].Values = config.NodeAffinityPreset.Values - } - - // NodeAffinity - switch config.NodeAffinityPreset.Type { - case softAffinityType: - affinity.NodeAffinity = &corev1.NodeAffinity{ - PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{ - { - Weight: 1, - Preference: corev1.NodeSelectorTerm{ - MatchExpressions: expressions, - }, - }, - }, - } - case hardAffinityType: - affinity.NodeAffinity = &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: []corev1.NodeSelectorTerm{ - { - MatchExpressions: expressions, - }, - }, - }, - } - } - - return affinity -} - -func addPrefixToSecondaryDisk(secondaryDisks []SecondaryDisks, prefix string) { - for i := range secondaryDisks { - secondaryDisks[i].Name = fmt.Sprintf("%s-%s", prefix, secondaryDisks[i].Name) - } -} - -func getTopologySpreadConstraints(config *Config, matchLabels map[string]string) []corev1.TopologySpreadConstraint { - if len(config.TopologySpreadConstraints) != 0 { - for i := range config.TopologySpreadConstraints { - config.TopologySpreadConstraints[i].LabelSelector = &metav1.LabelSelector{MatchLabels: matchLabels} - } - return config.TopologySpreadConstraints - } - // Return default TopologySpreadConstraint - return []corev1.TopologySpreadConstraint{ - { - MaxSkew: 1, - TopologyKey: topologyKeyHostname, - WhenUnsatisfiable: corev1.ScheduleAnyway, - LabelSelector: &metav1.LabelSelector{MatchLabels: matchLabels}, - }, - } -} diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go deleted file mode 100644 index 431207287..000000000 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ /dev/null @@ -1,345 +0,0 @@ -/* -Copyright 2022 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubevirt - -import ( - "bytes" - "context" - "embed" - "html/template" - "path" - "reflect" - "testing" - - kubevirtv1 "kubevirt.io/api/core/v1" - cdiv1beta1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" - - cloudprovidertesting "github.com/kubermatic/machine-controller/pkg/cloudprovider/testing" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/equality" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/apimachinery/pkg/util/diff" - ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" - fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -var ( - //go:embed testdata - vmManifestsFS embed.FS - vmDir = "testdata" - fakeclient ctrlruntimeclient.WithWatch - expectedVms map[string]*kubevirtv1.VirtualMachine -) - -func init() { - fakeclient = fakectrlruntimeclient.NewClientBuilder().Build() - objs := runtimeFromYaml(fakeclient, vmManifestsFS, vmDir) - expectedVms = toVirtualMachines(objs) -} - -type kubevirtProviderSpecConf struct { - OsImageDV string // if OsImage from DV and not from http source - Instancetype *kubevirtv1.InstancetypeMatcher - Preference *kubevirtv1.PreferenceMatcher - OperatingSystem string - TopologySpreadConstraint bool - Affinity bool - AffinityValues bool - SecondaryDisks bool - OsImageSource imageSource - OsImageSourceURL string - PullMethod cdiv1beta1.RegistryPullMethod -} - -func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { - var out bytes.Buffer - tmpl, err := template.New("test").Parse(`{ - "cloudProvider": "kubevirt", - "cloudProviderSpec": { - "clusterName": "cluster-name", - "auth": { - "kubeconfig": "eyJhcGlWZXJzaW9uIjoidjEiLCJjbHVzdGVycyI6W3siY2x1c3RlciI6eyJjZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YSI6IiIsInNlcnZlciI6Imh0dHBzOi8vOTUuMjE2LjIwLjE0Njo2NDQzIn0sIm5hbWUiOiJrdWJlcm5ldGVzIn1dLCJjb250ZXh0cyI6W3siY29udGV4dCI6eyJjbHVzdGVyIjoia3ViZXJuZXRlcyIsIm5hbWVzcGFjZSI6Imt1YmUtc3lzdGVtIiwidXNlciI6Imt1YmVybmV0ZXMtYWRtaW4ifSwibmFtZSI6Imt1YmVybmV0ZXMtYWRtaW5Aa3ViZXJuZXRlcyJ9XSwiY3VycmVudC1jb250ZXh0Ijoia3ViZXJuZXRlcy1hZG1pbkBrdWJlcm5ldGVzIiwia2luZCI6IkNvbmZpZyIsInByZWZlcmVuY2VzIjp7fSwidXNlcnMiOlt7Im5hbWUiOiJrdWJlcm5ldGVzLWFkbWluIiwidXNlciI6eyJjbGllbnQtY2VydGlmaWNhdGUtZGF0YSI6IiIsImNsaWVudC1rZXktZGF0YSI6IiJ9fV19" - }, - {{- if .TopologySpreadConstraint }} - "topologySpreadConstraints": [{ - "maxSkew": "2", - "topologyKey": "key1", - "whenUnsatisfiable": "DoNotSchedule"},{ - "maxSkew": "3", - "topologyKey": "key2", - "whenUnsatisfiable": "ScheduleAnyway"}], - {{- end }} - {{- if .Affinity }} - "affinity": { - "nodeAffinityPreset": { - "type": "hard", - "key": "key1" - {{- if .AffinityValues }} - , "values": [ - "foo1", "foo2" ] - {{- end }} - } - }, - {{- end }} - "virtualMachine": { - {{- if .Instancetype }} - "instancetype": { - "name": "{{ .Instancetype.Name }}", - "kind": "{{ .Instancetype.Kind }}" - }, - {{- end }} - {{- if .Preference }} - "preference": { - "name": "{{ .Preference.Name }}", - "kind": "{{ .Preference.Kind }}" - }, - {{- end }} - "template": { - "cpus": "2", - "memory": "2Gi", - {{- if .SecondaryDisks }} - "secondaryDisks": [{ - "size": "20Gi", - "storageClassName": "longhorn2"},{ - "size": "30Gi", - "storageClassName": "longhorn3"}], - {{- end }} - "primaryDisk": { - {{- if .OsImageDV }} - "osImage": "{{ .OsImageDV }}", - {{- else }} - "osImage": "{{ if .OsImageSourceURL }}{{ .OsImageSourceURL }}{{ else }}http://x.y.z.t/ubuntu.img{{ end }}", - {{- end }} - {{- if .PullMethod }} - "pullMethod": "{{ .PullMethod }}", - {{- end}} - "size": "10Gi", - {{- if .OsImageSource }} - "source": "{{ .OsImageSource }}", - {{- end }} - "storageClassName": "longhorn" - } - } - } - }, - "operatingSystem": "ubuntu", - "operatingSystemSpec": { - "disableAutoUpdate": false, - "disableLocksmithD": true, - "disableUpdateEngine": false - } -}`) - if err != nil { - t.Fatalf("Error occurred while parsing kubevirt provider spec template: %v", err) - } - err = tmpl.Execute(&out, k) - if err != nil { - t.Fatalf("Error occurred while executing kubevirt provider spec template: %v", err) - } - t.Logf("Generated providerSpec: %s", out.String()) - return out.Bytes() -} - -var ( - userdata = "fake-userdata" - testNamespace = "test-namespace" -) - -func TestNewVirtualMachine(t *testing.T) { - tests := []struct { - name string - specConf kubevirtProviderSpecConf - }{ - { - name: "nominal-case", - specConf: kubevirtProviderSpecConf{}, - }, - { - name: "instancetype-preference-standard", - specConf: kubevirtProviderSpecConf{ - Instancetype: &kubevirtv1.InstancetypeMatcher{ - Name: "standard-it", - Kind: "VirtualMachineInstancetype", - }, - Preference: &kubevirtv1.PreferenceMatcher{ - Name: "standard-pref", - Kind: "VirtualMachinePreference", - }, - }, - }, - { - name: "instancetype-preference-custom", - specConf: kubevirtProviderSpecConf{ - Instancetype: &kubevirtv1.InstancetypeMatcher{ - Name: "custom-it", - Kind: "VirtualMachineClusterInstancetype", - }, - Preference: &kubevirtv1.PreferenceMatcher{ - Name: "custom-pref", - Kind: "VirtualMachineClusterPreference", - }, - }, - }, - { - name: "topologyspreadconstraints", - specConf: kubevirtProviderSpecConf{TopologySpreadConstraint: true}, - }, - { - name: "affinity", - specConf: kubevirtProviderSpecConf{Affinity: true, AffinityValues: true}, - }, - { - name: "affinity-no-values", - specConf: kubevirtProviderSpecConf{Affinity: true, AffinityValues: false}, - }, - { - name: "secondary-disks", - specConf: kubevirtProviderSpecConf{SecondaryDisks: true}, - }, - { - name: "custom-local-disk", - specConf: kubevirtProviderSpecConf{OsImageDV: "ns/dvname"}, - }, - { - name: "http-image-source", - specConf: kubevirtProviderSpecConf{OsImageSource: httpSource}, - }, - { - name: "registry-image-source", - specConf: kubevirtProviderSpecConf{OsImageSource: registrySource, OsImageSourceURL: "docker://x.y.z.t/ubuntu.img:latest"}, - }, - { - name: "registry-image-source-pod", - specConf: kubevirtProviderSpecConf{OsImageSource: registrySource, OsImageSourceURL: "docker://x.y.z.t/ubuntu.img:latest", PullMethod: cdiv1beta1.RegistryPullPod}, - }, - { - name: "pvc-image-source", - specConf: kubevirtProviderSpecConf{OsImageSource: pvcSource, OsImageDV: "ns/dvname"}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - p := &provider{ - // Note that configVarResolver is not used in this test as the getConfigFunc is mocked. - configVarResolver: &providerconfig.ConfigPointerVarResolver{ - Cvr: providerconfig.NewConfigVarResolver(context.Background(), fakeclient), - }, - } - - machine := cloudprovidertesting.Creator{ - Name: tt.name, - Namespace: "kubevirt", - ProviderSpecGetter: tt.specConf.rawProviderSpec, - }.CreateMachine(t) - - c, pc, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - t.Fatalf("provider.getConfig() error %v", err) - } - // Do not rely on POD_NAMESPACE env variable, force to known value - c.Namespace = testNamespace - - // Check the created VirtualMachine - vm, _ := p.newVirtualMachine(context.TODO(), c, pc, machine, "udsn", userdata, fakeMachineDeploymentNameAndRevisionForMachineGetter(), fixedMacAddressGetter) - vm.TypeMeta.APIVersion, vm.TypeMeta.Kind = kubevirtv1.VirtualMachineGroupVersionKind.ToAPIVersionAndKind() - - if !equality.Semantic.DeepEqual(vm, expectedVms[tt.name]) { - t.Errorf("Diff %v", diff.ObjectGoPrintDiff(expectedVms[tt.name], vm)) - } - }) - } -} - -func fakeMachineDeploymentNameAndRevisionForMachineGetter() machineDeploymentNameGetter { - return func() (string, error) { - return "md-name", nil - } -} - -func toVirtualMachines(objects []runtime.Object) map[string]*kubevirtv1.VirtualMachine { - vms := make(map[string]*kubevirtv1.VirtualMachine) - for _, o := range objects { - if vm, ok := o.(*kubevirtv1.VirtualMachine); ok { - vms[vm.Name] = vm - } - } - return vms -} - -func fixedMacAddressGetter() (string, error) { - return "b6:f5:b4:fe:45:1d", nil -} - -// runtimeFromYaml returns a list of Kubernetes runtime objects from their yaml templates. -// It returns the objects for all files included in the ManifestFS folder, skipping (with error log) the yaml files -// that would not contain correct yaml files. -func runtimeFromYaml(client ctrlruntimeclient.Client, fs embed.FS, dir string) []runtime.Object { - decode := serializer.NewCodecFactory(client.Scheme()).UniversalDeserializer().Decode - - files, _ := fs.ReadDir(dir) - objects := make([]runtime.Object, 0, len(files)) - - for _, f := range files { - manifest, err := fs.ReadFile(path.Join(dir, f.Name())) - if err != nil { - continue - } - obj, _, err := decode(manifest, nil, nil) - // Skip and log but continue with others - if err != nil { - continue - } - objects = append(objects, obj) - } - - return objects -} -func TestTopologySpreadConstraint(t *testing.T) { - tests := []struct { - desc string - config Config - expected []corev1.TopologySpreadConstraint - }{ - { - desc: "default topology constraint", - config: Config{TopologySpreadConstraints: nil}, - expected: []corev1.TopologySpreadConstraint{ - {MaxSkew: 1, TopologyKey: topologyKeyHostname, WhenUnsatisfiable: corev1.ScheduleAnyway, LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"md": "test-md"}}}, - }, - }, - { - desc: "custom topology constraint", - config: Config{TopologySpreadConstraints: []corev1.TopologySpreadConstraint{{MaxSkew: 1, TopologyKey: "test-topology-key", WhenUnsatisfiable: corev1.DoNotSchedule}}}, - expected: []corev1.TopologySpreadConstraint{ - {MaxSkew: 1, TopologyKey: "test-topology-key", WhenUnsatisfiable: corev1.DoNotSchedule, LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"md": "test-md"}}}, - }, - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - result := getTopologySpreadConstraints(&test.config, map[string]string{"md": "test-md"}) - if !reflect.DeepEqual(result, test.expected) { - t.Errorf("expected ToplogySpreadConstraint: %v, got: %v", test.expected, result) - } - }) - } -} diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml deleted file mode 100644 index 7d01a42c7..000000000 --- a/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml +++ /dev/null @@ -1,83 +0,0 @@ -apiVersion: kubevirt.io/v1 -kind: VirtualMachine -metadata: - annotations: - labels: - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - kubevirt.io/vm: affinity-no-values - md: md-name - name: affinity-no-values - namespace: test-namespace -spec: - dataVolumeTemplates: - - metadata: - creationTimestamp: null - name: affinity-no-values - spec: - pvc: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - storageClassName: longhorn - source: - http: - url: http://x.y.z.t/ubuntu.img - runStrategy: Once - template: - metadata: - labels: - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - kubevirt.io/vm: affinity-no-values - md: md-name - spec: - affinity: - nodeAffinity: # Section present if nodeAffinityPreset.type != "" - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: key1 - operator: Exists - domain: - devices: - disks: - - disk: - bus: virtio - name: datavolumedisk - - disk: - bus: virtio - name: cloudinitdisk - interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default - bridge: {} - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: "2" - memory: 2Gi - networks: - - name: default - pod: {} - terminationGracePeriodSeconds: 30 - topologyspreadconstraints: - - maxskew: 1 - topologykey: kubernetes.io/hostname - whenunsatisfiable: ScheduleAnyway - labelselector: - matchlabels: - md: md-name - volumes: - - dataVolume: - name: affinity-no-values - name: datavolumedisk - - cloudInitNoCloud: - secretRef: - name: udsn - name: cloudinitdisk - evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml deleted file mode 100644 index a28aded56..000000000 --- a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml +++ /dev/null @@ -1,86 +0,0 @@ -apiVersion: kubevirt.io/v1 -kind: VirtualMachine -metadata: - annotations: - labels: - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - kubevirt.io/vm: affinity - md: md-name - name: affinity - namespace: test-namespace -spec: - dataVolumeTemplates: - - metadata: - creationTimestamp: null - name: affinity - spec: - pvc: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - storageClassName: longhorn - source: - http: - url: http://x.y.z.t/ubuntu.img - runStrategy: Once - template: - metadata: - labels: - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - kubevirt.io/vm: affinity - md: md-name - spec: - affinity: - nodeAffinity: # Section present if nodeAffinityPreset.type != "" - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: key1 - operator: In - values: - - foo1 - - foo2 - domain: - devices: - disks: - - disk: - bus: virtio - name: datavolumedisk - - disk: - bus: virtio - name: cloudinitdisk - interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default - bridge: {} - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: "2" - memory: 2Gi - networks: - - name: default - pod: {} - terminationGracePeriodSeconds: 30 - topologyspreadconstraints: - - maxskew: 1 - topologykey: kubernetes.io/hostname - whenunsatisfiable: ScheduleAnyway - labelselector: - matchlabels: - md: md-name - volumes: - - dataVolume: - name: affinity - name: datavolumedisk - - cloudInitNoCloud: - secretRef: - name: udsn - name: cloudinitdisk - evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml deleted file mode 100644 index b77494b63..000000000 --- a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml +++ /dev/null @@ -1,78 +0,0 @@ -apiVersion: kubevirt.io/v1 -kind: VirtualMachine -metadata: - annotations: - labels: - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - kubevirt.io/vm: custom-local-disk - md: md-name - name: custom-local-disk - namespace: test-namespace -spec: - dataVolumeTemplates: - - metadata: - name: custom-local-disk - spec: - pvc: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - storageClassName: longhorn - source: - pvc: - namespace: ns - name: dvname - runStrategy: Once - template: - metadata: - creationTimestamp: null - labels: - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - kubevirt.io/vm: custom-local-disk - md: md-name - spec: - affinity: {} - domain: - devices: - disks: - - disk: - bus: virtio - name: datavolumedisk - - disk: - bus: virtio - name: cloudinitdisk - interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default - bridge: {} - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: "2" - memory: 2Gi - networks: - - name: default - pod: {} - terminationGracePeriodSeconds: 30 - topologyspreadconstraints: - - maxskew: 1 - topologykey: kubernetes.io/hostname - whenunsatisfiable: ScheduleAnyway - labelselector: - matchlabels: - md: md-name - volumes: - - dataVolume: - name: custom-local-disk - name: datavolumedisk - - cloudInitNoCloud: - secretRef: - name: udsn - name: cloudinitdisk - evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml deleted file mode 100644 index caef1ed52..000000000 --- a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml +++ /dev/null @@ -1,77 +0,0 @@ -apiVersion: kubevirt.io/v1 -kind: VirtualMachine -metadata: - annotations: - labels: - kubevirt.io/vm: http-image-source - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - md: md-name - name: http-image-source - namespace: test-namespace -spec: - dataVolumeTemplates: - - metadata: - name: http-image-source - spec: - pvc: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - storageClassName: longhorn - source: - http: - url: http://x.y.z.t/ubuntu.img - runStrategy: Once - template: - metadata: - creationTimestamp: null - labels: - kubevirt.io/vm: http-image-source - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - md: md-name - spec: - affinity: {} - domain: - devices: - disks: - - disk: - bus: virtio - name: datavolumedisk - - disk: - bus: virtio - name: cloudinitdisk - interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default - bridge: {} - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: "2" - memory: 2Gi - networks: - - name: default - pod: {} - terminationGracePeriodSeconds: 30 - topologyspreadconstraints: - - maxskew: 1 - topologykey: kubernetes.io/hostname - whenunsatisfiable: ScheduleAnyway - labelselector: - matchlabels: - md: md-name - volumes: - - dataVolume: - name: http-image-source - name: datavolumedisk - - cloudInitNoCloud: - secretRef: - name: udsn - name: cloudinitdisk - evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml deleted file mode 100644 index ba699302e..000000000 --- a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml +++ /dev/null @@ -1,76 +0,0 @@ -apiVersion: kubevirt.io/v1 -kind: VirtualMachine -metadata: - annotations: - labels: - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - kubevirt.io/vm: instancetype-preference-custom - md: md-name - name: instancetype-preference-custom - namespace: test-namespace -spec: - dataVolumeTemplates: - - metadata: - creationTimestamp: null - name: instancetype-preference-custom - spec: - pvc: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - storageClassName: longhorn - source: - http: - url: http://x.y.z.t/ubuntu.img - runStrategy: Once - instancetype: - kind: VirtualMachineClusterInstancetype - name: custom-it - preference: - kind: VirtualMachineClusterPreference - name: custom-pref - template: - metadata: - labels: - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - kubevirt.io/vm: instancetype-preference-custom - md: md-name - spec: - affinity: {} - domain: - devices: - disks: - - disk: - bus: virtio - name: datavolumedisk - - disk: - bus: virtio - name: cloudinitdisk - interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default - bridge: {} - networks: - - name: default - pod: {} - terminationGracePeriodSeconds: 30 - topologyspreadconstraints: - - maxskew: 1 - topologykey: kubernetes.io/hostname - whenunsatisfiable: ScheduleAnyway - labelselector: - matchlabels: - md: md-name - volumes: - - dataVolume: - name: instancetype-preference-custom - name: datavolumedisk - - cloudInitNoCloud: - secretRef: - name: udsn - name: cloudinitdisk - evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml deleted file mode 100644 index 1f54c87a7..000000000 --- a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml +++ /dev/null @@ -1,76 +0,0 @@ -apiVersion: kubevirt.io/v1 -kind: VirtualMachine -metadata: - annotations: - labels: - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - kubevirt.io/vm: instancetype-preference-standard - md: md-name - name: instancetype-preference-standard - namespace: test-namespace -spec: - dataVolumeTemplates: - - metadata: - name: instancetype-preference-standard - spec: - pvc: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - storageClassName: longhorn - source: - http: - url: http://x.y.z.t/ubuntu.img - runStrategy: Once - instancetype: - kind: VirtualMachineInstancetype - name: standard-it - preference: - kind: VirtualMachinePreference - name: standard-pref - template: - metadata: - creationTimestamp: null - labels: - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - kubevirt.io/vm: instancetype-preference-standard - md: md-name - spec: - affinity: {} - domain: - devices: - disks: - - disk: - bus: virtio - name: datavolumedisk - - disk: - bus: virtio - name: cloudinitdisk - interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default - bridge: {} - networks: - - name: default - pod: {} - terminationGracePeriodSeconds: 30 - topologyspreadconstraints: - - maxskew: 1 - topologykey: kubernetes.io/hostname - whenunsatisfiable: ScheduleAnyway - labelselector: - matchlabels: - md: md-name - volumes: - - dataVolume: - name: instancetype-preference-standard - name: datavolumedisk - - cloudInitNoCloud: - secretRef: - name: udsn - name: cloudinitdisk - evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml deleted file mode 100644 index 24afa309c..000000000 --- a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml +++ /dev/null @@ -1,77 +0,0 @@ -apiVersion: kubevirt.io/v1 -kind: VirtualMachine -metadata: - annotations: - labels: - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - kubevirt.io/vm: nominal-case - md: md-name - name: nominal-case - namespace: test-namespace -spec: - dataVolumeTemplates: - - metadata: - name: nominal-case - spec: - pvc: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - storageClassName: longhorn - source: - http: - url: http://x.y.z.t/ubuntu.img - runStrategy: Once - template: - metadata: - creationTimestamp: null - labels: - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - kubevirt.io/vm: nominal-case - md: md-name - spec: - affinity: {} - domain: - devices: - disks: - - disk: - bus: virtio - name: datavolumedisk - - disk: - bus: virtio - name: cloudinitdisk - interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default - bridge: {} - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: "2" - memory: 2Gi - networks: - - name: default - pod: {} - terminationGracePeriodSeconds: 30 - topologyspreadconstraints: - - maxskew: 1 - topologykey: kubernetes.io/hostname - whenunsatisfiable: ScheduleAnyway - labelselector: - matchlabels: - md: md-name - volumes: - - dataVolume: - name: nominal-case - name: datavolumedisk - - cloudInitNoCloud: - secretRef: - name: udsn - name: cloudinitdisk - evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml deleted file mode 100644 index 978213ec3..000000000 --- a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml +++ /dev/null @@ -1,78 +0,0 @@ -apiVersion: kubevirt.io/v1 -kind: VirtualMachine -metadata: - annotations: - labels: - kubevirt.io/vm: pvc-image-source - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - md: md-name - name: pvc-image-source - namespace: test-namespace -spec: - dataVolumeTemplates: - - metadata: - name: pvc-image-source - spec: - pvc: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - storageClassName: longhorn - source: - pvc: - namespace: ns - name: dvname - runStrategy: Once - template: - metadata: - creationTimestamp: null - labels: - kubevirt.io/vm: pvc-image-source - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - md: md-name - spec: - affinity: {} - domain: - devices: - disks: - - disk: - bus: virtio - name: datavolumedisk - - disk: - bus: virtio - name: cloudinitdisk - interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default - bridge: {} - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: "2" - memory: 2Gi - networks: - - name: default - pod: {} - terminationGracePeriodSeconds: 30 - topologyspreadconstraints: - - maxskew: 1 - topologykey: kubernetes.io/hostname - whenunsatisfiable: ScheduleAnyway - labelselector: - matchlabels: - md: md-name - volumes: - - dataVolume: - name: pvc-image-source - name: datavolumedisk - - cloudInitNoCloud: - secretRef: - name: udsn - name: cloudinitdisk - evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml deleted file mode 100644 index 9a8115c1d..000000000 --- a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml +++ /dev/null @@ -1,78 +0,0 @@ -apiVersion: kubevirt.io/v1 -kind: VirtualMachine -metadata: - annotations: - labels: - kubevirt.io/vm: registry-image-source-pod - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - md: md-name - name: registry-image-source-pod - namespace: test-namespace -spec: - dataVolumeTemplates: - - metadata: - name: registry-image-source-pod - spec: - pvc: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - storageClassName: longhorn - source: - registry: - url: docker://x.y.z.t/ubuntu.img:latest - pullMethod: pod - runStrategy: Once - template: - metadata: - creationTimestamp: null - labels: - kubevirt.io/vm: registry-image-source-pod - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - md: md-name - spec: - affinity: {} - domain: - devices: - disks: - - disk: - bus: virtio - name: datavolumedisk - - disk: - bus: virtio - name: cloudinitdisk - interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default - bridge: {} - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: "2" - memory: 2Gi - networks: - - name: default - pod: {} - terminationGracePeriodSeconds: 30 - topologyspreadconstraints: - - maxskew: 1 - topologykey: kubernetes.io/hostname - whenunsatisfiable: ScheduleAnyway - labelselector: - matchlabels: - md: md-name - volumes: - - dataVolume: - name: registry-image-source-pod - name: datavolumedisk - - cloudInitNoCloud: - secretRef: - name: udsn - name: cloudinitdisk - evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml deleted file mode 100644 index ee0548b09..000000000 --- a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml +++ /dev/null @@ -1,78 +0,0 @@ -apiVersion: kubevirt.io/v1 -kind: VirtualMachine -metadata: - annotations: - labels: - kubevirt.io/vm: registry-image-source - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - md: md-name - name: registry-image-source - namespace: test-namespace -spec: - dataVolumeTemplates: - - metadata: - name: registry-image-source - spec: - pvc: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - storageClassName: longhorn - source: - registry: - url: docker://x.y.z.t/ubuntu.img:latest - pullMethod: node - runStrategy: Once - template: - metadata: - creationTimestamp: null - labels: - kubevirt.io/vm: registry-image-source - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - md: md-name - spec: - affinity: {} - domain: - devices: - disks: - - disk: - bus: virtio - name: datavolumedisk - - disk: - bus: virtio - name: cloudinitdisk - interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default - bridge: {} - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: "2" - memory: 2Gi - networks: - - name: default - pod: {} - terminationGracePeriodSeconds: 30 - topologyspreadconstraints: - - maxskew: 1 - topologykey: kubernetes.io/hostname - whenunsatisfiable: ScheduleAnyway - labelselector: - matchlabels: - md: md-name - volumes: - - dataVolume: - name: registry-image-source - name: datavolumedisk - - cloudInitNoCloud: - secretRef: - name: udsn - name: cloudinitdisk - evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml deleted file mode 100644 index 0ce57a4b7..000000000 --- a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml +++ /dev/null @@ -1,115 +0,0 @@ -apiVersion: kubevirt.io/v1 -kind: VirtualMachine -metadata: - annotations: - labels: - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - kubevirt.io/vm: secondary-disks - md: md-name - name: secondary-disks - namespace: test-namespace -spec: - dataVolumeTemplates: - - metadata: - name: secondary-disks - spec: - pvc: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - storageClassName: longhorn - source: - http: - url: http://x.y.z.t/ubuntu.img - - metadata: - name: secondary-disks-secondarydisk0 - spec: - pvc: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 20Gi - storageClassName: longhorn2 - source: - http: - url: http://x.y.z.t/ubuntu.img - - metadata: - name: secondary-disks-secondarydisk1 - spec: - pvc: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 30Gi - storageClassName: longhorn3 - source: - http: - url: http://x.y.z.t/ubuntu.img - runStrategy: Once - template: - metadata: - creationTimestamp: null - labels: - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - kubevirt.io/vm: secondary-disks - md: md-name - spec: - affinity: {} - domain: - devices: - disks: - - disk: - bus: virtio - name: datavolumedisk - - disk: - bus: virtio - name: cloudinitdisk - - disk: - bus: virtio - name: secondary-disks-secondarydisk0 - - disk: - bus: virtio - name: secondary-disks-secondarydisk1 - interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default - bridge: {} - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: "2" - memory: 2Gi - networks: - - name: default - pod: {} - terminationGracePeriodSeconds: 30 - topologyspreadconstraints: - - maxskew: 1 - topologykey: kubernetes.io/hostname - whenunsatisfiable: ScheduleAnyway - labelselector: - matchlabels: - md: md-name - volumes: - - dataVolume: - name: secondary-disks - name: datavolumedisk - - cloudInitNoCloud: - secretRef: - name: udsn - name: cloudinitdisk - - dataVolume: - name: secondary-disks-secondarydisk0 - name: secondary-disks-secondarydisk0 - - dataVolume: - name: secondary-disks-secondarydisk1 - name: secondary-disks-secondarydisk1 - evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml deleted file mode 100644 index 363460724..000000000 --- a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml +++ /dev/null @@ -1,83 +0,0 @@ -apiVersion: kubevirt.io/v1 -kind: VirtualMachine -metadata: - annotations: - labels: - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - kubevirt.io/vm: topologyspreadconstraints - md: md-name - name: topologyspreadconstraints - namespace: test-namespace -spec: - dataVolumeTemplates: - - metadata: - name: topologyspreadconstraints - spec: - pvc: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - storageClassName: longhorn - source: - http: - url: http://x.y.z.t/ubuntu.img - runStrategy: Once - template: - metadata: - creationTimestamp: null - labels: - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - kubevirt.io/vm: topologyspreadconstraints - md: md-name - spec: - affinity: {} - domain: - devices: - disks: - - disk: - bus: virtio - name: datavolumedisk - - disk: - bus: virtio - name: cloudinitdisk - interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default - bridge: {} - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: "2" - memory: 2Gi - networks: - - name: default - pod: {} - terminationGracePeriodSeconds: 30 - topologyspreadconstraints: - - maxskew: 2 - topologykey: key1 - whenunsatisfiable: DoNotSchedule - labelselector: - matchlabels: - md: md-name - - maxskew: 3 - topologykey: key2 - whenunsatisfiable: ScheduleAnyway - labelselector: - matchlabels: - md: md-name - volumes: - - dataVolume: - name: topologyspreadconstraints - name: datavolumedisk - - cloudInitNoCloud: - secretRef: - name: udsn - name: cloudinitdisk - evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/types/cloudconfig.go b/pkg/cloudprovider/provider/kubevirt/types/cloudconfig.go deleted file mode 100644 index 8d41053e3..000000000 --- a/pkg/cloudprovider/provider/kubevirt/types/cloudconfig.go +++ /dev/null @@ -1,36 +0,0 @@ -/* -Copyright 2021 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "gopkg.in/yaml.v3" -) - -type CloudConfig struct { - // Kubeconfig used to connect to the cluster that runs KubeVirt - Kubeconfig string `yaml:"kubeconfig"` - // Namespace used in KubeVirt cloud-controller-manager as infra cluster namespace. - Namespace string `yaml:"namespace"` -} - -func (c *CloudConfig) String() (string, error) { - out, err := yaml.Marshal(c) - if err != nil { - return "", err - } - return string(out), nil -} diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go deleted file mode 100644 index 6b3434b2a..000000000 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ /dev/null @@ -1,129 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - kubevirtv1 "kubevirt.io/api/core/v1" - - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - - corev1 "k8s.io/api/core/v1" -) - -var SupportedOS = map[providerconfigtypes.OperatingSystem]*struct{}{ - providerconfigtypes.OperatingSystemCentOS: nil, - providerconfigtypes.OperatingSystemUbuntu: nil, - providerconfigtypes.OperatingSystemRHEL: nil, - providerconfigtypes.OperatingSystemFlatcar: nil, - providerconfigtypes.OperatingSystemRockyLinux: nil, -} - -type RawConfig struct { - ClusterName *providerconfigtypes.ConfigVarString `json:"clusterName"` - Auth Auth `json:"auth,omitempty"` - VirtualMachine VirtualMachine `json:"virtualMachine,omitempty"` - Affinity Affinity `json:"affinity,omitempty"` - TopologySpreadConstraints []TopologySpreadConstraint `json:"topologySpreadConstraints"` -} - -// Auth. -type Auth struct { - Kubeconfig *providerconfigtypes.ConfigVarString `json:"kubeconfig,omitempty"` -} - -// VirtualMachine. -type VirtualMachine struct { - // Deprecated: use Instancetype/Preference instead. - Flavor Flavor `json:"flavor,omitempty"` - // Instancetype is optional. - Instancetype *kubevirtv1.InstancetypeMatcher `json:"instancetype,omitempty"` - // Preference is optional. - Preference *kubevirtv1.PreferenceMatcher `json:"preference,omitempty"` - Template Template `json:"template,omitempty"` - DNSPolicy *providerconfigtypes.ConfigVarString `json:"dnsPolicy,omitempty"` - DNSConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"` -} - -// Flavor. -type Flavor struct { - Name *providerconfigtypes.ConfigVarString `json:"name,omitempty"` - Profile *providerconfigtypes.ConfigVarString `json:"profile,omitempty"` -} - -// Template. -type Template struct { - CPUs *providerconfigtypes.ConfigVarString `json:"cpus,omitempty"` - Memory *providerconfigtypes.ConfigVarString `json:"memory,omitempty"` - PrimaryDisk PrimaryDisk `json:"primaryDisk,omitempty"` - SecondaryDisks []SecondaryDisks `json:"secondaryDisks,omitempty"` -} - -// PrimaryDisk. -type PrimaryDisk struct { - Disk - OsImage *providerconfigtypes.ConfigVarString `json:"osImage,omitempty"` - // Source describes the VM Disk Image source. - Source providerconfigtypes.ConfigVarString `json:"source,omitempty"` - // PullMethod describes the VM Disk Image source optional pull method for registry source. Defaults to 'node'. - PullMethod providerconfigtypes.ConfigVarString `json:"pullMethod,omitempty"` -} - -// SecondaryDisks. -type SecondaryDisks struct { - Disk -} - -// Disk. -type Disk struct { - Size providerconfigtypes.ConfigVarString `json:"size,omitempty"` - StorageClassName providerconfigtypes.ConfigVarString `json:"storageClassName,omitempty"` - StorageAccessType providerconfigtypes.ConfigVarString `json:"storageAccessType,omitempty"` -} - -// Affinity. -type Affinity struct { - // Deprecated: Use TopologySpreadConstraint instead. - PodAffinityPreset *providerconfigtypes.ConfigVarString `json:"podAffinityPreset,omitempty"` - // Deprecated: Use TopologySpreadConstraint instead. - PodAntiAffinityPreset *providerconfigtypes.ConfigVarString `json:"podAntiAffinityPreset,omitempty"` - NodeAffinityPreset NodeAffinityPreset `json:"nodeAffinityPreset,omitempty"` -} - -// NodeAffinityPreset. -type NodeAffinityPreset struct { - Type *providerconfigtypes.ConfigVarString `json:"type,omitempty"` - Key *providerconfigtypes.ConfigVarString `json:"key,omitempty"` - Values []providerconfigtypes.ConfigVarString `json:"values,omitempty"` -} - -// TopologySpreadConstraint describes topology spread constraints for VMs. -type TopologySpreadConstraint struct { - // MaxSkew describes the degree to which VMs may be unevenly distributed. - MaxSkew *providerconfigtypes.ConfigVarString `json:"maxSkew,omitempty"` - // TopologyKey is the key of infra-node labels. - TopologyKey *providerconfigtypes.ConfigVarString `json:"topologyKey,omitempty"` - // WhenUnsatisfiable indicates how to deal with a VM if it doesn't satisfy - // the spread constraint. - WhenUnsatisfiable *providerconfigtypes.ConfigVarString `json:"whenUnsatisfiable,omitempty"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/linode/provider.go b/pkg/cloudprovider/provider/linode/provider.go deleted file mode 100644 index cec85cfb6..000000000 --- a/pkg/cloudprovider/provider/linode/provider.go +++ /dev/null @@ -1,469 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package linode - -import ( - "context" - "crypto/rand" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "net/http" - "strconv" - "strings" - "time" - - "github.com/linode/linodego" - "go.uber.org/zap" - "golang.org/x/oauth2" - - common "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/common/ssh" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - linodetypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/linode/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" -) - -type provider struct { - configVarResolver *providerconfig.ConfigPointerVarResolver -} - -// New returns a linode provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return &provider{configVarResolver: &providerconfig.ConfigPointerVarResolver{Cvr: configVarResolver}} -} - -type Config struct { - Token string - Region string - Type string - Backups bool - PrivateNetworking bool - Tags []string -} - -const ( - createCheckTimeout = 5 * time.Minute - cloudinitStackScriptID = 392559 -) - -type TokenSource struct { - AccessToken string -} - -func (t *TokenSource) Token() (*oauth2.Token, error) { - token := &oauth2.Token{ - AccessToken: t.AccessToken, - } - return token, nil -} - -func getSlugForOS(os providerconfigtypes.OperatingSystem) (string, error) { - switch os { - case providerconfigtypes.OperatingSystemUbuntu: - return "linode/ubuntu18.04", nil - - /** - // StackScript for CloudInit is not centos7 ready - case providerconfigtypes.OperatingSystemCentOS: - return "linode/centos7", nil - **/ - } - return "", providerconfigtypes.ErrOSNotSupported -} - -func getClient(ctx context.Context, token string) linodego.Client { - tokenSource := &TokenSource{ - AccessToken: token, - } - - oauthClient := oauth2.NewClient(ctx, tokenSource) - - client := linodego.NewClient(oauthClient) - client.SetUserAgent("Kubermatic linodego") - - return client -} - -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - - pconfig, err := providerconfigtypes.GetConfig(provSpec) - if err != nil { - return nil, nil, err - } - - if pconfig.OperatingSystemSpec.Raw == nil { - return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") - } - - rawConfig, err := linodetypes.GetConfig(*pconfig) - if err != nil { - return nil, nil, err - } - - c := Config{} - c.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Token, "LINODE_TOKEN") - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"token\" field, error = %w", err) - } - c.Region, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Region) - if err != nil { - return nil, nil, err - } - c.Type, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Type) - if err != nil { - return nil, nil, err - } - c.Backups, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.Backups) - if err != nil { - return nil, nil, err - } - c.PrivateNetworking, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.PrivateNetworking) - if err != nil { - return nil, nil, err - } - - for _, tag := range rawConfig.Tags { - tagVal, err := p.configVarResolver.GetConfigVarStringValue(&tag) - if err != nil { - return nil, nil, err - } - c.Tags = append(c.Tags, tagVal) - } - - return &c, pconfig, err -} - -func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { - return spec, nil -} - -func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { - c, pc, err := p.getConfig(spec.ProviderSpec) - if err != nil { - return fmt.Errorf("failed to parse config: %w", err) - } - - if c.Token == "" { - return errors.New("token is missing") - } - - if c.Region == "" { - return errors.New("region is missing") - } - - if c.Type == "" { - return errors.New("type is missing") - } - - _, err = getSlugForOS(pc.OperatingSystem) - if err != nil { - return fmt.Errorf("invalid operating system specified %q: %w", pc.OperatingSystem, err) - } - - client := getClient(ctx, c.Token) - - _, err = client.GetRegion(ctx, c.Region) - if err != nil { - return err - } - - _, err = client.GetType(ctx, c.Type) - if err != nil { - return err - } - - return nil -} - -func createRandomPassword() (string, error) { - rawRootPass := make([]byte, 50) - _, err := rand.Read(rawRootPass) - if err != nil { - return "", fmt.Errorf("Failed to generate random password") - } - rootPass := base64.StdEncoding.EncodeToString(rawRootPass) - return rootPass, nil -} - -func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { - c, pc, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - - client := getClient(ctx, c.Token) - - sshkey, err := ssh.NewKey() - if err != nil { - return nil, fmt.Errorf("failed to generate ssh key: %w", err) - } - - slug, err := getSlugForOS(pc.OperatingSystem) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, invalid operating system specified %q: %v", pc.OperatingSystem, err), - } - } - - randomPassword, err := createRandomPassword() - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to generate password for Linode, due to %v", err), - } - } - - createRequest := linodego.InstanceCreateOptions{ - Image: slug, - Label: fmt.Sprintf("%.32s", machine.Spec.Name), - Region: c.Region, - Type: c.Type, - PrivateIP: c.PrivateNetworking, - RootPass: randomPassword, - BackupsEnabled: c.Backups, - AuthorizedKeys: []string{strings.TrimSpace(sshkey.PublicKey)}, - Tags: append(c.Tags, string(machine.UID)), - StackScriptID: cloudinitStackScriptID, - StackScriptData: map[string]string{ - "userdata": base64.StdEncoding.EncodeToString([]byte(userdata)), - }, - } - - linode, err := client.CreateInstance(ctx, createRequest) - if err != nil { - return nil, linodeStatusAndErrToTerminalError(err) - } - - linode, err = client.WaitForInstanceStatus(ctx, linode.ID, linodego.InstanceRunning, int(createCheckTimeout/time.Second)) - if err != nil { - return nil, linodeStatusAndErrToTerminalError(err) - } - - return &linodeInstance{linode: linode}, err -} - -func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { - instance, err := p.Get(ctx, log, machine, data) - if err != nil { - if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { - return true, nil - } - return false, err - } - - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return false, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - client := getClient(ctx, c.Token) - - linodeID, err := strconv.Atoi(instance.ID()) - if err != nil { - return false, fmt.Errorf("failed to convert instance id %s to int: %w", instance.ID(), err) - } - - err = client.DeleteInstance(ctx, linodeID) - if err != nil { - return false, linodeStatusAndErrToTerminalError(err) - } - - return false, nil -} - -func getListOptions(name string) *linodego.ListOptions { - filter, _ := json.Marshal(map[string]interface{}{ - "label": fmt.Sprintf("%.32s", name), - }) - - listOptions := linodego.NewListOptions(0, string(filter)) - return listOptions -} - -func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - - client := getClient(ctx, c.Token) - - listOptions := getListOptions(machine.Spec.Name) - linodes, err := client.ListInstances(ctx, listOptions) - - if err != nil { - return nil, linodeStatusAndErrToTerminalError(err) - } - - for i, linode := range linodes { - if sets.NewString(linode.Tags...).Has(string(machine.UID)) { - return &linodeInstance{linode: &linodes[i]}, nil - } - } - - return nil, cloudprovidererrors.ErrInstanceNotFound -} - -func (p *provider) MigrateUID(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return fmt.Errorf("failed to decode providerconfig: %w", err) - } - client := getClient(ctx, c.Token) - listOptions := getListOptions(machine.Spec.Name) - linodes, err := client.ListInstances(ctx, listOptions) - if err != nil { - return fmt.Errorf("failed to list linodes: %w", err) - } - - for _, linode := range linodes { - if sets.NewString(linode.Tags...).Has(string(machine.UID)) { - updateOpts := linode.GetUpdateOptions() - - tags := []string{string(newUID)} - if updateOpts.Tags != nil { - oldUID := string(machine.UID) - for _, existingTag := range *updateOpts.Tags { - if existingTag != oldUID { - tags = append(tags, existingTag) - } - } - } - updateOpts.Tags = &tags - _, err = client.UpdateInstance(ctx, linode.ID, updateOpts) - if err != nil { - return fmt.Errorf("failed to revise linode UID tags: %w", err) - } - } - } - - return nil -} - -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return "", "", nil -} - -func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { - labels := make(map[string]string) - - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err == nil { - labels["type"] = c.Type - labels["region"] = c.Region - } - - return labels, err -} - -type linodeInstance struct { - linode *linodego.Instance -} - -func (d *linodeInstance) Name() string { - return d.linode.Label -} - -func (d *linodeInstance) ID() string { - return strconv.Itoa(d.linode.ID) -} - -func (d *linodeInstance) ProviderID() string { - if d == nil || d.ID() == "" { - return "" - } - return fmt.Sprintf("linode://%s", d.ID()) -} - -func (d *linodeInstance) HostID() string { - return "" -} - -func (d *linodeInstance) Addresses() map[string]v1.NodeAddressType { - addresses := map[string]v1.NodeAddressType{} - for _, n := range d.linode.IPv4 { - addresses[n.String()] = v1.NodeInternalIP - } - addresses[d.linode.IPv6] = v1.NodeInternalIP - return addresses -} - -func (d *linodeInstance) Status() instance.Status { - switch d.linode.Status { - case linodego.InstanceProvisioning, linodego.InstanceBooting: - return instance.StatusCreating - case linodego.InstanceRunning: - return instance.StatusRunning - case linodego.InstanceDeleting: - return instance.StatusDeleting - default: - // Cloning, Migrating, Offline, Rebooting, - // Rebuilding, Resizing, Restoring, ShuttingDown - return instance.StatusUnknown - } -} - -// linodeStatusAndErrToTerminalError judges if the given HTTP status -// can be qualified as a "terminal" error, for more info see v1alpha1.MachineStatus - -// if the given error doesn't qualify the error passed as -// an argument will be returned. -func linodeStatusAndErrToTerminalError(err error) error { - status := 0 - var apiErr *linodego.Error - if errors.As(err, &apiErr) { - status = apiErr.Code - } - - switch status { - case http.StatusUnauthorized: - // authorization primitives come from MachineSpec - // thus we are setting InvalidConfigurationMachineError - return cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: "A request has been rejected due to invalid credentials which were taken from the MachineSpec", - } - default: - return err - } -} - -func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { - return nil -} diff --git a/pkg/cloudprovider/provider/linode/types/types.go b/pkg/cloudprovider/provider/linode/types/types.go deleted file mode 100644 index d8f27b435..000000000 --- a/pkg/cloudprovider/provider/linode/types/types.go +++ /dev/null @@ -1,37 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" -) - -type RawConfig struct { - Token *providerconfigtypes.ConfigVarString `json:"token,omitempty"` - Region *providerconfigtypes.ConfigVarString `json:"region"` - Type *providerconfigtypes.ConfigVarString `json:"type"` - Backups *providerconfigtypes.ConfigVarBool `json:"backups"` - PrivateNetworking *providerconfigtypes.ConfigVarBool `json:"private_networking"` - Tags []providerconfigtypes.ConfigVarString `json:"tags,omitempty"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/nutanix/client.go b/pkg/cloudprovider/provider/nutanix/client.go deleted file mode 100644 index 4394c3b48..000000000 --- a/pkg/cloudprovider/provider/nutanix/client.go +++ /dev/null @@ -1,471 +0,0 @@ -/* -Copyright 2021 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package nutanix - -import ( - "context" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "strings" - "time" - - nutanixclient "github.com/nutanix-cloud-native/prism-go-client" - nutanixv3 "github.com/nutanix-cloud-native/prism-go-client/v3" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - nutanixtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/nutanix/types" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/utils/ptr" -) - -const ( - invalidCredentials = "invalid Nutanix Credentials" -) - -type ClientSet struct { - Prism *nutanixv3.Client -} - -func GetClientSet(config *Config) (*ClientSet, error) { - if config == nil { - return nil, errors.New("no configuration passed") - } - - if config.Username == "" { - return nil, errors.New("no username specified") - } - - if config.Password == "" { - return nil, errors.New("no password specified") - } - - if config.Endpoint == "" { - return nil, errors.New("no endpoint specified") - } - - // set up 9440 as default port if none is passed via config - port := 9440 - if config.Port != nil { - port = *config.Port - } - - credentials := nutanixclient.Credentials{ - URL: fmt.Sprintf("%s:%d", config.Endpoint, port), - Endpoint: config.Endpoint, - Port: fmt.Sprint(port), - Username: config.Username, - Password: config.Password, - Insecure: config.AllowInsecure, - } - - if config.ProxyURL != "" { - credentials.ProxyURL = config.ProxyURL - } - - clientV3, err := nutanixv3.NewV3Client(credentials) - if err != nil { - return nil, err - } - - return &ClientSet{ - Prism: clientV3, - }, nil -} - -func createVM(ctx context.Context, client *ClientSet, name string, conf Config, userdata string) (instance.Instance, error) { - cluster, err := getClusterByName(ctx, client, conf.ClusterName) - if err != nil { - return nil, err - } - - subnet, err := getSubnetByName(ctx, client, conf.SubnetName, *cluster.Metadata.UUID) - if err != nil { - return nil, err - } - - nicList := []*nutanixv3.VMNic{ - { - SubnetReference: &nutanixv3.Reference{ - Kind: ptr.To(nutanixtypes.SubnetKind), - UUID: subnet.Metadata.UUID, - }, - }, - } - - for _, subnet := range conf.AdditionalSubnetNames { - additionalSubnet, err := getSubnetByName(ctx, client, subnet, *cluster.Metadata.UUID) - if err != nil { - return nil, err - } - additionalSubnetNic := &nutanixv3.VMNic{ - SubnetReference: &nutanixv3.Reference{ - Kind: ptr.To(nutanixtypes.SubnetKind), - UUID: additionalSubnet.Metadata.UUID, - }, - } - nicList = append(nicList, additionalSubnetNic) - } - - image, err := getImageByName(ctx, client, conf.ImageName) - if err != nil { - return nil, err - } - - request := &nutanixv3.VMIntentInput{ - Metadata: &nutanixv3.Metadata{ - Kind: ptr.To(nutanixtypes.VMKind), - Categories: conf.Categories, - }, - Spec: &nutanixv3.VM{ - Name: ptr.To(name), - ClusterReference: &nutanixv3.Reference{ - Kind: ptr.To(nutanixtypes.ClusterKind), - UUID: cluster.Metadata.UUID, - }, - }, - } - - resources := &nutanixv3.VMResources{ - PowerState: ptr.To("ON"), - NumSockets: ptr.To(conf.CPUs), - MemorySizeMib: ptr.To(conf.MemoryMB), - NicList: nicList, - DiskList: []*nutanixv3.VMDisk{ - { - DeviceProperties: &nutanixv3.VMDiskDeviceProperties{ - DeviceType: ptr.To("DISK"), - DiskAddress: &nutanixv3.DiskAddress{ - DeviceIndex: ptr.To(int64(0)), - AdapterType: ptr.To("SCSI"), - }, - }, - DataSourceReference: &nutanixv3.Reference{ - Kind: ptr.To(nutanixtypes.ImageKind), - UUID: image.Metadata.UUID, - }, - }, - }, - GuestCustomization: &nutanixv3.GuestCustomization{ - CloudInit: &nutanixv3.GuestCustomizationCloudInit{ - UserData: ptr.To(base64.StdEncoding.EncodeToString([]byte(userdata))), - }, - }, - } - - if conf.ProjectName != "" { - project, err := getProjectByName(ctx, client, conf.ProjectName) - if err != nil { - return nil, fmt.Errorf("failed to get project: %w", err) - } - - request.Metadata.ProjectReference = &nutanixv3.Reference{ - Kind: ptr.To(nutanixtypes.ProjectKind), - UUID: project.Metadata.UUID, - } - } - - if conf.CPUCores != nil { - resources.NumVcpusPerSocket = conf.CPUCores - } - - if conf.CPUPassthrough != nil { - resources.EnableCPUPassthrough = conf.CPUPassthrough - } - - if conf.DiskSizeGB != nil { - resources.DiskList[0].DiskSizeMib = ptr.To(*conf.DiskSizeGB * 1024) - } - - request.Spec.Resources = resources - - resp, err := client.Prism.V3.CreateVM(ctx, request) - if err != nil { - return nil, wrapNutanixError(err) - } - - taskUUID := resp.Status.ExecutionContext.TaskUUID.(string) - - if err := waitForCompletion(ctx, client, taskUUID, time.Second*10, time.Minute*15); err != nil { - return nil, fmt.Errorf("failed to wait for task: %w", err) - } - - if resp.Metadata.UUID == nil { - return nil, errors.New("did not get response with UUID") - } - - if err := waitForPowerState(ctx, client, *resp.Metadata.UUID, time.Second*10, time.Minute*10); err != nil { - return nil, fmt.Errorf("failed to wait for power state: %w", err) - } - - vm, err := client.Prism.V3.GetVM(ctx, *resp.Metadata.UUID) - if err != nil { - return nil, wrapNutanixError(err) - } - - if vm.Status.Name == nil { - return nil, fmt.Errorf("request for VM UUID '%s' did not return name", *resp.Metadata.UUID) - } - - addresses, err := getIPs(ctx, client, *vm.Metadata.UUID, time.Second*5, time.Minute*10) - if err != nil { - return nil, fmt.Errorf("failed to get addresses: %w", err) - } - - return Server{ - name: *vm.Status.Name, - id: *resp.Metadata.UUID, - status: instance.StatusRunning, - addresses: addresses, - }, nil -} - -func getSubnetByName(ctx context.Context, client *ClientSet, name, clusterID string) (*nutanixv3.SubnetIntentResponse, error) { - filter := fmt.Sprintf("name==%s", name) - subnets, err := client.Prism.V3.ListAllSubnet(ctx, filter, nil) - - if err != nil { - return nil, wrapNutanixError(err) - } - - for _, subnet := range subnets.Entities { - if subnet != nil && subnet.Status != nil && subnet.Status.Name != nil && *subnet.Status.Name == name { - // some subnet types (e.g. VPC overlays) do not come with a cluster reference; we don't need to check them - if subnet.Status.ClusterReference == nil || (subnet.Status.ClusterReference.UUID != nil && *subnet.Status.ClusterReference.UUID == clusterID) { - return subnet, nil - } - } - } - - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("no subnet found for name==%s", name), - } -} - -func getProjectByName(ctx context.Context, client *ClientSet, name string) (*nutanixv3.Project, error) { - filter := fmt.Sprintf("name==%s", name) - projects, err := client.Prism.V3.ListAllProject(ctx, filter) - - if err != nil { - return nil, wrapNutanixError(err) - } - - if projects == nil || projects.Entities == nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("no project found for name==%s", name), - } - } - - for _, project := range projects.Entities { - if project != nil && project.Status != nil && project.Status.Name == name { - return project, nil - } - } - - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("no project found for name==%s", name), - } -} - -func getClusterByName(ctx context.Context, client *ClientSet, name string) (*nutanixv3.ClusterIntentResponse, error) { - filter := fmt.Sprintf("name==%s", name) - clusters, err := client.Prism.V3.ListAllCluster(ctx, filter) - - if err != nil { - return nil, wrapNutanixError(err) - } - - if clusters == nil || clusters.Entities == nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("no cluster found for name==%s", name), - } - } - - for _, cluster := range clusters.Entities { - if cluster.Status != nil && cluster.Status.Name != nil && *cluster.Status.Name == name { - return cluster, nil - } - } - - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("no cluster found for name==%s", name), - } -} - -func getImageByName(ctx context.Context, client *ClientSet, name string) (*nutanixv3.ImageIntentResponse, error) { - filter := fmt.Sprintf("name==%s", name) - images, err := client.Prism.V3.ListAllImage(ctx, filter) - - if err != nil { - return nil, wrapNutanixError(err) - } - - if images == nil || images.Entities == nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("no image found for name==%s", name), - } - } - - for _, image := range images.Entities { - if image.Status != nil && image.Status.Name != nil && *image.Status.Name == name { - return image, nil - } - } - - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("no image found for name==%s", name), - } -} - -func getVMByName(ctx context.Context, client *ClientSet, name string, projectID *string) (*nutanixv3.VMIntentResource, error) { - filter := fmt.Sprintf("vm_name==%s", name) - vms, err := client.Prism.V3.ListAllVM(ctx, filter) - - if err != nil { - return nil, wrapNutanixError(err) - } - - for _, vm := range vms.Entities { - if *vm.Status.Name == name { - if projectID != nil && vm.Metadata != nil && vm.Metadata.ProjectReference != nil && - vm.Metadata.ProjectReference.UUID != nil && *vm.Metadata.ProjectReference.UUID != *projectID { - continue - } - return vm, nil - } - } - - return nil, cloudprovidererrors.ErrInstanceNotFound -} - -func getIPs(ctx context.Context, client *ClientSet, vmID string, interval time.Duration, timeout time.Duration) (map[string]corev1.NodeAddressType, error) { - addresses := make(map[string]corev1.NodeAddressType) - - err := wait.PollUntilContextTimeout(ctx, interval, timeout, false, func(ctx context.Context) (bool, error) { - vm, err := client.Prism.V3.GetVM(ctx, vmID) - if err != nil { - return false, wrapNutanixError(err) - } - - if len(vm.Status.Resources.NicList) == 0 || len(vm.Status.Resources.NicList[0].IPEndpointList) == 0 { - return false, nil - } - - ip := *vm.Status.Resources.NicList[0].IPEndpointList[0].IP - addresses[ip] = corev1.NodeInternalIP - - return true, nil - }) - if err != nil { - return map[string]corev1.NodeAddressType{}, err - } - - return addresses, nil -} - -func waitForCompletion(ctx context.Context, client *ClientSet, taskID string, interval time.Duration, timeout time.Duration) error { - return wait.PollUntilContextTimeout(ctx, interval, timeout, false, func(ctx context.Context) (bool, error) { - task, err := client.Prism.V3.GetTask(ctx, taskID) - if err != nil { - return false, wrapNutanixError(err) - } - - if task.Status == nil { - return false, nil - } - - switch *task.Status { - case "INVALID_UUID", "FAILED": - return false, fmt.Errorf("bad status: %s", *task.Status) - case "QUEUED", "RUNNING": - return false, nil - case "SUCCEEDED": - return true, nil - default: - return false, fmt.Errorf("unknown status: %s", *task.Status) - } - }) -} - -func waitForPowerState(ctx context.Context, client *ClientSet, vmID string, interval time.Duration, timeout time.Duration) error { - return wait.PollUntilContextTimeout(ctx, interval, timeout, false, func(ctx context.Context) (bool, error) { - vm, err := client.Prism.V3.GetVM(ctx, vmID) - if err != nil { - return false, wrapNutanixError(err) - } - - if vm.Status == nil || vm.Status.Resources == nil || vm.Status.Resources.PowerState == nil { - return false, nil - } - - switch *vm.Status.Resources.PowerState { - case "ON": - return true, nil - case "OFF": - return false, nil - default: - return false, fmt.Errorf("unexpected power state: %s", *vm.Status.Resources.PowerState) - } - }) -} - -func wrapNutanixError(initialErr error) error { - if initialErr == nil { - return nil - } - - var resp nutanixtypes.ErrorResponse - - if err := json.Unmarshal([]byte(initialErr.Error()), &resp); err != nil { - // invalid credentials are returned with a simple string - if strings.Contains(initialErr.Error(), invalidCredentials) { - return cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: initialErr.Error(), - } - } - - // failed to parse error, let's make sure it doesnt't have new lines at least - return fmt.Errorf("api error: %s", strings.ReplaceAll(initialErr.Error(), "\n", "")) - } - - // TODO: handle different states by potentially returning a TerminalError - // this needs experience with errors coming from Nutanix because the state - // values are not defined anywhere. So if you hit an error that qualifies, - // why not add something handling it! - switch resp.State { - default: - var msgs []string - for _, msg := range resp.MessageList { - msgs = append(msgs, fmt.Sprintf("%s: %s", msg.Message, msg.Reason)) - } - - return fmt.Errorf("api error (%s, code %d): %s", resp.State, resp.Code, strings.Join(msgs, ", ")) - } -} diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go deleted file mode 100644 index 6989f0ebc..000000000 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ /dev/null @@ -1,454 +0,0 @@ -/* -Copyright 2021 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package nutanix - -import ( - "context" - "errors" - "fmt" - "strconv" - "time" - - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - nutanixtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/nutanix/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - - corev1 "k8s.io/api/core/v1" - ktypes "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/ptr" -) - -type Config struct { - Endpoint string - Port *int - Username string - Password string - AllowInsecure bool - ProxyURL string - - ClusterName string - ProjectName string - SubnetName string - AdditionalSubnetNames []string - ImageName string - - Categories map[string]string - - CPUs int64 - CPUCores *int64 - CPUPassthrough *bool - MemoryMB int64 - DiskSizeGB *int64 -} - -type provider struct { - configVarResolver *providerconfig.ConfigPointerVarResolver -} - -// Server holds Nutanix server information. -type Server struct { - name string - id string - status instance.Status - addresses map[string]corev1.NodeAddressType -} - -// Ensures that Server implements Instance interface. -var _ instance.Instance = &Server{} - -// Ensures that provider implements Provider interface. -var _ cloudprovidertypes.Provider = &provider{} - -func (nutanixServer Server) Name() string { - return nutanixServer.name -} - -func (nutanixServer Server) ID() string { - return nutanixServer.id -} - -func (nutanixServer Server) ProviderID() string { - if nutanixServer.ID() == "" { - return "" - } - return fmt.Sprintf("nutanix://%s", nutanixServer.ID()) -} - -func (nutanixServer Server) HostID() string { - return "" -} - -func (nutanixServer Server) Addresses() map[string]corev1.NodeAddressType { - return nutanixServer.addresses -} - -func (nutanixServer Server) Status() instance.Status { - return nutanixServer.status -} - -// New returns a nutanix provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - provider := &provider{configVarResolver: &providerconfig.ConfigPointerVarResolver{Cvr: configVarResolver}} - return provider -} - -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *nutanixtypes.RawConfig, error) { - if provSpec.Value == nil { - return nil, nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - - pconfig, err := providerconfigtypes.GetConfig(provSpec) - if err != nil { - return nil, nil, nil, err - } - - if pconfig.OperatingSystemSpec.Raw == nil { - return nil, nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") - } - - rawConfig, err := nutanixtypes.GetConfig(*pconfig) - if err != nil { - return nil, nil, nil, err - } - - c := Config{} - - c.Endpoint, err = p.configVarResolver.GetConfigVarStringValueOrEnv(&rawConfig.Endpoint, "NUTANIX_ENDPOINT") - if err != nil { - return nil, nil, nil, err - } - - port, err := p.configVarResolver.GetConfigVarStringValueOrEnv(&rawConfig.Port, "NUTANIX_PORT") - if err != nil { - return nil, nil, nil, err - } - - if port != "" { - // we parse the port into an int to make sure we're being passed a somewhat valid port value - portInt, err := strconv.Atoi(port) - if err != nil { - return nil, nil, nil, err - } - c.Port = ptr.To(portInt) - } - - c.Username, err = p.configVarResolver.GetConfigVarStringValueOrEnv(&rawConfig.Username, "NUTANIX_USERNAME") - if err != nil { - return nil, nil, nil, err - } - - c.Password, err = p.configVarResolver.GetConfigVarStringValueOrEnv(&rawConfig.Password, "NUTANIX_PASSWORD") - if err != nil { - return nil, nil, nil, err - } - - c.AllowInsecure, err = p.configVarResolver.GetConfigVarBoolValueOrEnv(&rawConfig.AllowInsecure, "NUTANIX_INSECURE") - if err != nil { - return nil, nil, nil, err - } - - c.ProxyURL, err = p.configVarResolver.GetConfigVarStringValueOrEnv(&rawConfig.ProxyURL, "NUTANIX_PROXY_URL") - if err != nil { - return nil, nil, nil, err - } - - c.ClusterName, err = p.configVarResolver.GetConfigVarStringValueOrEnv(&rawConfig.ClusterName, "NUTANIX_CLUSTER_NAME") - if err != nil { - return nil, nil, nil, err - } - - if rawConfig.ProjectName != nil { - c.ProjectName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ProjectName) - if err != nil { - return nil, nil, nil, err - } - } - - c.SubnetName, err = p.configVarResolver.GetConfigVarStringValue(&rawConfig.SubnetName) - if err != nil { - return nil, nil, nil, err - } - - c.AdditionalSubnetNames = append(c.AdditionalSubnetNames, rawConfig.AdditionalSubnetNames...) - - c.ImageName, err = p.configVarResolver.GetConfigVarStringValue(&rawConfig.ImageName) - if err != nil { - return nil, nil, nil, err - } - - c.Categories = rawConfig.Categories - - c.CPUs = rawConfig.CPUs - c.CPUCores = rawConfig.CPUCores - c.CPUPassthrough = rawConfig.CPUPassthrough - c.MemoryMB = rawConfig.MemoryMB - c.DiskSizeGB = rawConfig.DiskSize - - return &c, pconfig, rawConfig, nil -} - -func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { - return spec, nil -} - -func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { - config, _, _, err := p.getConfig(spec.ProviderSpec) - if err != nil { - return fmt.Errorf("failed to parse machineSpec: %w", err) - } - - client, err := GetClientSet(config) - if err != nil { - return fmt.Errorf("failed to construct client: %w", err) - } - - cluster, err := getClusterByName(ctx, client, config.ClusterName) - if err != nil { - return fmt.Errorf("failed to get cluster: %w", err) - } - - if config.ProjectName != "" { - if _, err := getProjectByName(ctx, client, config.ProjectName); err != nil { - return fmt.Errorf("failed to get project: %w", err) - } - } - - if _, err := getSubnetByName(ctx, client, config.SubnetName, *cluster.Metadata.UUID); err != nil { - return fmt.Errorf("failed to get subnet: %w", err) - } - - for _, subnet := range config.AdditionalSubnetNames { - if _, err := getSubnetByName(ctx, client, subnet, *cluster.Metadata.UUID); err != nil { - return fmt.Errorf("failed to get subnet: %w", err) - } - } - - image, err := getImageByName(ctx, client, config.ImageName) - if err != nil { - return fmt.Errorf("failed to get image: %w", err) - } - - var imageSizeBytes int64 - - if image.Status != nil && image.Status.Resources.SizeBytes != nil { - imageSizeBytes = *image.Status.Resources.SizeBytes - } else { - return fmt.Errorf("failed to read image size for '%s'", config.ImageName) - } - - if config.DiskSizeGB != nil && *config.DiskSizeGB*1024*1024*1024 < imageSizeBytes { - return fmt.Errorf("requested disk size (%d bytes) is smaller than image size (%d bytes)", *config.DiskSizeGB*1024*1024*1024, *image.Status.Resources.SizeBytes) - } - - return nil -} - -func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { - vm, err := p.create(ctx, machine, userdata) - if err != nil { - _, cleanupErr := p.Cleanup(ctx, log, machine, data) - if cleanupErr != nil { - return nil, fmt.Errorf("cleaning up failed with err %w after creation failed with err %w", cleanupErr, err) - } - return nil, err - } - return vm, nil -} - -func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { - config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("failed to parse machineSpec: %v", err), - } - } - - client, err := GetClientSet(config) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("failed to construct client: %v", err), - } - } - - return createVM(ctx, client, machine.Name, *config, userdata) -} - -func (p *provider) Cleanup(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { - return p.cleanup(ctx, machine, data) -} - -func (p *provider) cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { - config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return false, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("failed to parse machineSpec: %v", err), - } - } - - client, err := GetClientSet(config) - if err != nil { - return false, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("failed to construct client: %v", err), - } - } - - var projectID *string - - if config.ProjectName != "" { - project, err := getProjectByName(ctx, client, config.ProjectName) - if err != nil { - return false, err - } - - projectID = project.Metadata.UUID - } - - vm, err := getVMByName(ctx, client, machine.Name, projectID) - if err != nil { - if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { - // VM is gone already - return true, nil - } - - return false, err - } - - if vm.Metadata == nil || vm.Metadata.UUID == nil { - return false, fmt.Errorf("failed to get valid VM metadata for machine '%s'", machine.Name) - } - - // TODO: figure out if VM is already in deleting state - - resp, err := client.Prism.V3.DeleteVM(ctx, *vm.Metadata.UUID) - if err != nil { - return false, err - } - - taskID, ok := resp.Status.ExecutionContext.TaskUUID.(string) - if !ok { - return false, errors.New("failed to parse deletion task UUID") - } - - if err := waitForCompletion(ctx, client, taskID, time.Second*5, time.Minute*10); err != nil { - return false, fmt.Errorf("failed to wait for completion: %w", err) - } - - return true, nil -} - -func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("failed to parse machineSpec: %v", err), - } - } - - client, err := GetClientSet(config) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("failed to construct client: %v", err), - } - } - - var projectID *string - - if config.ProjectName != "" { - project, err := getProjectByName(ctx, client, config.ProjectName) - if err != nil { - return nil, err - } - - projectID = project.Metadata.UUID - } - - vm, err := getVMByName(ctx, client, machine.Name, projectID) - if err != nil { - return nil, err - } - - if vm.Status == nil || vm.Status.Resources == nil || vm.Status.Resources.PowerState == nil { - return nil, fmt.Errorf("could not read power state for VM '%s'", machine.Name) - } - - var status instance.Status - - switch *vm.Status.Resources.PowerState { - case "ON": - status = instance.StatusRunning - case "OFF": - status = instance.StatusCreating - default: - status = instance.StatusUnknown - } - - addresses := make(map[string]corev1.NodeAddressType) - - if len(vm.Status.Resources.NicList) > 0 && len(vm.Status.Resources.NicList[0].IPEndpointList) > 0 { - ip := *vm.Status.Resources.NicList[0].IPEndpointList[0].IP - addresses[ip] = corev1.NodeInternalIP - } else { - return nil, fmt.Errorf("could not find any IP addresses for VM '%s'", machine.Name) - } - - return Server{ - name: *vm.Status.Name, - id: *vm.Metadata.UUID, - status: status, - addresses: addresses, - }, nil -} - -func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ ktypes.UID) error { - return nil -} - -// GetCloudConfig returns an empty cloud configuration for Nutanix as no CCM exists. -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return "", "", nil -} - -func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { - labels := make(map[string]string) - - config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return labels, fmt.Errorf("failed to parse config: %w", err) - } - - labels["size"] = fmt.Sprintf("%d-cpus-%d-mb", config.CPUs, config.MemoryMB) - labels["cluster"] = config.ClusterName - - return labels, nil -} - -func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { - return nil -} diff --git a/pkg/cloudprovider/provider/nutanix/types/types.go b/pkg/cloudprovider/provider/nutanix/types/types.go deleted file mode 100644 index a2283b721..000000000 --- a/pkg/cloudprovider/provider/nutanix/types/types.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -Copyright 2021 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" -) - -const ( - VMKind = "vm" - ProjectKind = "project" - ClusterKind = "cluster" - SubnetKind = "subnet" - DiskKind = "disk" - ImageKind = "image" -) - -type RawConfig struct { - Endpoint providerconfigtypes.ConfigVarString `json:"endpoint"` - Port providerconfigtypes.ConfigVarString `json:"port"` - Username providerconfigtypes.ConfigVarString `json:"username"` - Password providerconfigtypes.ConfigVarString `json:"password"` - AllowInsecure providerconfigtypes.ConfigVarBool `json:"allowInsecure"` - ProxyURL providerconfigtypes.ConfigVarString `json:"proxyURL,omitempty"` - - ClusterName providerconfigtypes.ConfigVarString `json:"clusterName"` - ProjectName *providerconfigtypes.ConfigVarString `json:"projectName,omitempty"` - SubnetName providerconfigtypes.ConfigVarString `json:"subnetName"` - AdditionalSubnetNames []string `json:"additionalSubnetNames,omitempty"` - ImageName providerconfigtypes.ConfigVarString `json:"imageName"` - - // VM sizing configuration - CPUs int64 `json:"cpus"` - CPUCores *int64 `json:"cpuCores,omitempty"` - CPUPassthrough *bool `json:"cpuPassthrough,omitempty"` - MemoryMB int64 `json:"memoryMB"` - DiskSize *int64 `json:"diskSize,omitempty"` - - // Metadata related configuration - Categories map[string]string `json:"categories,omitempty"` -} - -type ErrorResponse struct { - APIVersion string `json:"api_version"` - Kind string `json:"kind"` - State string `json:"state"` - MessageList []ErrorResponseMsg `json:"message_list"` - Code int32 `json:"code"` -} - -type ErrorResponseMsg struct { - Message string `json:"message"` - Reason string `json:"reason"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/opennebula/provider.go b/pkg/cloudprovider/provider/opennebula/provider.go deleted file mode 100644 index 41ab9b6c0..000000000 --- a/pkg/cloudprovider/provider/opennebula/provider.go +++ /dev/null @@ -1,471 +0,0 @@ -/* -Copyright 2022 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package opennebula - -import ( - "context" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "strconv" - "strings" - - "github.com/OpenNebula/one/src/oca/go/src/goca" - "github.com/OpenNebula/one/src/oca/go/src/goca/schemas/shared" - "github.com/OpenNebula/one/src/oca/go/src/goca/schemas/vm" - "github.com/OpenNebula/one/src/oca/go/src/goca/schemas/vm/keys" - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - opennebulatypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/opennebula/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" -) - -type provider struct { - configVarResolver *providerconfig.ConfigVarResolver -} - -type CloudProviderSpec struct { - PassValidation bool `json:"passValidation"` -} - -const ( - machineUIDContextKey = "K8S_MACHINE_UID" -) - -// New returns a OpenNebula provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return &provider{configVarResolver: configVarResolver} -} - -type Config struct { - // Auth details - Username string - Password string - Endpoint string - - // Machine details - CPU *float64 - VCPU *int - Memory *int - Image string - Datastore string - DiskSize *int - Network string - EnableVNC bool - VMTemplateExtra map[string]string -} - -func getClient(config *Config) *goca.Client { - return goca.NewDefaultClient(goca.NewConfig(config.Username, config.Password, config.Endpoint)) -} - -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - - pconfig, err := providerconfigtypes.GetConfig(provSpec) - if err != nil { - return nil, nil, err - } - - rawConfig, err := opennebulatypes.GetConfig(*pconfig) - if err != nil { - return nil, nil, err - } - - c := Config{} - c.Username, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Username, "ONE_USERNAME") - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"username\" field, error = %w", err) - } - - c.Password, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Password, "ONE_PASSWORD") - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"password\" field, error = %w", err) - } - - c.Endpoint, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Endpoint, "ONE_ENDPOINT") - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"endpoint\" field, error = %w", err) - } - - c.CPU = rawConfig.CPU - - c.VCPU = rawConfig.VCPU - - c.Memory = rawConfig.Memory - - c.Image, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Image) - if err != nil { - return nil, nil, err - } - - c.Datastore, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Datastore) - if err != nil { - return nil, nil, err - } - - c.DiskSize = rawConfig.DiskSize - - c.Network, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Network) - if err != nil { - return nil, nil, err - } - - c.EnableVNC, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.EnableVNC) - if err != nil { - return nil, nil, err - } - - c.VMTemplateExtra = rawConfig.VMTemplateExtra - - return &c, pconfig, err -} - -func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { - _, pc, err := p.getConfig(spec.ProviderSpec) - if err != nil { - return fmt.Errorf("failed to parse config: %w", err) - } - - opennebulaCloudProviderSpec := CloudProviderSpec{} - if err = json.Unmarshal(pc.CloudProviderSpec.Raw, &opennebulaCloudProviderSpec); err != nil { - return fmt.Errorf("failed to parse cloud provider spec: %w", err) - } - - return nil -} - -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (string, string, error) { - return "", "", nil -} - -func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - - client := getClient(c) - - // build a template - tpl := vm.NewTemplate() - - // add extra template vars first - for key, value := range c.VMTemplateExtra { - tpl.Add(keys.Template(key), value) - } - - tpl.Add(keys.Name, machine.Spec.Name) - tpl.CPU(*c.CPU).Memory(*c.Memory).VCPU(*c.VCPU) - - disk := tpl.AddDisk() - disk.Add(shared.Image, c.Image) - disk.Add(shared.Datastore, c.Datastore) - disk.Add(shared.Size, *c.DiskSize) - - nic := tpl.AddNIC() - nic.Add(shared.Network, c.Network) - nic.Add(shared.Model, "virtio") - - if c.EnableVNC { - err = tpl.AddIOGraphic(keys.GraphicType, "VNC") - if err != nil { - return nil, fmt.Errorf("failed to add graphic type to iographic in template: %w", err) - } - err = tpl.AddIOGraphic(keys.Listen, "0.0.0.0") - if err != nil { - return nil, fmt.Errorf("failed to add listen address to iographic in template: %w", err) - } - } - - err = tpl.AddCtx(keys.NetworkCtx, "YES") - if err != nil { - return nil, fmt.Errorf("failed to add network to context in template: %w", err) - } - err = tpl.AddCtx(keys.SSHPubKey, "$USER[SSH_PUBLIC_KEY]") - if err != nil { - return nil, fmt.Errorf("failed to add SSH public key to context in template: %w", err) - } - - err = tpl.AddCtx(machineUIDContextKey, string(machine.UID)) - if err != nil { - return nil, fmt.Errorf("failed to add machine UID to context in template: %w", err) - } - err = tpl.AddCtx("USER_DATA", base64.StdEncoding.EncodeToString([]byte(userdata))) - if err != nil { - return nil, fmt.Errorf("failed to add user data to context in template: %w", err) - } - err = tpl.AddCtx("USER_DATA_ENCODING", "base64") - if err != nil { - return nil, fmt.Errorf("failed to add user data encoding to context in template: %w", err) - } - err = tpl.AddCtx("SET_HOSTNAME", machine.Spec.Name) - if err != nil { - return nil, fmt.Errorf("failed to add desired hostname to context in template: %w", err) - } - - controller := goca.NewController(client) - - // create VM from the generated template above - vmID, err := controller.VMs().Create(tpl.String(), false) - if err != nil { - return nil, fmt.Errorf("failed to create VM: %w", err) - } - - vm, err := controller.VM(vmID).Info(false) - if err != nil { - return nil, fmt.Errorf("failed to fetch VM information: %w", err) - } - - return &openNebulaInstance{vm}, nil -} - -func (p *provider) Cleanup(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { - instance, err := p.get(machine) - if err != nil { - if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { - return true, nil - } - return false, err - } - - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return false, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - - client := getClient(c) - controller := goca.NewController(client) - - vmctrl := controller.VM(instance.vm.ID) - err = vmctrl.TerminateHard() - // ignore error of nonexistent machines by matching for "NO_EXISTS", the error string is something like "OpenNebula error [NO_EXISTS]: [one.vm.action] Error getting virtual machine [999914743]." - if err != nil && !strings.Contains(err.Error(), "NO_EXISTS") { - return false, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("failed to delete virtual machine, due to %v", err), - } - } - - return true, nil -} - -func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - return p.get(machine) -} - -func (p *provider) get(machine *clusterv1alpha1.Machine) (*openNebulaInstance, error) { - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - - client := getClient(c) - controller := goca.NewController(client) - - vmPool, err := controller.VMs().Info() - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("failed to list virtual machines, due to %v", err), - } - } - - // first collect all IDs, the vm infos in the vmPool don't contain the context which has the uid - var vmIDs []int - for _, vm := range vmPool.VMs { - if vm.Name != machine.Spec.Name { - continue - } - - vmIDs = append(vmIDs, vm.ID) - } - - // go over each vm that matches the name and check if the uid is the same - for _, vmID := range vmIDs { - vm, err := controller.VM(vmID).Info(false) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("failed to get info for VM %v, due to %v", vmID, err), - } - } - - uid, err := vm.Template.GetCtx(machineUIDContextKey) - if err != nil { - // ignore errors like "key blabla not found" - continue - } - - if uid == string(machine.UID) { - return &openNebulaInstance{vm}, nil - } - } - - return nil, cloudprovidererrors.ErrInstanceNotFound -} - -func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { - return spec, nil -} - -func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - - instance, err := p.get(machine) - if err != nil { - return cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to get instance, due to %v", err), - } - } - - client := getClient(c) - - // get current template - tpl := &instance.vm.Template - contextVector, err := tpl.GetVector(keys.ContextVec) - if err != nil { - return cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to get VM template context vector, due to %v", err), - } - } - - // replace the old uid in context with the new one - contextVector.Del(machineUIDContextKey) - err = contextVector.AddPair(machineUIDContextKey, string(newUID)) - if err != nil { - return fmt.Errorf("failed to add the new machine UID to context in template: %w", err) - } - - // create a new template that only has the context vector in it so it gets properly replaced - tpl = vm.NewTemplate() - for _, pair := range contextVector.Pairs { - key := pair.XMLName.Local - value := pair.Value - err = tpl.AddCtx(keys.Context(key), value) - if err != nil { - return fmt.Errorf("failed to add %s to context in template: %w", key, err) - } - } - - // finally, update the VM template - controller := goca.NewController(client) - vmCtrl := controller.VM(instance.vm.ID) - err = vmCtrl.UpdateConf(tpl.String()) - if err != nil { - return cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to update VM template, due to %v", err), - } - } - - return nil -} - -func (p *provider) MachineMetricsLabels(_ *clusterv1alpha1.Machine) (map[string]string, error) { - return map[string]string{}, nil -} - -func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { - return nil -} - -type openNebulaInstance struct { - vm *vm.VM -} - -func (i *openNebulaInstance) Name() string { - return i.vm.Name -} - -func (i *openNebulaInstance) ID() string { - return strconv.Itoa(i.vm.ID) -} - -func (i *openNebulaInstance) ProviderID() string { - if i.vm == nil || i.vm.ID == 0 { - return "" - } - return "opennebula://" + strconv.Itoa(i.vm.ID) -} - -func (i *openNebulaInstance) HostID() string { - return i.ID() -} - -func (i *openNebulaInstance) Addresses() map[string]v1.NodeAddressType { - addresses := map[string]v1.NodeAddressType{} - - for _, nic := range i.vm.Template.GetNICs() { - ip, _ := nic.Get(shared.IP) - addresses[ip] = v1.NodeInternalIP - } - - return addresses -} - -func (i *openNebulaInstance) Status() instance.Status { - // state is the general state of the VM, lcmState is the state of the life-cycle manager of the VM - // lcmState is anything else other than LcmInit when the VM's state is Active - state, lcmState, _ := i.vm.State() - switch state { - case vm.Init, vm.Pending, vm.Hold: - return instance.StatusCreating - case vm.Active: - switch lcmState { - case vm.LcmInit, vm.Prolog, vm.Boot: - return instance.StatusCreating - case vm.Epilog: - return instance.StatusDeleting - default: - return instance.StatusRunning - } - case vm.Done: - return instance.StatusDeleted - default: - return instance.StatusUnknown - } -} diff --git a/pkg/cloudprovider/provider/opennebula/types/types.go b/pkg/cloudprovider/provider/opennebula/types/types.go deleted file mode 100644 index 6e69755fc..000000000 --- a/pkg/cloudprovider/provider/opennebula/types/types.go +++ /dev/null @@ -1,46 +0,0 @@ -/* -Copyright 2022 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" -) - -type RawConfig struct { - // Auth details - Username providerconfigtypes.ConfigVarString `json:"username,omitempty"` - Password providerconfigtypes.ConfigVarString `json:"password,omitempty"` - Endpoint providerconfigtypes.ConfigVarString `json:"endpoint,omitempty"` - - // Machine details - CPU *float64 `json:"cpu"` - VCPU *int `json:"vcpu"` - Memory *int `json:"memory"` - Image providerconfigtypes.ConfigVarString `json:"image"` - Datastore providerconfigtypes.ConfigVarString `json:"datastore"` - DiskSize *int `json:"diskSize"` - Network providerconfigtypes.ConfigVarString `json:"network"` - EnableVNC providerconfigtypes.ConfigVarBool `json:"enableVNC"` - VMTemplateExtra map[string]string `json:"vmTemplateExtra,omitempty"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/scaleway/provider.go b/pkg/cloudprovider/provider/scaleway/provider.go deleted file mode 100644 index 7debf514f..000000000 --- a/pkg/cloudprovider/provider/scaleway/provider.go +++ /dev/null @@ -1,458 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package scaleway - -import ( - "context" - "errors" - "fmt" - "strings" - - "github.com/scaleway/scaleway-sdk-go/api/instance/v1" - "github.com/scaleway/scaleway-sdk-go/scw" - "github.com/scaleway/scaleway-sdk-go/validation" - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - cloudInstance "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - scalewaytypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/scaleway/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" -) - -type provider struct { - configVarResolver *providerconfig.ConfigPointerVarResolver -} - -// New returns a Scaleway provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return &provider{configVarResolver: &providerconfig.ConfigPointerVarResolver{Cvr: configVarResolver}} -} - -type Config struct { - AccessKey string - SecretKey string - ProjectID string - Zone string - CommercialType string - IPv6 bool - Tags []string -} - -func (c *Config) getInstanceAPI() (*instance.API, error) { - client, err := scw.NewClient( - scw.WithAuth(c.AccessKey, c.SecretKey), - scw.WithDefaultZone(scw.Zone(c.Zone)), - scw.WithDefaultProjectID(c.ProjectID), - scw.WithUserAgent("kubermatic/machine-controller"), - ) - if err != nil { - return nil, fmt.Errorf("failed to initialize the scaleway client: %w", err) - } - - return instance.NewAPI(client), nil -} - -func getImageNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { - switch os { - case providerconfigtypes.OperatingSystemUbuntu: - // ubuntu_focal doesn't work (see https://bugs.launchpad.net/ubuntu/+source/linux-kvm/+bug/1880522) - // modprobe ip_vs will fail - return "ubuntu_bionic", nil - case providerconfigtypes.OperatingSystemCentOS: - return "centos_7.6", nil - } - return "", providerconfigtypes.ErrOSNotSupported -} - -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - - pconfig, err := providerconfigtypes.GetConfig(provSpec) - if err != nil { - return nil, nil, err - } - - if pconfig.OperatingSystemSpec.Raw == nil { - return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") - } - - rawConfig, err := scalewaytypes.GetConfig(*pconfig) - if err != nil { - return nil, nil, err - } - - c := Config{} - c.AccessKey, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.AccessKey, scw.ScwAccessKeyEnv) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"access_key\" field, error = %w", err) - } - c.SecretKey, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.SecretKey, scw.ScwSecretKeyEnv) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"secret_key\" field, error = %w", err) - } - c.ProjectID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ProjectID) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"project_id\" field, error = %w", err) - } - c.Zone, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Zone) - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"zone\" field, error = %w", err) - } - c.CommercialType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.CommercialType) - if err != nil { - return nil, nil, err - } - c.IPv6, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.IPv6) - if err != nil { - return nil, nil, err - } - c.Tags = rawConfig.Tags - - return &c, pconfig, err -} - -func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { - return spec, nil -} - -func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { - c, pc, err := p.getConfig(spec.ProviderSpec) - if err != nil { - return fmt.Errorf("failed to parse config: %w", err) - } - - if !validation.IsAccessKey(c.AccessKey) { - return fmt.Errorf("invalid access key format '%s', expected SCWXXXXXXXXXXXXXXXXX format", c.AccessKey) - } - if !validation.IsSecretKey(c.SecretKey) { - return fmt.Errorf("invalid secret key format '%s', expected a UUID: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", c.SecretKey) - } - if !validation.IsProjectID(c.ProjectID) { - return fmt.Errorf("invalid project ID format '%s', expected a UUID: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", c.ProjectID) - } - - _, err = scw.ParseZone(c.Zone) - if err != nil { - return err - } - - if c.CommercialType == "" { - return errors.New("commercial type is missing") - } - - _, err = getImageNameForOS(pc.OperatingSystem) - if err != nil { - return fmt.Errorf("invalid operating system specified %q: %w", pc.OperatingSystem, err) - } - - return nil -} - -func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (cloudInstance.Instance, error) { - c, pc, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - - api, err := c.getInstanceAPI() - if err != nil { - return nil, err - } - - imageName, err := getImageNameForOS(pc.OperatingSystem) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, invalid operating system specified %q: %v", pc.OperatingSystem, err), - } - } - createServerRequest := &instance.CreateServerRequest{ - Image: imageName, - Name: machine.Spec.Name, - CommercialType: c.CommercialType, - Tags: append(c.Tags, string(machine.UID)), - EnableIPv6: c.IPv6, - } - - serverResp, err := api.CreateServer(createServerRequest, scw.WithContext(ctx)) - if err != nil { - return nil, scalewayErrToTerminalError(err) - } - - err = api.SetServerUserData(&instance.SetServerUserDataRequest{ - Key: "cloud-init", - ServerID: serverResp.Server.ID, - Content: strings.NewReader(userdata), - }) - if err != nil { - return nil, scalewayErrToTerminalError(err) - } - - log.Debugw("Scaleway server got fully created", "server", serverResp.Server.ID) - - return &scwServer{server: serverResp.Server}, err -} - -func (p *provider) Cleanup(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { - i, err := p.get(machine) - if err != nil { - if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { - return true, nil - } - return false, err - } - - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return false, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - api, err := c.getInstanceAPI() - if err != nil { - return false, err - } - - _, err = api.ServerAction(&instance.ServerActionRequest{ - Action: instance.ServerActionTerminate, - ServerID: i.ID(), - }, scw.WithContext(ctx)) - if err != nil { - return false, scalewayErrToTerminalError(err) - } - - return false, nil -} - -func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (cloudInstance.Instance, error) { - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - - api, err := c.getInstanceAPI() - if err != nil { - return nil, err - } - - i, err := p.get(machine) - if err != nil { - return nil, err - } - - if i.server.State == instance.ServerStateStopped || i.server.State == instance.ServerStateStoppedInPlace { - _, err := api.ServerAction(&instance.ServerActionRequest{ - Action: instance.ServerActionPoweron, - ServerID: i.server.ID, - }) - if err != nil { - return nil, scalewayErrToTerminalError(err) - } - - return nil, fmt.Errorf("scaleway instance %s is in a stopped state, powering the instance on is in progress", i.Name()) - } - - return i, nil -} - -func (p *provider) get(machine *clusterv1alpha1.Machine) (*scwServer, error) { - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - - api, err := c.getInstanceAPI() - if err != nil { - return nil, err - } - - serversResp, err := api.ListServers(&instance.ListServersRequest{ - Name: scw.StringPtr(machine.Spec.Name), - Tags: []string{string(machine.UID)}, - }, scw.WithAllPages()) - if err != nil { - return nil, scalewayErrToTerminalError(err) - } - - for _, server := range serversResp.Servers { - if server.Name == machine.Spec.Name && sets.NewString(server.Tags...).Has(string(machine.UID)) { - return &scwServer{server: server}, nil - } - } - - return nil, cloudprovidererrors.ErrInstanceNotFound -} - -func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return fmt.Errorf("failed to decode providerconfig: %w", err) - } - api, err := c.getInstanceAPI() - if err != nil { - return err - } - - server, err := p.get(machine) - if err != nil { - return err - } - - oldTags := server.server.Tags - newTags := []string{string(newUID)} - for _, oldTag := range oldTags { - if oldTag != string(machine.UID) { - newTags = append(newTags, oldTag) - } - } - - _, err = api.UpdateServer(&instance.UpdateServerRequest{ - Tags: scw.StringsPtr(newTags), - ServerID: server.ID(), - }) - if err != nil { - return scalewayErrToTerminalError(err) - } - - return nil -} - -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return "", "", nil -} - -func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { - labels := make(map[string]string) - - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err == nil { - labels["commercial_type"] = c.CommercialType - labels["zone"] = c.Zone - } - - return labels, err -} - -type scwServer struct { - server *instance.Server -} - -func (s *scwServer) HostID() string { - return "" -} - -func (s *scwServer) Name() string { - return s.server.Name -} - -func (s *scwServer) ID() string { - return s.server.ID -} - -// TODO: Implement once we start supporting Scaleway CCM. -func (s *scwServer) ProviderID() string { - return "" -} - -func (s *scwServer) Addresses() map[string]corev1.NodeAddressType { - addresses := map[string]corev1.NodeAddressType{} - if s.server.PrivateIP != nil { - addresses[*s.server.PrivateIP] = corev1.NodeInternalIP - } - - if s.server.PublicIP != nil { - addresses[s.server.PublicIP.Address.String()] = corev1.NodeExternalIP - } - - if s.server.IPv6 != nil { - addresses[s.server.IPv6.Address.String()] = corev1.NodeExternalIP - } - - return addresses -} - -func (s *scwServer) Status() cloudInstance.Status { - switch s.server.State { - case instance.ServerStateStarting: - return cloudInstance.StatusCreating - case instance.ServerStateRunning: - return cloudInstance.StatusRunning - case instance.ServerStateStopping: - return cloudInstance.StatusDeleting - default: - return cloudInstance.StatusUnknown - } -} - -// scalewayErrToTerminalError judges if the given error -// can be qualified as a "terminal" error, for more info see v1alpha1.MachineStatus - -// if the given error doesn't qualify the error passed as -// an argument will be returned. -func scalewayErrToTerminalError(err error) error { - var deniedErr *scw.PermissionsDeniedError - var invalidArgErr *scw.InvalidArgumentsError - var outOfStackErr *scw.OutOfStockError - var quotaErr *scw.QuotasExceededError - - if errors.As(err, &deniedErr) { - return cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: "A request has been rejected due to invalid credentials which were taken from the MachineSpec", - } - } else if errors.As(err, &invalidArgErr) { - return cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: "A request has been rejected due to invalid arguments which were taken from the MachineSpec", - } - } else if errors.As(err, &outOfStackErr) { - return cloudprovidererrors.TerminalError{ - Reason: common.InsufficientResourcesMachineError, - Message: "A request has been rejected due to out of stocks", - } - } else if errors.As(err, "aErr) { - return cloudprovidererrors.TerminalError{ - Reason: common.InsufficientResourcesMachineError, - Message: "A request has been rejected due to insufficient quotas", - } - } - return err -} - -func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { - return nil -} diff --git a/pkg/cloudprovider/provider/scaleway/types/types.go b/pkg/cloudprovider/provider/scaleway/types/types.go deleted file mode 100644 index 495a165a6..000000000 --- a/pkg/cloudprovider/provider/scaleway/types/types.go +++ /dev/null @@ -1,38 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" -) - -type RawConfig struct { - AccessKey *providerconfigtypes.ConfigVarString `json:"accessKey,omitempty"` - SecretKey *providerconfigtypes.ConfigVarString `json:"secretKey,omitempty"` - ProjectID *providerconfigtypes.ConfigVarString `json:"projectId,omitempty"` - Zone *providerconfigtypes.ConfigVarString `json:"zone,omitempty"` - CommercialType *providerconfigtypes.ConfigVarString `json:"commercialType"` - IPv6 *providerconfigtypes.ConfigVarBool `json:"ipv6"` - Tags []string `json:"tags,omitempty"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/client.go b/pkg/cloudprovider/provider/vmwareclouddirector/client.go deleted file mode 100644 index c7f6e7878..000000000 --- a/pkg/cloudprovider/provider/vmwareclouddirector/client.go +++ /dev/null @@ -1,185 +0,0 @@ -/* -Copyright 2022 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vmwareclouddirector - -import ( - "errors" - "fmt" - "net/url" - "path" - "strings" - - "github.com/vmware/go-vcloud-director/v2/govcd" - "github.com/vmware/go-vcloud-director/v2/types/v56" - - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" -) - -type Client struct { - Auth *Auth - VCDClient *govcd.VCDClient -} - -func NewClient(username, password, apiToken, org, url, vdc string, allowInsecure bool) (*Client, error) { - client := Client{ - Auth: &Auth{ - Username: username, - Password: password, - APIToken: apiToken, - Organization: org, - URL: url, - VDC: vdc, - AllowInsecure: allowInsecure, - }, - } - - vcdClient, err := client.GetAuthenticatedClient() - if err != nil { - return nil, err - } - - client.VCDClient = vcdClient - return &client, nil -} - -func (c *Client) GetAuthenticatedClient() (*govcd.VCDClient, error) { - // Ensure that all required fields for authentication are provided - // Fail early, without any API calls, if some required field is missing. - if c.Auth == nil { - return nil, fmt.Errorf("authentication configuration not provided") - } - - // If API token is provided, use it for authentication. - if c.Auth.APIToken == "" { - if c.Auth.Username == "" { - return nil, fmt.Errorf("username not provided") - } - if c.Auth.Password == "" { - return nil, fmt.Errorf("password not provided") - } - } - if c.Auth.URL == "" { - return nil, fmt.Errorf("URL not provided") - } - if c.Auth.Organization == "" { - return nil, fmt.Errorf("organization name not provided") - } - - // Ensure that `/api` suffix exists in the cloud director URL. - apiEndpoint, err := url.Parse(c.Auth.URL) - if err != nil { - return nil, fmt.Errorf("failed to parse url '%s': %w", c.Auth.URL, err) - } - if !strings.HasSuffix(c.Auth.URL, "/api") { - apiEndpoint.Path = path.Join(apiEndpoint.Path, "api") - } - - vcdClient := govcd.NewVCDClient(*apiEndpoint, c.Auth.AllowInsecure) - - if c.Auth.APIToken != "" { - err = vcdClient.SetToken(c.Auth.Organization, govcd.ApiTokenHeader, c.Auth.APIToken) - if err != nil { - return nil, fmt.Errorf("failed to authenticate with VMware Cloud Director using API Token: %w", err) - } - return vcdClient, nil - } - - err = vcdClient.Authenticate(c.Auth.Username, c.Auth.Password, c.Auth.Organization) - if err != nil { - return nil, fmt.Errorf("failed to authenticate with VMware Cloud Director: %w", err) - } - - return vcdClient, nil -} - -func (c *Client) GetOrganization() (*govcd.Org, error) { - if c.Auth.Organization == "" { - return nil, errors.New("organization must be configured") - } - - org, err := c.VCDClient.GetOrgByNameOrId(c.Auth.Organization) - if err != nil { - return nil, fmt.Errorf("failed to get organization '%s': %w", c.Auth.Organization, err) - } - return org, err -} - -func (c *Client) GetVDCForOrg(org govcd.Org) (*govcd.Vdc, error) { - if c.Auth.VDC == "" { - return nil, errors.New("Organization VDC must be configured") - } - vcd, err := org.GetVDCByNameOrId(c.Auth.VDC, false) - if err != nil { - return nil, fmt.Errorf("failed to get Organization VDC '%s': %w", c.Auth.VDC, err) - } - return vcd, err -} - -func (c *Client) GetVMByName(vappName, vmName string) (*govcd.VM, error) { - _, _, vapp, err := c.GetOrganizationVDCAndVapp(vappName) - if err != nil { - return nil, err - } - - // We don't need ID here since we explicitly set the name field when creating the resource. - vm, err := vapp.GetVMByName(vmName, true) - if err != nil && errors.Is(err, govcd.ErrorEntityNotFound) { - return nil, cloudprovidererrors.ErrInstanceNotFound - } - return vm, err -} - -func (c *Client) GetOrganizationVDCAndVapp(vappName string) (*govcd.Org, *govcd.Vdc, *govcd.VApp, error) { - org, err := c.GetOrganization() - if err != nil { - return nil, nil, nil, err - } - - vdc, err := c.GetVDCForOrg(*org) - if err != nil { - return nil, nil, nil, err - } - - // Ensure that the vApp has already been created. - vapp, err := vdc.GetVAppByNameOrId(vappName, true) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get vApp '%s': %w", vappName, err) - } - return org, vdc, vapp, nil -} - -// GetVappNetworkType checks if the network exists and returns the network type. -func GetVappNetworkType(networkName string, vapp govcd.VApp) (NetworkType, error) { - networkConfig, err := vapp.GetNetworkConfig() - if err != nil { - return NoneNetworkType, fmt.Errorf("error getting vApp networks: %w", err) - } - - for _, netConfig := range networkConfig.NetworkConfig { - if netConfig.NetworkName == networkName || netConfig.ID == networkName { - switch { - case netConfig.NetworkName == types.NoneNetwork: - return NoneNetworkType, nil - case govcd.IsVappNetwork(netConfig.Configuration): - return VAppNetworkType, nil - default: - return OrgNetworkType, nil - } - } - } - return NoneNetworkType, fmt.Errorf("network '%s' not found: %w", networkName, err) -} diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/helper.go b/pkg/cloudprovider/provider/vmwareclouddirector/helper.go deleted file mode 100644 index 419e4e817..000000000 --- a/pkg/cloudprovider/provider/vmwareclouddirector/helper.go +++ /dev/null @@ -1,305 +0,0 @@ -/* -Copyright 2022 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vmwareclouddirector - -import ( - "encoding/base64" - "fmt" - "net/http" - "net/url" - "path" - - "github.com/vmware/go-vcloud-director/v2/govcd" - "github.com/vmware/go-vcloud-director/v2/types/v56" - vcdapitypes "github.com/vmware/go-vcloud-director/v2/types/v56" - - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - - "k8s.io/utils/ptr" -) - -var internalDiskBusTypes = map[string]string{ - "ide": "1", - "parallel": "3", - "sas": "4", - "paravirtual": "5", - "sata": "6", - "nvme": "7", -} - -func getComputePolicy(name string, policies []*govcd.VdcComputePolicy) *govcd.VdcComputePolicy { - for _, policy := range policies { - if policy.VdcComputePolicy == nil { - continue - } - if policy.VdcComputePolicy.Name == name || policy.VdcComputePolicy.ID == name { - return policy - } - } - return nil -} - -func createVM(client *Client, machine *clusterv1alpha1.Machine, c *Config, org *govcd.Org, vdc *govcd.Vdc, vapp *govcd.VApp) error { - // 1. We need the template HREF for the VM. - catalog, err := org.GetCatalogByNameOrId(c.Catalog, true) - if err != nil { - return fmt.Errorf("failed to get catalog '%s': %w", c.Catalog, err) - } - - // Catalog item can be a vApp template OVA or media ISO file. - catalogItem, err := catalog.GetCatalogItemByNameOrId(c.Template, true) - if err != nil { - return fmt.Errorf("failed to get catalog item '%s' in catalog '%s': %w", c.Template, c.Catalog, err) - } - - vAppTemplate, err := catalogItem.GetVAppTemplate() - if err != nil { - return fmt.Errorf("failed to get vApp template '%s' in catalog '%s': %w", c.Template, c.Catalog, err) - } - - templateHref := vAppTemplate.VAppTemplate.HREF - if vAppTemplate.VAppTemplate.Children != nil && len(vAppTemplate.VAppTemplate.Children.VM) != 0 { - templateHref = vAppTemplate.VAppTemplate.Children.VM[0].HREF - } - - // 2. Retrieve Sizing and Placement Compute Policy if required. - var computePolicy *types.ComputePolicy - if c.SizingPolicy != nil || c.PlacementPolicy != nil { - allPolicies, err := org.GetAllVdcComputePolicies(url.Values{}) - if err != nil { - return fmt.Errorf("failed to get template all VDC compute policies: %w", err) - } - - if c.SizingPolicy != nil && *c.SizingPolicy != "" { - sizingPolicy := getComputePolicy(*c.SizingPolicy, allPolicies) - if sizingPolicy == nil { - return fmt.Errorf("sizing policy '%s' doesn't exist", *c.SizingPolicy) - } - if computePolicy == nil { - computePolicy = &types.ComputePolicy{} - } - computePolicy.VmSizingPolicy = &vcdapitypes.Reference{ - HREF: sizingPolicy.VdcComputePolicy.ID, - } - } - - if c.PlacementPolicy != nil && *c.PlacementPolicy != "" { - placementPolicy := getComputePolicy(*c.PlacementPolicy, allPolicies) - if placementPolicy == nil { - return fmt.Errorf("placement policy '%s' doesn't exist", *c.PlacementPolicy) - } - if computePolicy == nil { - computePolicy = &types.ComputePolicy{} - } - computePolicy.VmPlacementPolicy = &vcdapitypes.Reference{ - HREF: placementPolicy.VdcComputePolicy.ID, - } - } - } - - // 3. Retrieve Storage Profile - var storageProfile *types.Reference - if c.StorageProfile != nil && *c.StorageProfile != defaultStorageProfile { - for _, sp := range vdc.Vdc.VdcStorageProfiles.VdcStorageProfile { - if sp.Name == *c.StorageProfile || sp.ID == *c.StorageProfile { - storageProfile = sp - break - } - } - if storageProfile == nil { - if err != nil { - return fmt.Errorf("failed to get storage profile '%s': %w", *c.StorageProfile, err) - } - } - } - - // 4. At this point we are ready to create our initial VMs. - // - // Multiple API calls to re-compose the vApp are handled in a synchronous manner, where each request has to wait - // for the previous request to complete. This can cause a huge overhead in terms of time. - // - // It is not possible to customize compute, disk and network for a VM at initial creation time when we are using templates. So we rely on - // vApp re-composition to apply the needed customization, performed at later stages. - vAppRecomposition := &types.ReComposeVAppParams{ - Ovf: types.XMLNamespaceOVF, - Xsi: types.XMLNamespaceXSI, - Xmlns: types.XMLNamespaceVCloud, - Deploy: false, - Name: vapp.VApp.Name, - PowerOn: false, - Description: vapp.VApp.Description, - SourcedItem: &types.SourcedCompositionItemParam{ - Source: &types.Reference{ - HREF: templateHref, - Name: machine.Name, - }, - InstantiationParams: &types.InstantiationParams{ - NetworkConnectionSection: &vcdapitypes.NetworkConnectionSection{ - NetworkConnection: []*vcdapitypes.NetworkConnection{ - { - Network: c.Network, - NeedsCustomization: false, - IsConnected: true, - IPAddressAllocationMode: string(c.IPAllocationMode), - NetworkAdapterType: "VMXNET3", - }, - }, - }, - }, - StorageProfile: storageProfile, - ComputePolicy: computePolicy, - }, - AllEULAsAccepted: true, - } - - apiEndpoint, err := url.Parse(vapp.VApp.HREF) - if err != nil { - return fmt.Errorf("error getting vapp href '%s': %w", c.Auth.URL, err) - } - apiEndpoint.Path = path.Join(apiEndpoint.Path, "action/recomposeVApp") - - task, err := client.VCDClient.Client.ExecuteTaskRequest(apiEndpoint.String(), http.MethodPost, - types.MimeRecomposeVappParams, "error instantiating a new VM: %s", vAppRecomposition) - if err != nil { - return fmt.Errorf("failed to execute API call to create VM: %w", err) - } - - // Wait for VM to be created this should take around 1-3 minutes - if err = task.WaitTaskCompletion(); err != nil { - return fmt.Errorf("failed to wait for VM creation task to complete: %w", err) - } - return nil -} - -func recomposeComputeAndDisk(config *Config, vm *govcd.VM) (*govcd.VM, error) { - needsComputeRecomposition := false - needsDiskRecomposition := false - // Perform compute recomposition if SizingPolicy was not specified. - vmSpecSection := vm.VM.VmSpecSection - if config.SizingPolicy == nil || *config.SizingPolicy == "" { - vmSpecSection.MemoryResourceMb.Configured = config.MemoryMB - vmSpecSection.NumCpus = ptr.To(int(config.CPUs)) - vmSpecSection.NumCoresPerSocket = ptr.To(int(config.CPUCores)) - needsComputeRecomposition = true - } - - // Perform disk recomposition if required. - if vmSpecSection.DiskSection != nil { - for i, internalDisk := range vmSpecSection.DiskSection.DiskSettings { - // We are only concerned with template disk and not named/independent disks. - if internalDisk.Disk == nil { - if config.DiskSizeGB != nil && *config.DiskSizeGB > 0 { - vmSpecSection.DiskSection.DiskSettings[i].SizeMb = (*config.DiskSizeGB) * 1024 - needsDiskRecomposition = true - } - if config.DiskIOPS != nil && *config.DiskIOPS > 0 { - vmSpecSection.DiskSection.DiskSettings[i].Iops = ptr.To(*config.DiskIOPS) - needsDiskRecomposition = true - } - if config.DiskBusType != nil && *config.DiskBusType != "" { - vmSpecSection.DiskSection.DiskSettings[i].AdapterType = internalDiskBusTypes[*config.DiskBusType] - needsDiskRecomposition = true - } - } - } - } - - if !needsDiskRecomposition { - // Update treats same values as changes and fails. Although if set to nil, it assumes that no changes are required for this field. - vmSpecSection.DiskSection = nil - } - - var err error - // Execute disk and compute recomposition on our VM - if needsComputeRecomposition || needsDiskRecomposition { - description := vm.VM.Description - vm, err = vm.UpdateVmSpecSection(vmSpecSection, description) - if err != nil { - return nil, fmt.Errorf("error updating VM spec section: %w", err) - } - } - return vm, nil -} - -func setUserData(userdata string, vm *govcd.VM, isFlatcar bool) error { - userdataBase64 := base64.StdEncoding.EncodeToString([]byte(userdata)) - props := map[string]string{ - "disk.enableUUID": "1", - "instance-id": vm.VM.Name, - } - - if isFlatcar { - props["guestinfo.ignition.config.data"] = userdataBase64 - props["guestinfo.ignition.config.data.encoding"] = "base64" - } else { - props["user-data"] = userdataBase64 - } - - vmProperties := &vcdapitypes.ProductSectionList{ - ProductSection: &vcdapitypes.ProductSection{ - Info: "Custom properties", - Property: []*vcdapitypes.Property{}, - }, - } - for key, value := range props { - property := &vcdapitypes.Property{ - UserConfigurable: true, - Type: "string", - Key: key, - Label: key, - Value: &vcdapitypes.Value{Value: value}, - } - vmProperties.ProductSection.Property = append(vmProperties.ProductSection.Property, property) - } - - // Set guest properties on the VM - _, err := vm.SetProductSectionList(vmProperties) - if err != nil { - return fmt.Errorf("error setting guest properties for VM: %w", err) - } - return nil -} - -func addMetadata(vm *govcd.VM, metadata *map[string]string) error { - // Nothing to do here. - if metadata == nil { - return nil - } - - for key, val := range *metadata { - err := vm.AddMetadataEntry(vcdapitypes.MetadataStringValue, key, val) - if err != nil { - return fmt.Errorf("error adding metadata for VM: %w", err) - } - } - return nil -} - -func setComputerName(vm *govcd.VM, machineName string) error { - customizationSection, err := vm.GetGuestCustomizationSection() - if err != nil { - return fmt.Errorf("error retrieving guest customization section for VM: %w", err) - } - - customizationSection.ComputerName = machineName - - _, err = vm.SetGuestCustomizationSection(customizationSection) - if err != nil { - return fmt.Errorf("error adding metadata for VM: %w", err) - } - return nil -} diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go deleted file mode 100644 index eb4363d4a..000000000 --- a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go +++ /dev/null @@ -1,585 +0,0 @@ -/* -Copyright 2022 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vmwareclouddirector - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "net/url" - - "github.com/vmware/go-vcloud-director/v2/govcd" - "go.uber.org/zap" - - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - vcdtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vmwareclouddirector/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/ptr" -) - -const ( - defaultDiskType = "paravirtual" - defaultStorageProfile = "*" - defaultDiskIOPS = 0 -) - -type NetworkType string - -const ( - VAppNetworkType NetworkType = "vapp" - OrgNetworkType NetworkType = "org" - // Network with a NIC that is not attached to any network. - NoneNetworkType NetworkType = "none" -) - -type provider struct { - configVarResolver *providerconfig.ConfigVarResolver -} - -type Auth struct { - Username string - Password string - APIToken string - Organization string - URL string - VDC string - AllowInsecure bool -} - -type Config struct { - Auth `json:",inline"` - - // VM configuration. - VApp string - Template string - Catalog string - PlacementPolicy *string - SizingPolicy *string - - // Network configuration. - Network string - IPAllocationMode vcdtypes.IPAllocationMode - - // Compute configuration. - CPUs int64 - CPUCores int64 - MemoryMB int64 - - // Storage configuration. - DiskSizeGB *int64 - DiskBusType *string - DiskIOPS *int64 - StorageProfile *string - - // Metadata configuration. - Metadata *map[string]string -} - -// New returns a VMware Cloud Director provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return &provider{configVarResolver: configVarResolver} -} - -// Ensures that Server implements Instance interface. -var _ instance.Instance = &Server{} - -// Server holds VMware Cloud Director VM information. -type Server struct { - name string - id string - status instance.Status - addresses map[string]corev1.NodeAddressType -} - -func (s Server) Name() string { - return s.name -} - -func (s Server) ID() string { - return s.id -} - -func (s Server) ProviderID() string { - if s.ID() == "" { - return "" - } - return fmt.Sprintf("vmware-cloud-director://%s", s.ID()) -} - -func (s Server) HostID() string { - return "" -} - -func (s Server) Addresses() map[string]corev1.NodeAddressType { - return s.addresses -} - -func (s Server) Status() instance.Status { - return s.status -} - -func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { - _, _, rawConfig, err := p.getConfig(spec.ProviderSpec) - if err != nil { - return spec, err - } - - // Set defaults. - if rawConfig.IPAllocationMode == "" { - rawConfig.IPAllocationMode = vcdtypes.DHCPIPAllocationMode - } - - // These defaults will have no effect if DiskSizeGB is not specified - if rawConfig.DiskBusType == nil { - rawConfig.DiskBusType = ptr.To(defaultDiskType) - } - if rawConfig.DiskIOPS == nil { - rawConfig.DiskIOPS = ptr.To(int64(defaultDiskIOPS)) - } - spec.ProviderSpec.Value, err = setProviderSpec(*rawConfig, spec.ProviderSpec) - return spec, err -} - -func (p *provider) Cleanup(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { - c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return false, fmt.Errorf("failed to parse config: %w", err) - } - - client, err := NewClient(c.Username, c.Password, c.APIToken, c.Organization, c.URL, c.VDC, c.AllowInsecure) - if err != nil { - return false, fmt.Errorf("failed to create VMware Cloud Director client: %w", err) - } - - vm, err := client.GetVMByName(c.VApp, machine.Name) - if err != nil { - if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { - return true, nil - } - return false, err - } - - vmStatus, err := vm.GetStatus() - if err != nil { - return false, fmt.Errorf("failed to get VM status: %w", err) - } - - // Turn off VM if it's `ON` - if vmStatus == "POWERED_ON" { - task, err := vm.PowerOff() - if err != nil { - return false, fmt.Errorf("failed to turn off VM: %w", err) - } - if err = task.WaitTaskCompletion(); err != nil { - return false, fmt.Errorf("error waiting for VM power off task to complete: %w", err) - } - } - - if err := vm.Delete(); err != nil { - return false, fmt.Errorf("failed to destroy vm %s: %w", vm.VM.Name, err) - } - return true, nil -} - -func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { - vm, err := p.create(ctx, machine, userdata) - if err != nil { - _, cleanupErr := p.Cleanup(ctx, log, machine, data) - if cleanupErr != nil { - return nil, fmt.Errorf("cleaning up failed with err %w after creation failed with err %w", cleanupErr, err) - } - return nil, err - } - return vm, nil -} - -func (p *provider) create(_ context.Context, machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { - c, providerConfig, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, fmt.Errorf("failed to parse config: %w", err) - } - - client, err := NewClient(c.Username, c.Password, c.APIToken, c.Organization, c.URL, c.VDC, c.AllowInsecure) - if err != nil { - return nil, fmt.Errorf("failed to create VMware Cloud Director client: %w", err) - } - - // Fetch the organization, VDC, and vApp resources. - org, vdc, vapp, err := client.GetOrganizationVDCAndVapp(c.VApp) - if err != nil { - return nil, err - } - - // 1. Create Standalone VM from template. - err = createVM(client, machine, c, org, vdc, vapp) - if err != nil { - return nil, fmt.Errorf("failed to create VM: %w", err) - } - - // 2. Fetch updated vApp - err = vapp.Refresh() - if err != nil { - return nil, fmt.Errorf("failed to get updated vApp '%s' after recompoisition: %w", c.VApp, err) - } - - // 3. Fetch updated VM - vm, err := vapp.GetVMByName(machine.Name, true) - if err != nil { - return nil, err - } - - // 4. Perform VM recomposition for compute and disks - vm, err = recomposeComputeAndDisk(c, vm) - if err != nil { - return nil, err - } - - // 5. Before powering on the VM, configure customization to attach userdata with the VM - // update guest properties. - err = setUserData(userdata, vm, providerConfig.OperatingSystem == providerconfigtypes.OperatingSystemFlatcar) - if err != nil { - return nil, err - } - - // 6. Fetch updated VM. - err = vm.Refresh() - if err != nil { - return nil, err - } - - // 7. Add Metadata to VM. - err = addMetadata(vm, c.Metadata) - if err != nil { - return nil, err - } - - // 8. Set computer name for the VM - err = setComputerName(vm, machine.Name) - if err != nil { - return nil, err - } - - // 9. Finally power on the VM after performing all required actions. - task, err := vm.PowerOn() - if err != nil { - return nil, fmt.Errorf("failed to turn on VM: %w", err) - } - if err = task.WaitTaskCompletion(); err != nil { - return nil, fmt.Errorf("error waiting for VM bootstrap to complete: %w", err) - } - - return p.getInstance(vm) -} - -func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, fmt.Errorf("failed to parse config: %w", err) - } - - client, err := NewClient(c.Username, c.Password, c.APIToken, c.Organization, c.URL, c.VDC, c.AllowInsecure) - if err != nil { - return nil, fmt.Errorf("failed to create VMware Cloud Director client: %w", err) - } - - vm, err := client.GetVMByName(c.VApp, machine.Name) - if err != nil { - return nil, err - } - - return p.getInstance(vm) -} - -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return "", "", nil -} - -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *vcdtypes.RawConfig, error) { - if provSpec.Value == nil { - return nil, nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - - pconfig, err := providerconfigtypes.GetConfig(provSpec) - if err != nil { - return nil, nil, nil, err - } - - if pconfig.OperatingSystemSpec.Raw == nil { - return nil, nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") - } - - rawConfig, err := vcdtypes.GetConfig(*pconfig) - if err != nil { - return nil, nil, nil, err - } - - c := Config{} - - c.APIToken, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.APIToken, "VCD_API_TOKEN") - if err != nil { - return nil, nil, nil, fmt.Errorf(`failed to get the value of "apiToken" field, error = %w`, err) - } - - c.Username, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Username, "VCD_USER") - if err != nil { - return nil, nil, nil, fmt.Errorf(`failed to get the value of "username" field, error = %w`, err) - } - - c.Password, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Password, "VCD_PASSWORD") - if err != nil { - return nil, nil, nil, fmt.Errorf(`failed to get the value of "password" field, error = %w`, err) - } - - c.Organization, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Organization, "VCD_ORG") - if err != nil { - return nil, nil, nil, fmt.Errorf(`failed to get the value of "organization" field, error = %w`, err) - } - - c.URL, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.URL, "VCD_URL") - if err != nil { - return nil, nil, nil, fmt.Errorf(`failed to get the value of "url" field, error = %w`, err) - } - - c.VDC, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.VDC, "VCD_VDC") - if err != nil { - return nil, nil, nil, fmt.Errorf(`failed to get the value of "vdc" field, error = %w`, err) - } - - c.AllowInsecure, err = p.configVarResolver.GetConfigVarBoolValueOrEnv(rawConfig.AllowInsecure, "VCD_ALLOW_UNVERIFIED_SSL") - if err != nil { - return nil, nil, nil, fmt.Errorf(`failed to get the value of "allowInsecure" field, error = %w`, err) - } - - c.VApp, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VApp) - if err != nil { - return nil, nil, nil, err - } - - c.Template, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Template) - if err != nil { - return nil, nil, nil, err - } - - c.Catalog, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Catalog) - if err != nil { - return nil, nil, nil, err - } - - c.Network, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Network) - if err != nil { - return nil, nil, nil, err - } - - c.IPAllocationMode = rawConfig.IPAllocationMode - - if rawConfig.DiskSizeGB != nil && *rawConfig.DiskSizeGB < 0 { - return nil, nil, nil, fmt.Errorf(`value for "diskSizeGB" should either be nil or greater than or equal to 0`) - } - c.DiskSizeGB = rawConfig.DiskSizeGB - - if rawConfig.DiskIOPS != nil && *rawConfig.DiskIOPS < 0 { - return nil, nil, nil, fmt.Errorf(`value for "diskIOPS" should either be nil or greater than or equal to 0`) - } - c.DiskIOPS = rawConfig.DiskIOPS - - if rawConfig.CPUs <= 0 { - return nil, nil, nil, fmt.Errorf(`value for "cpus" should be greater than 0`) - } - c.CPUs = rawConfig.CPUs - - if rawConfig.CPUCores <= 0 { - return nil, nil, nil, fmt.Errorf(`value for "cpuCores" should be greater than 0`) - } - c.CPUCores = rawConfig.CPUCores - - if rawConfig.MemoryMB <= 4 { - return nil, nil, nil, fmt.Errorf(`value for "memoryMB" should be greater than 0`) - } - if rawConfig.MemoryMB%4 != 0 { - return nil, nil, nil, fmt.Errorf(`value for "memoryMB" should be a multiple of 4`) - } - c.MemoryMB = rawConfig.MemoryMB - - c.DiskBusType = rawConfig.DiskBusType - c.StorageProfile = rawConfig.StorageProfile - c.Metadata = rawConfig.Metadata - c.SizingPolicy = rawConfig.SizingPolicy - c.PlacementPolicy = rawConfig.PlacementPolicy - return &c, pconfig, rawConfig, err -} - -func (p *provider) getInstance(vm *govcd.VM) (instance.Instance, error) { - vmStatus, err := vm.GetStatus() - if err != nil { - return nil, fmt.Errorf("failed to get VM status: %w", err) - } - - var status instance.Status - - switch vmStatus { - case "POWERED_ON": - status = instance.StatusRunning - case "POWERED_OFF", "PARTIALLY_POWERED_OFF": - status = instance.StatusCreating - default: - status = instance.StatusUnknown - } - - addresses := make(map[string]corev1.NodeAddressType) - if vm.VM.NetworkConnectionSection != nil && vm.VM.NetworkConnectionSection.NetworkConnection != nil { - for _, nic := range vm.VM.NetworkConnectionSection.NetworkConnection { - if nic.ExternalIPAddress != "" { - addresses[nic.ExternalIPAddress] = corev1.NodeExternalIP - } - if nic.IPAddress != "" { - addresses[nic.IPAddress] = corev1.NodeInternalIP - } - } - } - - return Server{name: vm.VM.Name, status: status, addresses: addresses, id: vm.VM.ID}, nil -} - -func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { - labels := make(map[string]string) - - c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err == nil { - labels["size"] = fmt.Sprintf("%d-cpus-%d-mb", c.CPUs, c.MemoryMB) - labels["vapp"] = c.VApp - labels["vdc"] = c.VDC - labels["organization"] = c.Organization - } - - return labels, err -} - -func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ types.UID) error { - return nil -} - -func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { - return nil -} - -func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { - c, _, _, err := p.getConfig(spec.ProviderSpec) - if err != nil { - return fmt.Errorf("failed to parse config: %w", err) - } - - if c.APIToken != "" && (c.Password != "" || c.Username != "") { - return fmt.Errorf(`either "apiToken" or "username" and "password" must be specified`) - } - - client, err := NewClient(c.Username, c.Password, c.APIToken, c.Organization, c.URL, c.VDC, c.AllowInsecure) - if err != nil { - return fmt.Errorf("failed to create VMware Cloud Director client: %w", err) - } - - // Ensure that the organization, VDC, and vApp exists. - org, vdc, vapp, err := client.GetOrganizationVDCAndVapp(c.VApp) - if err != nil { - return err - } - - // Ensure that the catalog exists. - catalog, err := org.GetCatalogByNameOrId(c.Catalog, true) - if err != nil { - return fmt.Errorf("failed to get catalog '%s': %w", c.Catalog, err) - } - - // Ensure that the template exists in the catalog - // Catalog item can be a vApp template OVA or media ISO file. - catalogItem, err := catalog.GetCatalogItemByNameOrId(c.Template, true) - if err != nil { - return fmt.Errorf("failed to get template '%s' in catalog '%s': %w", c.Template, c.Catalog, err) - } - if c.DiskSizeGB != nil && catalogItem.CatalogItem.Size > *c.DiskSizeGB { - return fmt.Errorf("diskSizeGB '%v' cannot be less than the template size '%v': %w", *c.DiskSizeGB, catalogItem.CatalogItem.Size, err) - } - - // Ensure that the network exists - // It can either be a vApp network or a vApp Org network. - _, err = GetVappNetworkType(c.Network, *vapp) - if err != nil { - return fmt.Errorf("failed to get network '%s' for vapp '%s': %w", c.Network, c.VApp, err) - } - - if c.SizingPolicy != nil || c.PlacementPolicy != nil { - allPolicies, err := org.GetAllVdcComputePolicies(url.Values{}) - if err != nil { - return fmt.Errorf("failed to get template all VDC compute policies: %w", err) - } - - if c.SizingPolicy != nil && *c.SizingPolicy != "" { - sizingPolicy := getComputePolicy(*c.SizingPolicy, allPolicies) - if sizingPolicy == nil { - return fmt.Errorf("sizing policy '%s' doesn't exist", *c.SizingPolicy) - } - } - - if c.PlacementPolicy != nil && *c.PlacementPolicy != "" { - placementPolicy := getComputePolicy(*c.PlacementPolicy, allPolicies) - if placementPolicy == nil { - return fmt.Errorf("placement policy '%s' doesn't exist", *c.SizingPolicy) - } - } - } - - // Ensure that the storage profile exists. - if c.StorageProfile != nil && *c.StorageProfile != defaultStorageProfile { - _, err = vdc.FindStorageProfileReference(*c.StorageProfile) - if err != nil { - return fmt.Errorf("failed to get storage profile '%s': %w", *c.StorageProfile, err) - } - } - return nil -} - -func setProviderSpec(rawConfig vcdtypes.RawConfig, provSpec clusterv1alpha1.ProviderSpec) (*runtime.RawExtension, error) { - if provSpec.Value == nil { - return nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - - pconfig, err := providerconfigtypes.GetConfig(provSpec) - if err != nil { - return nil, err - } - - rawCloudProviderSpec, err := json.Marshal(rawConfig) - if err != nil { - return nil, err - } - - pconfig.CloudProviderSpec = runtime.RawExtension{Raw: rawCloudProviderSpec} - rawPconfig, err := json.Marshal(pconfig) - if err != nil { - return nil, err - } - - return &runtime.RawExtension{Raw: rawPconfig}, nil -} diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/types/types.go b/pkg/cloudprovider/provider/vmwareclouddirector/types/types.go deleted file mode 100644 index 1b4cb7b6d..000000000 --- a/pkg/cloudprovider/provider/vmwareclouddirector/types/types.go +++ /dev/null @@ -1,72 +0,0 @@ -/* -Copyright 2022 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" -) - -type IPAllocationMode string - -const ( - PoolIPAllocationMode IPAllocationMode = "POOL" - DHCPIPAllocationMode IPAllocationMode = "DHCP" -) - -// RawConfig represents VMware Cloud Director specific configuration. -type RawConfig struct { - // Provider configuration. - Username providerconfigtypes.ConfigVarString `json:"username"` - Password providerconfigtypes.ConfigVarString `json:"password"` - APIToken providerconfigtypes.ConfigVarString `json:"apiToken"` - Organization providerconfigtypes.ConfigVarString `json:"organization"` - URL providerconfigtypes.ConfigVarString `json:"url"` - VDC providerconfigtypes.ConfigVarString `json:"vdc"` - AllowInsecure providerconfigtypes.ConfigVarBool `json:"allowInsecure"` - - // VM configuration. - VApp providerconfigtypes.ConfigVarString `json:"vapp"` - Template providerconfigtypes.ConfigVarString `json:"template"` - Catalog providerconfigtypes.ConfigVarString `json:"catalog"` - PlacementPolicy *string `json:"placementPolicy,omitempty"` - - // Network configuration. - Network providerconfigtypes.ConfigVarString `json:"network"` - IPAllocationMode IPAllocationMode `json:"ipAllocationMode,omitempty"` - - // Compute configuration. - CPUs int64 `json:"cpus"` - CPUCores int64 `json:"cpuCores"` - MemoryMB int64 `json:"memoryMB"` - SizingPolicy *string `json:"sizingPolicy,omitempty"` - - // Storage configuration. - DiskSizeGB *int64 `json:"diskSizeGB,omitempty"` - DiskBusType *string `json:"diskBusType,omitempty"` - DiskIOPS *int64 `json:"diskIOPS,omitempty"` - StorageProfile *string `json:"storageProfile,omitempty"` - - // Metadata configuration. - Metadata *map[string]string `json:"metadata,omitempty"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/vsphere/client.go b/pkg/cloudprovider/provider/vsphere/client.go deleted file mode 100644 index 4a2e688eb..000000000 --- a/pkg/cloudprovider/provider/vsphere/client.go +++ /dev/null @@ -1,125 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vsphere - -import ( - "context" - "fmt" - "net/url" - - "github.com/vmware/govmomi" - "github.com/vmware/govmomi/find" - "github.com/vmware/govmomi/object" - "github.com/vmware/govmomi/session" - "github.com/vmware/govmomi/vapi/rest" - "github.com/vmware/govmomi/vim25" - "github.com/vmware/govmomi/vim25/soap" - - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - - utilruntime "k8s.io/apimachinery/pkg/util/runtime" -) - -type Session struct { - Client *govmomi.Client - Finder *find.Finder - Datacenter *object.Datacenter -} - -type RESTSession struct { - Client *rest.Client -} - -// NewSession creates a vCenter client with initialized finder. -func NewSession(ctx context.Context, config *Config) (*Session, error) { - vim25Client, err := createVim25Client(ctx, config) - if err != nil { - return nil, err - } - if err != nil { - return nil, err - } - - client := &govmomi.Client{ - Client: vim25Client, - SessionManager: session.NewManager(vim25Client), - } - - if err = client.Login(ctx, url.UserPassword(config.Username, config.Password)); err != nil { - return nil, fmt.Errorf("failed vsphere login: %w", err) - } - - finder := find.NewFinder(client.Client, true) - dc, err := finder.Datacenter(ctx, config.Datacenter) - if err != nil { - return nil, fmt.Errorf("failed to get vsphere datacenter: %w", err) - } - finder.SetDatacenter(dc) - - return &Session{ - Client: client, - Finder: finder, - Datacenter: dc, - }, nil -} - -// Logout closes the idling vCenter connections. -func (s *Session) Logout(ctx context.Context) { - if err := s.Client.Logout(ctx); err != nil { - utilruntime.HandleError(fmt.Errorf("vsphere client failed to logout: %w", err)) - } -} - -func NewRESTSession(ctx context.Context, config *Config) (*RESTSession, error) { - vim25Client, err := createVim25Client(ctx, config) - if err != nil { - return nil, err - } - - client := rest.NewClient(vim25Client) - - if err = client.Login(ctx, url.UserPassword(config.Username, config.Password)); err != nil { - return nil, fmt.Errorf("failed to login: %w", err) - } - - return &RESTSession{ - Client: client, - }, nil -} - -// Logout closes the idling vCenter connections. -func (s *RESTSession) Logout(ctx context.Context) { - if err := s.Client.Logout(ctx); err != nil { - utilruntime.HandleError(fmt.Errorf("vsphere REST client failed to logout: %w", err)) - } -} - -func createVim25Client(ctx context.Context, config *Config) (*vim25.Client, error) { - endpointURL, err := url.Parse(config.VSphereURL) - if err != nil { - return nil, err - } - - clientURL := endpointURL.JoinPath("/sdk") - - // creating the govmoni Client in roundabout way because we need to set the proper CA bundle: reference https://github.com/vmware/govmomi/issues/1200 - soapClient := soap.NewClient(clientURL, config.AllowInsecure) - // set our CA bundle - soapClient.DefaultTransport().TLSClientConfig.RootCAs = util.CABundle - - return vim25.NewClient(ctx, soapClient) -} diff --git a/pkg/cloudprovider/provider/vsphere/helper.go b/pkg/cloudprovider/provider/vsphere/helper.go deleted file mode 100644 index 18cef6a7b..000000000 --- a/pkg/cloudprovider/provider/vsphere/helper.go +++ /dev/null @@ -1,515 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vsphere - -import ( - "bytes" - "context" - "encoding/base64" - "errors" - "fmt" - "math" - "os" - "os/exec" - "text/template" - - "github.com/vmware/govmomi/object" - "github.com/vmware/govmomi/vapi/tags" - "github.com/vmware/govmomi/vim25/mo" - "github.com/vmware/govmomi/vim25/soap" - "github.com/vmware/govmomi/vim25/types" - "go.uber.org/zap" - - utilruntime "k8s.io/apimachinery/pkg/util/runtime" -) - -const ( - localTempDir = "/tmp" - metaDataTemplate = `instance-id: {{ .InstanceID}} -local-hostname: {{ .Hostname }}` -) - -func createClonedVM(ctx context.Context, log *zap.SugaredLogger, vmName string, config *Config, session *Session, containerLinuxUserdata string) (*object.VirtualMachine, error) { - tpl, err := session.Finder.VirtualMachine(ctx, config.TemplateVMName) - if err != nil { - return nil, fmt.Errorf("failed to get template vm: %w", err) - } - - // Find the target folder, if its included in the provider config. - var targetVMFolder *object.Folder - if config.Folder != "" { - // If non-absolute folder name is used, e.g. 'duplicate-folder' it can match - // multiple folders and thus fail. It will also gladly match a folder from - // a different datacenter. It is therefore preferable to use absolute folder - // paths, e.g. '/Datacenter/vm/nested/folder'. - // The target folder must already exist. - targetVMFolder, err = session.Finder.Folder(ctx, config.Folder) - if err != nil { - return nil, fmt.Errorf("failed to get target folder: %w", err) - } - } else { - // Do not query datacenter folders unless required - datacenterFolders, err := session.Datacenter.Folders(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get datacenter folders: %w", err) - } - targetVMFolder = datacenterFolders.VmFolder - } - - relocateSpec := types.VirtualMachineRelocateSpec{ - DiskMoveType: string(types.VirtualMachineRelocateDiskMoveOptionsMoveAllDiskBackingsAndConsolidate), - Folder: types.NewReference(targetVMFolder.Reference()), - Disk: []types.VirtualMachineRelocateSpecDiskLocator{}, - } - cloneSpec := types.VirtualMachineCloneSpec{ - PowerOn: false, - Template: false, - Location: relocateSpec, - } - datastoreref, err := resolveDatastoreRef(ctx, log, config, session, tpl, targetVMFolder, &cloneSpec) - if err != nil { - return nil, fmt.Errorf("failed to resolve datastore: %w", err) - } - - resourcepoolref, err := resolveResourcePoolRef(ctx, config, session) - if err != nil { - return nil, fmt.Errorf("failed to resolve resourcePool: %w", err) - } - - cloneSpec.Location.Datastore = datastoreref - cloneSpec.Location.Pool = resourcepoolref - // Create a cloned VM from the template VM's snapshot. - // We split the cloning from the reconfiguring as those actions differ on the permission side. - // It's nicer to tell which specific action failed due to lacking permissions. - clonedVMTask, err := tpl.Clone(ctx, targetVMFolder, vmName, cloneSpec) - if err != nil { - return nil, fmt.Errorf("failed to clone template vm: %w", err) - } - - if err := clonedVMTask.Wait(ctx); err != nil { - return nil, fmt.Errorf("error when waiting for result of clone task: %w", err) - } - virtualMachine, err := session.Finder.VirtualMachine(ctx, vmName) - if err != nil { - return nil, fmt.Errorf("failed to get virtual machine object after cloning: %w", err) - } - vmDevices, err := virtualMachine.Device(ctx) - if err != nil { - return nil, fmt.Errorf("failed to list devices of template VM: %w", err) - } - - var vAppAconfig *types.VmConfigSpec - if containerLinuxUserdata != "" { - userdataBase64 := base64.StdEncoding.EncodeToString([]byte(containerLinuxUserdata)) - - // The properties describing userdata will already exist in the Flatcar VM template. - // In order to overwrite them, we need to specify their numeric Key values, - // which we'll extract from that template. - var mvm mo.VirtualMachine - if err := virtualMachine.Properties(ctx, virtualMachine.Reference(), []string{"config", "config.vAppConfig", "config.vAppConfig.property"}, &mvm); err != nil { - return nil, fmt.Errorf("failed to extract vapp properties for flatcar: %w", err) - } - - var propertySpecs []types.VAppPropertySpec - if mvm.Config.VAppConfig.GetVmConfigInfo() == nil { - return nil, fmt.Errorf("no vm config found in template '%s'. Make sure you import the correct OVA with the appropriate flatcar settings", config.TemplateVMName) - } - - var ( - guestInfoUserData string - guestInfoUserDataEncoding string - ) - - guestInfoUserData = "guestinfo.ignition.config.data" - guestInfoUserDataEncoding = "guestinfo.ignition.config.data.encoding" - for _, item := range mvm.Config.VAppConfig.GetVmConfigInfo().Property { - switch item.Id { - case guestInfoUserData: - propertySpecs = append(propertySpecs, types.VAppPropertySpec{ - ArrayUpdateSpec: types.ArrayUpdateSpec{ - Operation: types.ArrayUpdateOperationEdit, - }, - Info: &types.VAppPropertyInfo{ - Key: item.Key, - Id: item.Id, - Value: userdataBase64, - }, - }) - case guestInfoUserDataEncoding: - propertySpecs = append(propertySpecs, types.VAppPropertySpec{ - ArrayUpdateSpec: types.ArrayUpdateSpec{ - Operation: types.ArrayUpdateOperationEdit, - }, - Info: &types.VAppPropertyInfo{ - Key: item.Key, - Id: item.Id, - Value: "base64", - }, - }) - } - } - - vAppAconfig = &types.VmConfigSpec{Property: propertySpecs} - } - - diskUUIDEnabled := true - var deviceSpecs []types.BaseVirtualDeviceConfigSpec - if config.DiskSizeGB != nil { - disks, err := getDisksFromVM(ctx, virtualMachine) - if err != nil { - return nil, fmt.Errorf("failed to get disks from VM: %w", err) - } - // If this is wrong, the resulting error is `Invalid operation for device '0` - // so verify again this is legit - if err := validateDiskResizing(disks, *config.DiskSizeGB); err != nil { - return nil, err - } - - log.Debugw("Increasing disk size", "targetgb", *config.DiskSizeGB) - disk := disks[0] - disk.CapacityInBytes = *config.DiskSizeGB * int64(math.Pow(1024, 3)) - diskspec := &types.VirtualDeviceConfigSpec{Operation: types.VirtualDeviceConfigSpecOperationEdit, Device: disk} - deviceSpecs = append(deviceSpecs, diskspec) - } - - if config.VMNetName != "" || len(config.Networks) > 0 { - networkSpecs, err := GetNetworkSpecs(ctx, session, vmDevices, config.VMNetName, config.Networks) - if err != nil { - return nil, fmt.Errorf("failed to get network specifications: %w", err) - } - deviceSpecs = append(deviceSpecs, networkSpecs...) - } - - vmConfig := types.VirtualMachineConfigSpec{ - DeviceChange: deviceSpecs, - Flags: &types.VirtualMachineFlagInfo{ - DiskUuidEnabled: &diskUUIDEnabled, - }, - NumCPUs: config.CPUs, - MemoryMB: config.MemoryMB, - VAppConfig: vAppAconfig, - } - reconfigureTask, err := virtualMachine.Reconfigure(ctx, vmConfig) - if err != nil { - return nil, fmt.Errorf("failed to reconfigure the VM: %w", err) - } - if err := reconfigureTask.Wait(ctx); err != nil { - return nil, fmt.Errorf("error when waiting for result of the reconfigure task: %w", err) - } - - // Ubuntu won't boot with attached floppy device, because it tries to write to it - // which fails, because the floppy device does not contain a floppy disk - // Upstream issue: https://bugs.launchpad.net/cloud-images/+bug/1573095 - if err := removeFloppyDevice(ctx, virtualMachine); err != nil { - return nil, fmt.Errorf("failed to remove floppy device: %w", err) - } - return virtualMachine, nil -} - -func resolveDatastoreRef(ctx context.Context, log *zap.SugaredLogger, config *Config, session *Session, vm *object.VirtualMachine, folder *object.Folder, cloneSpec *types.VirtualMachineCloneSpec) (*types.ManagedObjectReference, error) { - // Based on https://github.com/vmware/govmomi/blob/v0.22.1/govc/vm/clone.go#L358 - if config.DatastoreCluster != "" && config.Datastore == "" { - vmLog := log.With("vm", vm.Name(), "datastorecluster", config.DatastoreCluster) - vmLog.Infow("Choosing initial datastore placement for vm from datastore cluster") - - storagePod, err := session.Finder.DatastoreCluster(ctx, config.DatastoreCluster) - if err != nil { - return nil, fmt.Errorf("failed to get datastore cluster: %w", err) - } - - // Build pod selection spec from config spec - podSelectionSpec := types.StorageDrsPodSelectionSpec{ - StoragePod: types.NewReference(storagePod.Reference()), - } - - // Get the virtual machine reference - vmref := vm.Reference() - - // TODO(irozzo) moveAllDiskBackingsAndConsolidate does not seem to work with RecommendDatastore, - // try to better understand the reason and the implications. - // https://code.vmware.com/docs/4206/vsphere-web-services-api-reference/doc/vim.vm.RelocateSpec.DiskMoveOptions.html - cloneSpec.Location.DiskMoveType = string(types.VirtualMachineRelocateDiskMoveOptionsMoveAllDiskBackingsAndDisallowSharing) - // Build the placement spec - storagePlacementSpec := types.StoragePlacementSpec{ - Folder: types.NewReference(folder.Reference()), - Vm: &vmref, - CloneName: vm.Name(), - CloneSpec: cloneSpec, - PodSelectionSpec: podSelectionSpec, - Type: string(types.StoragePlacementSpecPlacementTypeClone), - } - - // Get the storage placement result - storageResourceManager := object.NewStorageResourceManager(session.Client.Client) - result, err := storageResourceManager.RecommendDatastores(ctx, storagePlacementSpec) - if err != nil { - return nil, fmt.Errorf("error occurred while getting storage placement recommendation: %w", err) - } - - // Get the recommendations - recommendations := result.Recommendations - if len(recommendations) == 0 { - return nil, fmt.Errorf("no recommendations") - } - - // Get the first recommendation - ds := recommendations[0].Action[0].(*types.StoragePlacementAction).Destination.Reference() - vmLog.Infow("Selected datastore from datastore cluster", "datastore", ds) - - return &ds, nil - } else if config.DatastoreCluster == "" && config.Datastore != "" { - datastore, err := session.Finder.Datastore(ctx, config.Datastore) - if err != nil { - return nil, fmt.Errorf("failed to get datastore: %w", err) - } - return types.NewReference(datastore.Reference()), nil - } - return nil, fmt.Errorf("please provide either a datastore or a datastore cluster") -} - -func uploadAndAttachISO(ctx context.Context, log *zap.SugaredLogger, session *Session, vmRef *object.VirtualMachine, localIsoFilePath string) error { - p := soap.DefaultUpload - remoteIsoFilePath := fmt.Sprintf("%s/%s", vmRef.Name(), "cloud-init.iso") - // Get the datastore where VM files are located - datastore, err := getDatastoreFromVM(ctx, session, vmRef) - if err != nil { - return fmt.Errorf("error getting datastore from VM %s: %w", vmRef.Name(), err) - } - uploadLog := log.With("datastore", datastore, "source", localIsoFilePath, "destination", remoteIsoFilePath) - uploadLog.Debug("Uploading userdata ISO to datastore") - if err := datastore.UploadFile(ctx, localIsoFilePath, remoteIsoFilePath, &p); err != nil { - return fmt.Errorf("failed to upload iso: %w", err) - } - uploadLog.Debug("Uploaded ISO file") - - // Find the cd-rom device and insert the cloud init iso file into it. - devices, err := vmRef.Device(ctx) - if err != nil { - return fmt.Errorf("failed to get devices: %w", err) - } - - // passing empty cd-rom name so that the first one gets returned - cdrom, err := devices.FindCdrom("") - if err != nil { - return fmt.Errorf("failed to find cdrom device: %w", err) - } - cdrom.Connectable.StartConnected = true - iso := datastore.Path(remoteIsoFilePath) - return vmRef.EditDevice(ctx, devices.InsertIso(cdrom, iso)) -} - -func generateLocalUserdataISO(userdata, name string) (string, error) { - // We must create a directory, because the iso-generation commands - // take a directory as input - userdataDir, err := os.MkdirTemp(localTempDir, name) - if err != nil { - return "", fmt.Errorf("failed to create local temp directory for userdata at %s: %w", userdataDir, err) - } - defer func() { - if err := os.RemoveAll(userdataDir); err != nil { - utilruntime.HandleError(fmt.Errorf("error cleaning up local userdata tempdir %s: %w", userdataDir, err)) - } - }() - - userdataFilePath := fmt.Sprintf("%s/user-data", userdataDir) - metadataFilePath := fmt.Sprintf("%s/meta-data", userdataDir) - isoFilePath := fmt.Sprintf("%s/%s.iso", localTempDir, name) - - metadataTmpl, err := template.New("metadata").Parse(metaDataTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse metadata template: %w", err) - } - metadata := &bytes.Buffer{} - templateContext := struct { - InstanceID string - Hostname string - }{ - InstanceID: name, - Hostname: name, - } - if err = metadataTmpl.Execute(metadata, templateContext); err != nil { - return "", fmt.Errorf("failed to render metadata: %w", err) - } - - if err := os.WriteFile(userdataFilePath, []byte(userdata), 0644); err != nil { - return "", fmt.Errorf("failed to locally write userdata file to %s: %w", userdataFilePath, err) - } - - if err := os.WriteFile(metadataFilePath, metadata.Bytes(), 0644); err != nil { - return "", fmt.Errorf("failed to locally write metadata file to %s: %w", userdataFilePath, err) - } - - var command string - var args []string - - if _, err := exec.LookPath("genisoimage"); err == nil { - command = "genisoimage" - args = []string{"-o", isoFilePath, "-volid", "cidata", "-joliet", "-rock", userdataDir} - } else if _, err := exec.LookPath("mkisofs"); err == nil { - command = "mkisofs" - args = []string{"-o", isoFilePath, "-V", "cidata", "-J", "-R", userdataDir} - } else { - return "", errors.New("system is missing genisoimage or mkisofs, can't generate userdata iso without it") - } - - cmd := exec.Command(command, args...) - if output, err := cmd.CombinedOutput(); err != nil { - return "", fmt.Errorf("error executing command `%s %s`: output: `%s`, error: `%w`", command, args, string(output), err) - } - - return isoFilePath, nil -} - -func removeFloppyDevice(ctx context.Context, virtualMachine *object.VirtualMachine) error { - vmDevices, err := virtualMachine.Device(ctx) - if err != nil { - return fmt.Errorf("failed to get device list: %w", err) - } - - // If there is more than one floppy device attached, you will simply get the first one. We - // assume this won't happen. - floppyDevice, err := vmDevices.FindFloppy("") - if err != nil { - if err.Error() == "no floppy device found" { - return nil - } - return fmt.Errorf("failed to find floppy: %w", err) - } - - if err := virtualMachine.RemoveDevice(ctx, false, floppyDevice); err != nil { - return fmt.Errorf("failed to remove floppy device: %w", err) - } - - return nil -} - -func getDisksFromVM(ctx context.Context, vm *object.VirtualMachine) ([]*types.VirtualDisk, error) { - var props mo.VirtualMachine - if err := vm.Properties(ctx, vm.Reference(), nil, &props); err != nil { - return nil, fmt.Errorf("error getting VM template reference: %w", err) - } - l := object.VirtualDeviceList(props.Config.Hardware.Device) - disks := l.SelectByType((*types.VirtualDisk)(nil)) - - var result []*types.VirtualDisk - for _, disk := range disks { - if assertedDisk := disk.(*types.VirtualDisk); assertedDisk != nil { - result = append(result, assertedDisk) - } - } - return result, nil -} - -func validateDiskResizing(disks []*types.VirtualDisk, requestedSize int64) error { - if diskLen := len(disks); diskLen != 1 { - return fmt.Errorf("expected vm to have exactly one disk, got %d", diskLen) - } - requestedCapacityInBytes := requestedSize * int64(math.Pow(1024, 3)) - if requestedCapacityInBytes < disks[0].CapacityInBytes { - attachedDiskSizeInGiB := disks[0].CapacityInBytes / int64(math.Pow(1024, 3)) - return fmt.Errorf("requested diskSizeGB %d is smaller than size of attached disk(%dGiB)", requestedSize, attachedDiskSizeInGiB) - } - return nil -} - -// getDatastoreFromVM gets the datastore where the VM files are located. -func getDatastoreFromVM(ctx context.Context, session *Session, vmRef *object.VirtualMachine) (*object.Datastore, error) { - var props mo.VirtualMachine - // Obtain VM properties - if err := vmRef.Properties(ctx, vmRef.Reference(), nil, &props); err != nil { - return nil, fmt.Errorf("error getting VM properties: %w", err) - } - datastorePathObj := new(object.DatastorePath) - isSuccess := datastorePathObj.FromString(props.Summary.Config.VmPathName) - if !isSuccess { - return nil, fmt.Errorf("Failed to parse volPath: %s", props.Summary.Config.VmPathName) - } - return session.Finder.Datastore(ctx, datastorePathObj.Datastore) -} - -func resolveResourcePoolRef(ctx context.Context, config *Config, session *Session) (*types.ManagedObjectReference, error) { - if config.ResourcePool != "" { - targetResourcePool, err := session.Finder.ResourcePool(ctx, config.ResourcePool) - if err != nil { - return nil, fmt.Errorf("failed to get target resourcepool: %w", err) - } - return types.NewReference(targetResourcePool.Reference()), nil - } - return nil, nil -} - -func attachTags(ctx context.Context, log *zap.SugaredLogger, config *Config, vm *object.VirtualMachine) error { - restAPISession, err := NewRESTSession(ctx, config) - if err != nil { - return fmt.Errorf("failed to create REST API session: %w", err) - } - defer restAPISession.Logout(ctx) - tagManager := tags.NewManager(restAPISession.Client) - log.Debug("Attaching tags") - for _, tag := range config.Tags { - tagID, err := determineTagID(ctx, tagManager, tag) - if err != nil { - return err - } - - if err := tagManager.AttachTag(ctx, tagID, vm.Reference()); err != nil { - log.Debugw("Failed to attach tag; it was successfully deleted", "tag", tag) - return fmt.Errorf("failed to attach tag to VM: %v %w", tag.Name, err) - } - } - return nil -} - -func detachTags(ctx context.Context, log *zap.SugaredLogger, config *Config, vm *object.VirtualMachine) error { - restAPISession, err := NewRESTSession(ctx, config) - if err != nil { - return fmt.Errorf("failed to create REST API session: %w", err) - } - defer restAPISession.Logout(ctx) - tagManager := tags.NewManager(restAPISession.Client) - - attachedTags, err := tagManager.GetAttachedTags(ctx, vm.Reference()) - if err != nil { - return fmt.Errorf("failed to get attached tags for the VM: %s, %w", vm.Name(), err) - } - log.Debug("Deleting tags") - for _, tag := range attachedTags { - tagID, err := determineTagID(ctx, tagManager, tag) - if err != nil { - return err - } - - err = tagManager.DetachTag(ctx, tagID, vm.Reference()) - if err != nil { - return fmt.Errorf("failed to delete tag: %v %w", tag, err) - } - } - - return nil -} - -func determineTagID(ctx context.Context, tagManager *tags.Manager, tag tags.Tag) (string, error) { - if tag.ID != "" { - return tag.ID, nil - } - - apiTag, err := tagManager.GetTagForCategory(ctx, tag.Name, tag.CategoryID) - if err != nil { - return "", fmt.Errorf("failed to retrieve tag: %v %w", tag.Name, err) - } - return apiTag.ID, nil -} diff --git a/pkg/cloudprovider/provider/vsphere/helper_test.go b/pkg/cloudprovider/provider/vsphere/helper_test.go deleted file mode 100644 index 257339db4..000000000 --- a/pkg/cloudprovider/provider/vsphere/helper_test.go +++ /dev/null @@ -1,242 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vsphere - -import ( - "context" - "log" - "strings" - "testing" - "time" - - "github.com/vmware/govmomi/object" - "github.com/vmware/govmomi/simulator" - "github.com/vmware/govmomi/vim25/methods" - "github.com/vmware/govmomi/vim25/soap" - "github.com/vmware/govmomi/vim25/types" - "go.uber.org/zap" -) - -func TestResolveDatastoreRef(t *testing.T) { - tests := []struct { - name string - config *Config - wantErr bool - }{ - { - name: "Only Datastore defined", - config: &Config{ - Datastore: "LocalDS_0", - }, - wantErr: false, - }, - { - name: "Only DatastoreCluster defined", - config: &Config{ - DatastoreCluster: "DC0_POD0", - }, - wantErr: false, - }, - { - name: "Unknown DatastoreCluster", - config: &Config{ - DatastoreCluster: "DC0_POD1", - }, - wantErr: true, - }, - { - name: "Both Datastore and DatastoreCluster defined", - config: &Config{ - Datastore: "LocalDS_0", - DatastoreCluster: "DC0_POD0", - }, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() - - model := simulator.VPX() - // Pod == StoragePod == StorageCluster - model.Pod++ - model.Cluster++ - - defer model.Remove() - err := model.Create() - if err != nil { - log.Fatal(err) - } - - // Override the default StorageResourceManager for the purpose of the unit test. - ds := simulator.Map.Any("Datastore").(*simulator.Datastore) - obj := simulator.Map.Get(model.ServiceContent.StorageResourceManager.Reference()).(*simulator.StorageResourceManager) - csrm := &CustomStorageResourceManager{obj, ds} - simulator.Map.Put(csrm) - - s := model.Service.NewServer() - defer s.Close() - - // Setup config to be able to login to the simulator - // Remove trailing `/sdk` as it is appended by the session constructor - tt.config.VSphereURL = strings.TrimSuffix(s.URL.String(), "/sdk") - tt.config.Username = simulator.DefaultLogin.Username() - tt.config.Password, _ = simulator.DefaultLogin.Password() - tt.config.Datacenter = "DC0" - - session, err := NewSession(ctx, tt.config) - defer session.Logout(ctx) - if err != nil { - t.Fatalf("error creating session: %v", err) - } - dc, err := session.Datacenter.Folders(ctx) - if err != nil { - t.Fatalf("error getting datacenter folders: %v", err) - } - vmFolder := dc.VmFolder - vms, err := session.Finder.VirtualMachineList(ctx, "*") - if err != nil { - t.Fatalf("error getting virtual machines: %v", err) - } - - got, err := resolveDatastoreRef(ctx, zap.NewNop().Sugar(), tt.config, session, vms[2], vmFolder, &types.VirtualMachineCloneSpec{}) - if (err != nil) != tt.wantErr { - t.Errorf("resolveDatastoreRef() error = %v, wantErr %v", err, tt.wantErr) - return - } - if err == nil && got == nil { - t.Errorf("resolveDatastoreRef() should be not empty") - } - }) - } -} - -type CustomStorageResourceManager struct { - *simulator.StorageResourceManager - ds *simulator.Datastore -} - -// RecommendDatastores always return a recommendation for the purposes of the test. -func (c *CustomStorageResourceManager) RecommendDatastores(_ *types.RecommendDatastores) soap.HasFault { - body := &methods.RecommendDatastoresBody{} - res := &types.RecommendDatastoresResponse{} - ds := c.ds.Reference() - res.Returnval.Recommendations = append(res.Returnval.Recommendations, types.ClusterRecommendation{ - Key: "0", - Type: "V1", - Time: time.Now(), - Reason: "storagePlacement", - ReasonText: "Satisfy storage initial placement requests", - WarningDetails: (*types.LocalizableMessage)(nil), - Prerequisite: nil, - Action: []types.BaseClusterAction{ - &types.StoragePlacementAction{ - ClusterAction: types.ClusterAction{ - Type: "StoragePlacementV1", - Target: (*types.ManagedObjectReference)(nil), - }, - Vm: (*types.ManagedObjectReference)(nil), - Destination: ds, - }, - }, - }, - ) - - body.Res = res - return body -} - -func TestResolveResourcePoolRef(t *testing.T) { - tests := []struct { - name string - config *Config - wantErr bool - wantResourcePool bool - expectedResourcePool string - }{ - { - name: "No Resource Pool specified", - config: &Config{}, - wantErr: false, - wantResourcePool: false, - }, - { - name: "Resource Pool specified", - config: &Config{ - ResourcePool: "DC0_C0_RP1", - }, - wantErr: false, - wantResourcePool: true, - expectedResourcePool: "DC0_C0_RP1", - }, - { - name: "Resource Pool specified missing", - config: &Config{ - ResourcePool: "DC0_C0_RP1_WRONG", - }, - wantErr: true, - wantResourcePool: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() - - model := simulator.VPX() - model.Pool++ - model.Cluster++ - - defer model.Remove() - err := model.Create() - if err != nil { - log.Fatal(err) - } - - s := model.Service.NewServer() - defer s.Close() - - // Setup config to be able to login to the simulator - // Remove trailing `/sdk` as it is appended by the session constructor - tt.config.VSphereURL = strings.TrimSuffix(s.URL.String(), "/sdk") - tt.config.Username = simulator.DefaultLogin.Username() - tt.config.Password, _ = simulator.DefaultLogin.Password() - tt.config.Datacenter = "DC0" - - session, err := NewSession(ctx, tt.config) - defer session.Logout(ctx) - if err != nil { - t.Fatalf("error creating session: %v", err) - } - - got, err := resolveResourcePoolRef(ctx, tt.config, session) - if (err != nil) != tt.wantErr { - t.Errorf("error = %v, wantErr %v", err, tt.wantErr) - return - } - if tt.wantResourcePool != (got != nil) { - t.Errorf("resourcePool = %v, wantResourcePool %v", got, tt.wantResourcePool) - } - if tt.wantResourcePool { - rp := object.NewResourcePool(session.Client.Client, got.Reference()) - n, _ := rp.ObjectName(ctx) - if e, a := tt.expectedResourcePool, n; e != a { - t.Errorf("expected resource pool %v but got %+v", e, a) - } - } - }) - } -} diff --git a/pkg/cloudprovider/provider/vsphere/network.go b/pkg/cloudprovider/provider/vsphere/network.go deleted file mode 100644 index 2f3b6dc3d..000000000 --- a/pkg/cloudprovider/provider/vsphere/network.go +++ /dev/null @@ -1,81 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vsphere - -import ( - "context" - "fmt" - - "github.com/vmware/govmomi/object" - "github.com/vmware/govmomi/vim25/types" -) - -const ( - ethCardType = "vmxnet3" -) - -// Based on https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/v1.7.0/pkg/services/govmomi/vcenter/clone.go#L372 -func GetNetworkSpecs(ctx context.Context, session *Session, devices object.VirtualDeviceList, network string, networks []string) ([]types.BaseVirtualDeviceConfigSpec, error) { - deviceSpecs := []types.BaseVirtualDeviceConfigSpec{} - - // Remove any existing NICs. - for _, dev := range devices.SelectByType((*types.VirtualEthernetCard)(nil)) { - deviceSpecs = append(deviceSpecs, &types.VirtualDeviceConfigSpec{ - Device: dev, - Operation: types.VirtualDeviceConfigSpecOperationRemove, - }) - } - - // Add the default network if no networks are specified. - if network != "" { - networks = append(networks, network) - } - - // Add NICs for each network. - deviceKey := int32(-100) - for _, net := range networks { - // Add new NICs based on the machine config. - ref, err := session.Finder.Network(ctx, net) - if err != nil { - return nil, fmt.Errorf("failed to find network %q: %w", net, err) - } - backing, err := ref.EthernetCardBackingInfo(ctx) - if err != nil { - return nil, fmt.Errorf("failed to create new ethernet card backing info for network %q: %w", net, err) - } - dev, err := object.EthernetCardTypes().CreateEthernetCard(ethCardType, backing) - if err != nil { - return nil, fmt.Errorf("failed to create new ethernet card %q for network %q: %v", ethCardType, net, ctx) - } - - // Get the actual NIC object. This is safe to assert without a check - // because "object.EthernetCardTypes().CreateEthernetCard" returns a - // "types.BaseVirtualEthernetCard" as a "types.BaseVirtualDevice". - nic := dev.(types.BaseVirtualEthernetCard).GetVirtualEthernetCard() - - // Assign a temporary device key to ensure that a unique one will be - // generated when the device is created. - nic.Key = deviceKey - - deviceSpecs = append(deviceSpecs, &types.VirtualDeviceConfigSpec{ - Device: dev, - Operation: types.VirtualDeviceConfigSpecOperationAdd, - }) - deviceKey-- - } - return deviceSpecs, nil -} diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go deleted file mode 100644 index 3166ac24f..000000000 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ /dev/null @@ -1,695 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vsphere - -import ( - "context" - "errors" - "fmt" - "net/url" - "os" - "strings" - - "github.com/vmware/govmomi/find" - "github.com/vmware/govmomi/object" - "github.com/vmware/govmomi/property" - "github.com/vmware/govmomi/vapi/tags" - "github.com/vmware/govmomi/vim25/mo" - "github.com/vmware/govmomi/vim25/types" - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - vspheretypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vsphere/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - - corev1 "k8s.io/api/core/v1" - ktypes "k8s.io/apimachinery/pkg/types" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" -) - -type provider struct { - configVarResolver *providerconfig.ConfigVarResolver -} - -// New returns a VSphere provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - provider := &provider{configVarResolver: configVarResolver} - return provider -} - -// Config contains vSphere provider configuration. -type Config struct { - TemplateVMName string - VMNetName string - Networks []string - Username string - Password string - VSphereURL string - Datacenter string - Cluster string - Folder string - ResourcePool string - Datastore string - DatastoreCluster string - AllowInsecure bool - VMAntiAffinity bool - CPUs int32 - MemoryMB int64 - DiskSizeGB *int64 - Tags []tags.Tag -} - -// Ensures that Server implements Instance interface. -var _ instance.Instance = &Server{} - -// Server holds vSphere server information. -type Server struct { - name string - id string - uuid string - status instance.Status - addresses map[string]corev1.NodeAddressType -} - -func (vsphereServer Server) HostID() string { - return vsphereServer.ID() -} - -func (vsphereServer Server) Name() string { - return vsphereServer.name -} - -func (vsphereServer Server) ID() string { - return vsphereServer.id -} - -func (vsphereServer Server) ProviderID() string { - if vsphereServer.uuid == "" { - return "" - } - return "vsphere://" + vsphereServer.uuid -} - -func (vsphereServer Server) Addresses() map[string]corev1.NodeAddressType { - return vsphereServer.addresses -} - -func (vsphereServer Server) Status() instance.Status { - return vsphereServer.status -} - -// Ensures that provider implements Provider interface. -var _ cloudprovidertypes.Provider = &provider{} - -func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { - return spec, nil -} - -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *vspheretypes.RawConfig, error) { - if provSpec.Value == nil { - return nil, nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - - pconfig, err := providerconfigtypes.GetConfig(provSpec) - if err != nil { - return nil, nil, nil, err - } - - if pconfig.OperatingSystemSpec.Raw == nil { - return nil, nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") - } - - rawConfig, err := vspheretypes.GetConfig(*pconfig) - if err != nil { - return nil, nil, nil, err - } - - c := Config{} - c.TemplateVMName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.TemplateVMName) - if err != nil { - return nil, nil, nil, err - } - - //nolint:staticcheck - //lint:ignore SA1019: rawConfig.VMNetName is deprecated: use networks instead. - c.VMNetName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VMNetName) - if err != nil { - return nil, nil, nil, err - } - - for _, network := range rawConfig.Networks { - networkValue, err := p.configVarResolver.GetConfigVarStringValue(network) - if err != nil { - return nil, nil, rawConfig, err - } - c.Networks = append(c.Networks, networkValue) - } - - c.Username, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Username, "VSPHERE_USERNAME") - if err != nil { - return nil, nil, nil, err - } - - c.Password, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Password, "VSPHERE_PASSWORD") - if err != nil { - return nil, nil, nil, err - } - - c.VSphereURL, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.VSphereURL, "VSPHERE_ADDRESS") - if err != nil { - return nil, nil, nil, err - } - - c.Datacenter, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Datacenter) - if err != nil { - return nil, nil, nil, err - } - - c.Cluster, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Cluster) - if err != nil { - return nil, nil, nil, err - } - - c.Folder, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Folder) - if err != nil { - return nil, nil, nil, err - } - - c.ResourcePool, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ResourcePool) - if err != nil { - return nil, nil, nil, err - } - - c.Datastore, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Datastore) - if err != nil { - return nil, nil, nil, err - } - - c.DatastoreCluster, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.DatastoreCluster) - if err != nil { - return nil, nil, nil, err - } - - c.AllowInsecure, err = p.configVarResolver.GetConfigVarBoolValueOrEnv(rawConfig.AllowInsecure, "VSPHERE_ALLOW_INSECURE") - if err != nil { - return nil, nil, nil, err - } - - c.VMAntiAffinity, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.VMAntiAffinity) - if err != nil { - return nil, nil, nil, err - } - - c.CPUs = rawConfig.CPUs - c.MemoryMB = rawConfig.MemoryMB - c.DiskSizeGB = rawConfig.DiskSizeGB - - for _, tag := range rawConfig.Tags { - c.Tags = append(c.Tags, tags.Tag{ - Description: tag.Description, - ID: tag.ID, - Name: tag.Name, - CategoryID: tag.CategoryID, - }) - } - - return &c, pconfig, rawConfig, nil -} - -func (p *provider) Validate(ctx context.Context, log *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { - config, _, _, err := p.getConfig(spec.ProviderSpec) - if err != nil { - return fmt.Errorf("failed to get config: %w", err) - } - - session, err := NewSession(ctx, config) - if err != nil { - return fmt.Errorf("failed to create vCenter session: %w", err) - } - defer session.Logout(ctx) - - if len(config.Networks) > 0 && config.VMNetName != "" { - return fmt.Errorf("both networks and vmNetName are specified, only one of them can be used") - } - - if config.Tags != nil { - restAPISession, err := NewRESTSession(ctx, config) - if err != nil { - return fmt.Errorf("failed to create REST API session: %w", err) - } - defer restAPISession.Logout(ctx) - tagManager := tags.NewManager(restAPISession.Client) - log.Debug("Found tags") - for _, tag := range config.Tags { - if tag.ID == "" && tag.Name == "" { - return fmt.Errorf("either tag id or name must be specified") - } - if tag.CategoryID == "" { - return fmt.Errorf("one of the tags category is empty") - } - if _, err := tagManager.GetCategory(ctx, tag.CategoryID); err != nil { - return fmt.Errorf("can't get the category with ID %s, %w", tag.CategoryID, err) - } - } - log.Debug("Tag validation passed") - } - - // Only and only one between datastore and datastre cluster should be - // present, otherwise an error is raised. - if config.DatastoreCluster != "" && config.Datastore == "" { - if _, err := session.Finder.DatastoreCluster(ctx, config.DatastoreCluster); err != nil { - return fmt.Errorf("failed to get datastore cluster %s: %w", config.DatastoreCluster, err) - } - } else if config.Datastore != "" && config.DatastoreCluster == "" { - if _, err := session.Finder.Datastore(ctx, config.Datastore); err != nil { - return fmt.Errorf("failed to get datastore %s: %w", config.Datastore, err) - } - } else { - return fmt.Errorf("one between datastore and datastore cluster should be specified: %w", err) - } - - if _, err := session.Finder.Folder(ctx, config.Folder); err != nil { - return fmt.Errorf("failed to get folder %q: %w", config.Folder, err) - } - - if _, err := p.get(ctx, config.Folder, spec, session.Finder); err == nil { - return fmt.Errorf("a vm %s/%s already exists", config.Folder, spec.Name) - } - - if config.ResourcePool != "" { - if _, err := session.Finder.ResourcePool(ctx, config.ResourcePool); err != nil { - return fmt.Errorf("failed to get resourcepool %q: %w", config.ResourcePool, err) - } - } - - templateVM, err := session.Finder.VirtualMachine(ctx, config.TemplateVMName) - if err != nil { - return fmt.Errorf("failed to get template vm %q: %w", config.TemplateVMName, err) - } - - disks, err := getDisksFromVM(ctx, templateVM) - if err != nil { - return fmt.Errorf("failed to get disks from VM: %w", err) - } - if diskLen := len(disks); diskLen != 1 { - return fmt.Errorf("expected vm to have exactly one disk, had %d", diskLen) - } - - if config.DiskSizeGB != nil { - if err := validateDiskResizing(disks, *config.DiskSizeGB); err != nil { - return err - } - } - - if config.VMAntiAffinity { - if config.Cluster == "" { - return fmt.Errorf("cluster is required for vm anti affinity") - } - _, err = session.Finder.ClusterComputeResource(ctx, config.Cluster) - if err != nil { - return fmt.Errorf("failed to get cluster %q, %w", config.Cluster, err) - } - } - - return nil -} - -func machineInvalidConfigurationTerminalError(err error) error { - return cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: err.Error(), - } -} - -func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { - vm, err := p.create(ctx, log, machine, userdata) - if err != nil { - _, cleanupErr := p.Cleanup(ctx, log, machine, data) - if cleanupErr != nil { - return nil, fmt.Errorf("cleaning up failed with err %w after creation failed with err %w", cleanupErr, err) - } - return nil, err - } - return vm, nil -} - -func (p *provider) create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { - config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, fmt.Errorf("failed to parse config: %w", err) - } - - session, err := NewSession(ctx, config) - if err != nil { - return nil, fmt.Errorf("failed to create vCenter session: %w", err) - } - defer session.Logout(ctx) - - var containerLinuxUserdata string - if pc.OperatingSystem == providerconfigtypes.OperatingSystemFlatcar { - containerLinuxUserdata = userdata - } - - virtualMachine, err := createClonedVM( - ctx, - log, - machine.Spec.Name, - config, - session, - containerLinuxUserdata, - ) - if err != nil { - return nil, machineInvalidConfigurationTerminalError(fmt.Errorf("failed to create cloned vm: '%w'", err)) - } - - if err := attachTags(ctx, log, config, virtualMachine); err != nil { - return nil, fmt.Errorf("failed to attach tags: %w", err) - } - - if config.VMAntiAffinity { - if err := p.createOrUpdateVMAntiAffinityRule(ctx, log, session, machine, config); err != nil { - return nil, fmt.Errorf("failed to add VM to anti affinity rule: %w", err) - } - } - - if pc.OperatingSystem != providerconfigtypes.OperatingSystemFlatcar { - localUserdataIsoFilePath, err := generateLocalUserdataISO(userdata, machine.Spec.Name) - if err != nil { - return nil, fmt.Errorf("failed to generate local userdadata iso: %w", err) - } - - defer func() { - err := os.Remove(localUserdataIsoFilePath) - if err != nil { - utilruntime.HandleError(fmt.Errorf("failed to clean up local userdata iso file at %s: %w", localUserdataIsoFilePath, err)) - } - }() - - if err := uploadAndAttachISO(ctx, log, session, virtualMachine, localUserdataIsoFilePath); err != nil { - // Destroy VM to avoid a leftover. - destroyTask, vmErr := virtualMachine.Destroy(ctx) - if vmErr != nil { - return nil, fmt.Errorf("failed to destroy vm %s after failing upload and attach userdata iso: %w / %w", virtualMachine.Name(), err, vmErr) - } - if vmErr := destroyTask.Wait(ctx); vmErr != nil { - return nil, fmt.Errorf("failed to destroy vm %s after failing upload and attach userdata iso: %w / %w", virtualMachine.Name(), err, vmErr) - } - return nil, machineInvalidConfigurationTerminalError(fmt.Errorf("failed to upload and attach userdata iso: %w", err)) - } - } - - powerOnTask, err := virtualMachine.PowerOn(ctx) - if err != nil { - return nil, fmt.Errorf("failed to power on machine: %w", err) - } - - if err := powerOnTask.Wait(ctx); err != nil { - return nil, fmt.Errorf("error when waiting for vm powerOn task: %w", err) - } - - return Server{name: virtualMachine.Name(), status: instance.StatusRunning, id: virtualMachine.Reference().Value, uuid: virtualMachine.UUID(ctx)}, nil -} - -func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { - config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return false, fmt.Errorf("failed to parse config: %w", err) - } - - session, err := NewSession(ctx, config) - if err != nil { - return false, fmt.Errorf("failed to create vCenter session: %w", err) - } - defer session.Logout(ctx) - - virtualMachine, err := p.get(ctx, config.Folder, machine.Spec, session.Finder) - if err != nil { - if cloudprovidererrors.IsNotFound(err) { - return true, nil - } - return false, fmt.Errorf("failed to get instance from vSphere: %w", err) - } - - if err := detachTags(ctx, log, config, virtualMachine); err != nil { - return false, fmt.Errorf("failed to delete tags: %w", err) - } - - if config.VMAntiAffinity { - if err := p.createOrUpdateVMAntiAffinityRule(ctx, log, session, machine, config); err != nil { - return false, fmt.Errorf("failed to update VMs in anti-affinity rule: %w", err) - } - } - - powerState, err := virtualMachine.PowerState(ctx) - if err != nil { - return false, fmt.Errorf("failed to get virtual machine power state: %w", err) - } - - // We cannot destroy a VM that's powered on, but we also - // cannot power off a machine that is already off. - if powerState != types.VirtualMachinePowerStatePoweredOff { - powerOffTask, err := virtualMachine.PowerOff(ctx) - if err != nil { - return false, fmt.Errorf("failed to poweroff vm %s: %w", virtualMachine.Name(), err) - } - if err = powerOffTask.Wait(ctx); err != nil { - return false, fmt.Errorf("failed to poweroff vm %s: %w", virtualMachine.Name(), err) - } - } - - virtualMachineDeviceList, err := virtualMachine.Device(ctx) - if err != nil { - return false, fmt.Errorf("failed to get devices for virtual machine: %w", err) - } - - pvs := &corev1.PersistentVolumeList{} - if err := data.Client.List(data.Ctx, pvs); err != nil { - return false, fmt.Errorf("failed to list PVs: %w", err) - } - - for _, pv := range pvs.Items { - if pv.Spec.VsphereVolume == nil { - continue - } - for _, device := range virtualMachineDeviceList { - if virtualMachineDeviceList.Type(device) == object.DeviceTypeDisk { - fileName := device.GetVirtualDevice().Backing.(types.BaseVirtualDeviceFileBackingInfo).GetVirtualDeviceFileBackingInfo().FileName - if pv.Spec.VsphereVolume.VolumePath == fileName { - if err := virtualMachine.RemoveDevice(ctx, true, device); err != nil { - return false, fmt.Errorf("error detaching pv-backing disk %s: %w", fileName, err) - } - } - } - } - } - - datastore, err := getDatastoreFromVM(ctx, session, virtualMachine) - if err != nil { - return false, fmt.Errorf("Error getting datastore from VM %s: %w", virtualMachine.Name(), err) - } - destroyTask, err := virtualMachine.Destroy(ctx) - if err != nil { - return false, fmt.Errorf("failed to destroy vm %s: %w", virtualMachine.Name(), err) - } - if err := destroyTask.Wait(ctx); err != nil { - return false, fmt.Errorf("failed to destroy vm %s: %w", virtualMachine.Name(), err) - } - - if pc.OperatingSystem != providerconfigtypes.OperatingSystemFlatcar { - filemanager := datastore.NewFileManager(session.Datacenter, false) - - if err := filemanager.Delete(ctx, virtualMachine.Name()); err != nil { - if err.Error() == fmt.Sprintf("File [%s] %s was not found", datastore.Name(), virtualMachine.Name()) { - return true, nil - } - return false, fmt.Errorf("failed to delete storage of deleted instance %s: %w", virtualMachine.Name(), err) - } - } - - log.Infow("Successfully destroyed vm", "vm", virtualMachine.Name()) - return true, nil -} - -func (p *provider) Get(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, fmt.Errorf("failed to parse config: %w", err) - } - - session, err := NewSession(ctx, config) - if err != nil { - return nil, fmt.Errorf("failed to create vCenter session: %w", err) - } - defer session.Logout(ctx) - - virtualMachine, err := p.get(ctx, config.Folder, machine.Spec, session.Finder) - if err != nil { - // Must not wrap because we match on the error type - return nil, err - } - - powerState, err := virtualMachine.PowerState(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get powerstate: %w", err) - } - - if powerState != types.VirtualMachinePowerStatePoweredOn { - powerOnTask, err := virtualMachine.PowerOn(ctx) - if err != nil { - return nil, fmt.Errorf("failed to power on instance that was in state %q: %w", powerState, err) - } - if err := powerOnTask.Wait(ctx); err != nil { - return nil, fmt.Errorf("failed waiting for instance to be powered on: %w", err) - } - // We must return here because the vendored code for determining if the guest - // utils are running yields an NPD when using with an instance that is not running - return Server{name: virtualMachine.Name(), status: instance.StatusUnknown, uuid: virtualMachine.UUID(ctx)}, nil - } - - // virtualMachine.IsToolsRunning panics when executed on a VM that is not powered on - addresses := map[string]corev1.NodeAddressType{} - isGuestToolsRunning, err := virtualMachine.IsToolsRunning(ctx) - if err != nil { - return nil, fmt.Errorf("failed to check if guest utils are running: %w", err) - } - if isGuestToolsRunning { - var moVirtualMachine mo.VirtualMachine - pc := property.DefaultCollector(session.Client.Client) - if err := pc.RetrieveOne(ctx, virtualMachine.Reference(), []string{"guest"}, &moVirtualMachine); err != nil { - return nil, fmt.Errorf("failed to retrieve guest info: %w", err) - } - - for _, nic := range moVirtualMachine.Guest.Net { - for _, address := range nic.IpAddress { - // Exclude ipv6 link-local addresses and default Docker bridge - if !strings.HasPrefix(address, "fe80:") && !strings.HasPrefix(address, "172.17.") { - addresses[address] = "" - } - } - } - } else { - log.Debug("Can't fetch the IP addresses for machine, the VMware guest utils are not running yet. This might take a few minutes") - } - - return Server{name: virtualMachine.Name(), status: instance.StatusRunning, addresses: addresses, id: virtualMachine.Reference().Value, uuid: virtualMachine.UUID(ctx)}, nil -} - -func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ ktypes.UID) error { - return nil -} - -func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { - c, _, _, err := p.getConfig(spec.ProviderSpec) - if err != nil { - return "", "", fmt.Errorf("failed to parse config: %w", err) - } - - passedURL := c.VSphereURL - // Required because url.Parse returns an empty string for the hostname if there was no schema - if !strings.HasPrefix(passedURL, "https://") { - passedURL = "https://" + passedURL - } - - u, err := url.Parse(passedURL) - if err != nil { - return "", "", fmt.Errorf("failed to parse '%s' as url: %w", passedURL, err) - } - - workingDir := c.Folder - // Default to basedir - if workingDir == "" { - workingDir = fmt.Sprintf("/%s/vm", c.Datacenter) - } - - datastore := c.Datastore - if datastore == "" { - datastore = c.DatastoreCluster - } - - cc := &vspheretypes.CloudConfig{ - Global: vspheretypes.GlobalOpts{ - User: c.Username, - Password: c.Password, - InsecureFlag: c.AllowInsecure, - VCenterPort: u.Port(), - }, - Disk: vspheretypes.DiskOpts{ - SCSIControllerType: "pvscsi", - }, - Workspace: vspheretypes.WorkspaceOpts{ - Datacenter: c.Datacenter, - VCenterIP: u.Hostname(), - DefaultDatastore: datastore, - Folder: workingDir, - }, - VirtualCenter: map[string]*vspheretypes.VirtualCenterConfig{ - u.Hostname(): { - VCenterPort: u.Port(), - Datacenters: c.Datacenter, - User: c.Username, - Password: c.Password, - }, - }, - } - - s, err := cc.String() - if err != nil { - return "", "", fmt.Errorf("failed to convert the cloud-config to string: %w", err) - } - - return s, "vsphere", nil -} - -func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { - labels := make(map[string]string) - - c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err == nil { - labels["size"] = fmt.Sprintf("%d-cpus-%d-mb", c.CPUs, c.MemoryMB) - labels["dc"] = c.Datacenter - } - - return labels, err -} - -func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { - return nil -} - -func (p *provider) get(ctx context.Context, folder string, spec clusterv1alpha1.MachineSpec, datacenterFinder *find.Finder) (*object.VirtualMachine, error) { - path := fmt.Sprintf("%s/%s", folder, spec.Name) - virtualMachineList, err := datacenterFinder.VirtualMachineList(ctx, path) - if err != nil { - if err.Error() == fmt.Sprintf("vm '%s' not found", path) { - return nil, cloudprovidererrors.ErrInstanceNotFound - } - return nil, fmt.Errorf("failed to list virtual machines: %w", err) - } - - if len(virtualMachineList) == 0 { - return nil, cloudprovidererrors.ErrInstanceNotFound - } - if n := len(virtualMachineList); n > 1 { - return nil, fmt.Errorf("expected to find at most one vm at %q, got %d", path, n) - } - return virtualMachineList[0], nil -} diff --git a/pkg/cloudprovider/provider/vsphere/provider_test.go b/pkg/cloudprovider/provider/vsphere/provider_test.go deleted file mode 100644 index 53e5493bf..000000000 --- a/pkg/cloudprovider/provider/vsphere/provider_test.go +++ /dev/null @@ -1,188 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vsphere - -import ( - "bytes" - "context" - "log" - "strings" - "testing" - "text/template" - - "github.com/vmware/govmomi/simulator" - "go.uber.org/zap" - - cloudprovidertesting "github.com/kubermatic/machine-controller/pkg/cloudprovider/testing" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - - "k8s.io/utils/ptr" -) - -type vsphereProviderSpecConf struct { - Datastore *string - DatastoreCluster *string - User string - Password string - URL string -} - -func (v vsphereProviderSpecConf) rawProviderSpec(t *testing.T) []byte { - var out bytes.Buffer - tmpl, err := template.New("test").Parse(`{ - "cloudProvider": "vsphere", - "cloudProviderSpec": { - "allowInsecure": false, - "vmAntiAffinity": true, - "cluster": "DC0_C0", - "cpus": 1, - "datacenter": "DC0", - {{- if .Datastore }} - "datastore": "{{ .Datastore }}", - {{- end }} - {{- if .DatastoreCluster }} - "datastoreCluster": "{{ .DatastoreCluster }}", - {{- end }} - "folder": "/", - "resourcePool": "/DC0/host/DC0_C0/Resources", - "memoryMB": 2000, - "password": "{{ .Password }}", - "templateVMName": "DC0_H0_VM0", - "username": "{{ .User }}", - "networks": [ - "" - ], - "vsphereURL": "{{ .URL }}" - }, - "operatingSystem": "flatcar", - "operatingSystemSpec": { - "disableAutoUpdate": false, - "disableLocksmithD": true, - "disableUpdateEngine": false - } -}`) - if err != nil { - t.Fatalf("Error occurred while parsing openstack provider spec template: %v", err) - } - err = tmpl.Execute(&out, v) - if err != nil { - t.Fatalf("Error occurred while executing openstack provider spec template: %v", err) - } - t.Logf("Generated providerSpec: %s", out.String()) - return out.Bytes() -} - -func TestValidate(t *testing.T) { - tests := []struct { - name string - args vsphereProviderSpecConf - getConfigErr error - wantErr bool - }{ - { - name: "Valid Datastore", - args: vsphereProviderSpecConf{ - Datastore: ptr.To("LocalDS_0"), - }, - getConfigErr: nil, - wantErr: false, - }, - { - name: "Valid Datastore end empty DatastoreCluster", - args: vsphereProviderSpecConf{ - Datastore: ptr.To("LocalDS_0"), - DatastoreCluster: ptr.To(""), - }, - getConfigErr: nil, - wantErr: false, - }, - { - name: "Valid DatastoreCluster", - args: vsphereProviderSpecConf{ - DatastoreCluster: ptr.To("DC0_POD0"), - }, - getConfigErr: nil, - wantErr: false, - }, - { - name: "Invalid Datastore", - args: vsphereProviderSpecConf{ - Datastore: ptr.To("LocalDS_10"), - }, - getConfigErr: nil, - wantErr: true, - }, - { - name: "Invalid DatastoreCluster", - args: vsphereProviderSpecConf{ - Datastore: ptr.To("DC0_POD10"), - }, - getConfigErr: nil, - wantErr: true, - }, - { - name: "Both Datastore and DatastoreCluster specified", - args: vsphereProviderSpecConf{ - Datastore: ptr.To("DC0_POD10"), - DatastoreCluster: ptr.To("DC0_POD0"), - }, - getConfigErr: nil, - wantErr: true, - }, - { - name: "Neither Datastore nor DatastoreCluster specified", - args: vsphereProviderSpecConf{}, - getConfigErr: nil, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - model := simulator.VPX() - // Pod == StoragePod == StorageCluster - model.Pod++ - model.Cluster++ - - defer model.Remove() - err := model.Create() - if err != nil { - log.Fatal(err) - } - - s := model.Service.NewServer() - defer s.Close() - - // Setup config to be able to login to the simulator - // Remove trailing `/sdk` as it is appended by the session constructor - vSphereURL := strings.TrimSuffix(s.URL.String(), "/sdk") - username := simulator.DefaultLogin.Username() - password, _ := simulator.DefaultLogin.Password() - p := &provider{ - // Note that configVarResolver is not used in this test as the getConfigFunc is mocked. - configVarResolver: &providerconfig.ConfigVarResolver{}, - } - tt.args.User = username - tt.args.Password = password - tt.args.URL = vSphereURL - m := cloudprovidertesting.Creator{Name: "test", Namespace: "vsphere", ProviderSpecGetter: tt.args.rawProviderSpec}. - CreateMachine(t) - if err := p.Validate(context.Background(), zap.NewNop().Sugar(), m.Spec); (err != nil) != tt.wantErr { - t.Errorf("provider.Validate() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/pkg/cloudprovider/provider/vsphere/rule.go b/pkg/cloudprovider/provider/vsphere/rule.go deleted file mode 100644 index 4fea8e276..000000000 --- a/pkg/cloudprovider/provider/vsphere/rule.go +++ /dev/null @@ -1,188 +0,0 @@ -/* -Copyright 2023 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vsphere - -import ( - "context" - "errors" - "fmt" - "strings" - "sync" - - "github.com/vmware/govmomi/find" - "github.com/vmware/govmomi/object" - "github.com/vmware/govmomi/vim25/mo" - "github.com/vmware/govmomi/vim25/types" - "go.uber.org/zap" - - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - - "k8s.io/utils/ptr" -) - -var lock sync.Mutex - -// createOrUpdateVMAntiAffinityRule creates or updates an anti affinity rule with the name in the given cluster. -// VMs are attached to the rule based on their folder path and name prefix in vsphere. -// A minimum of two VMs is required. -func (p *provider) createOrUpdateVMAntiAffinityRule(ctx context.Context, log *zap.SugaredLogger, session *Session, machine *clusterv1alpha1.Machine, config *Config) error { - lock.Lock() - defer lock.Unlock() - cluster, err := session.Finder.ClusterComputeResource(ctx, config.Cluster) - if err != nil { - return err - } - - machineSetName := machine.Name[:strings.LastIndex(machine.Name, "-")] - vmsInFolder, err := session.Finder.VirtualMachineList(ctx, strings.Join([]string{config.Folder, "*"}, "/")) - if err != nil { - if errors.Is(err, &find.NotFoundError{}) { - return removeVMAntiAffinityRule(ctx, session, config.Cluster, machineSetName) - } - return err - } - - var ruleVMRef []types.ManagedObjectReference - for _, vm := range vmsInFolder { - // Only add VMs with the same machineSetName to the rule and exclude the machine itself if it is being deleted - if strings.HasPrefix(vm.Name(), machineSetName) && !(vm.Name() == machine.Name && machine.DeletionTimestamp != nil) { - ruleVMRef = append(ruleVMRef, vm.Reference()) - } - } - - if len(ruleVMRef) == 0 { - log.Debugf("No VMs in folder %s with name prefix %s found", config.Folder, machineSetName) - return removeVMAntiAffinityRule(ctx, session, config.Cluster, machineSetName) - } else if len(ruleVMRef) < 2 { - // DRS rule must have at least two virtual machine members - log.Debugf("Not enough VMs in folder %s to create anti-affinity rule", config.Folder) - return nil - } - - info, err := findClusterAntiAffinityRuleByName(ctx, cluster, machineSetName) - if err != nil { - return err - } - - log.Debugf("Creating or updating anti-affinity rule for VMs %v in cluster %s", ruleVMRef, config.Cluster) - operation := types.ArrayUpdateOperationEdit - - //create new rule - if info == nil { - info = &types.ClusterAntiAffinityRuleSpec{ - ClusterRuleInfo: types.ClusterRuleInfo{ - Enabled: ptr.To(true), - Mandatory: ptr.To(false), - Name: machineSetName, - UserCreated: ptr.To(true), - }, - } - operation = types.ArrayUpdateOperationAdd - } - - info.Vm = ruleVMRef - spec := &types.ClusterConfigSpecEx{ - RulesSpec: []types.ClusterRuleSpec{ - { - ArrayUpdateSpec: types.ArrayUpdateSpec{ - Operation: operation, - }, - Info: info, - }, - }, - } - - log.Debugf("Performing %q for anti-affinity rule for VMs %v in cluster %s", operation, ruleVMRef, config.Cluster) - task, err := cluster.Reconfigure(ctx, spec, true) - if err != nil { - return err - } - - taskResult, err := task.WaitForResult(ctx) - if err != nil { - return fmt.Errorf("error waiting for cluster %v reconfiguration to complete", cluster.Name()) - } - if taskResult.State != types.TaskInfoStateSuccess { - return fmt.Errorf("cluster %v reconfiguration task was not successful", cluster.Name()) - } - log.Debugf("Successfully created/updated anti-affinity rule for machineset %v against machine %v", machineSetName, machine.Name) - - return nil -} - -// removeVMAntiAffinityRule removes an anti affinity rule with the name in the given cluster. -func removeVMAntiAffinityRule(ctx context.Context, session *Session, clusterPath string, name string) error { - cluster, err := session.Finder.ClusterComputeResource(ctx, clusterPath) - if err != nil { - return err - } - - info, err := findClusterAntiAffinityRuleByName(ctx, cluster, name) - if err != nil { - return err - } - - // no rule found - if info == nil { - return nil - } - - spec := &types.ClusterConfigSpecEx{ - RulesSpec: []types.ClusterRuleSpec{ - { - ArrayUpdateSpec: types.ArrayUpdateSpec{ - Operation: types.ArrayUpdateOperationRemove, - RemoveKey: info.Key, - }, - }, - }, - } - - task, err := cluster.Reconfigure(ctx, spec, true) - if err != nil { - return err - } - - taskResult, err := task.WaitForResult(ctx) - if err != nil { - return fmt.Errorf("error waiting for cluster %v reconfiguration to complete", cluster.Name()) - } - if taskResult.State != types.TaskInfoStateSuccess { - return fmt.Errorf("cluster %v reconfiguration task was not successful", cluster.Name()) - } - return nil -} - -func findClusterAntiAffinityRuleByName(ctx context.Context, cluster *object.ClusterComputeResource, name string) (*types.ClusterAntiAffinityRuleSpec, error) { - var props mo.ClusterComputeResource - if err := cluster.Properties(ctx, cluster.Reference(), nil, &props); err != nil { - return nil, err - } - - var info *types.ClusterAntiAffinityRuleSpec - for _, clusterRuleInfo := range props.ConfigurationEx.(*types.ClusterConfigInfoEx).Rule { - if clusterRuleInfo.GetClusterRuleInfo().Name == name { - if vmAffinityRuleInfo, ok := clusterRuleInfo.(*types.ClusterAntiAffinityRuleSpec); ok { - info = vmAffinityRuleInfo - break - } - return nil, fmt.Errorf("rule name %s in cluster %q is not a VM anti-affinity rule", name, cluster.Name()) - } - } - - return info, nil -} diff --git a/pkg/cloudprovider/provider/vsphere/types/cloudconfig.go b/pkg/cloudprovider/provider/vsphere/types/cloudconfig.go deleted file mode 100644 index a20bcda8b..000000000 --- a/pkg/cloudprovider/provider/vsphere/types/cloudconfig.go +++ /dev/null @@ -1,143 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "bytes" - "fmt" - "text/template" - - "github.com/Masterminds/sprig/v3" - - "github.com/kubermatic/machine-controller/pkg/ini" -) - -const ( - cloudConfigTpl = `[Global] -user = {{ .Global.User | iniEscape }} -password = {{ .Global.Password | iniEscape }} -port = {{ .Global.VCenterPort | iniEscape }} -insecure-flag = {{ .Global.InsecureFlag }} -working-dir = {{ .Global.WorkingDir | iniEscape }} -datacenter = {{ .Global.Datacenter | iniEscape }} -datastore = {{ .Global.DefaultDatastore | iniEscape }} -server = {{ .Global.VCenterIP | iniEscape }} -{{- if .Global.IPFamily }} -ip-family = {{ .Global.IPFamily | iniEscape }} -{{- end }} - -[Disk] -scsicontrollertype = {{ .Disk.SCSIControllerType | iniEscape }} - -[Workspace] -server = {{ .Workspace.VCenterIP | iniEscape }} -datacenter = {{ .Workspace.Datacenter | iniEscape }} -folder = {{ .Workspace.Folder | iniEscape }} -default-datastore = {{ .Workspace.DefaultDatastore | iniEscape }} -resourcepool-path = {{ .Workspace.ResourcePoolPath | iniEscape }} - -{{ range $name, $vc := .VirtualCenter }} -[VirtualCenter {{ $name | iniEscape }}] -user = {{ $vc.User | iniEscape }} -password = {{ $vc.Password | iniEscape }} -port = {{ $vc.VCenterPort }} -datacenters = {{ $vc.Datacenters | iniEscape }} -{{- if $vc.IPFamily }} -ip-family = {{ $vc.IPFamily | iniEscape }} -{{- end }} -{{ end }} -` -) - -type WorkspaceOpts struct { - VCenterIP string `gcfg:"server"` - Datacenter string `gcfg:"datacenter"` - Folder string `gcfg:"folder"` - DefaultDatastore string `gcfg:"default-datastore"` - ResourcePoolPath string `gcfg:"resourcepool-path"` -} - -type DiskOpts struct { - SCSIControllerType string `dcfg:"scsicontrollertype"` -} - -type GlobalOpts struct { - User string `gcfg:"user"` - Password string `gcfg:"password"` - InsecureFlag bool `gcfg:"insecure-flag"` - VCenterPort string `gcfg:"port"` - WorkingDir string `gcfg:"working-dir"` - Datacenter string `gcfg:"datacenter"` - DefaultDatastore string `gcfg:"datastore"` - VCenterIP string `gcfg:"server"` - ClusterID string `gcfg:"cluster-id"` - IPFamily string `gcfg:"ip-family"` // NOTE: supported only in case of out-of-tree CCM -} - -type VirtualCenterConfig struct { - User string `gcfg:"user"` - Password string `gcfg:"password"` - VCenterPort string `gcfg:"port"` - Datacenters string `gcfg:"datacenters"` - IPFamily string `gcfg:"ip-family"` // NOTE: supported only in case of out-of-tree CCM -} - -// CloudConfig is used to read and store information from the cloud configuration file. -type CloudConfig struct { - Global GlobalOpts - Disk DiskOpts - Workspace WorkspaceOpts - - VirtualCenter map[string]*VirtualCenterConfig -} - -// String converts CloudConfig into its formatted string representation. -func (c *CloudConfig) String() (string, error) { - funcMap := sprig.TxtFuncMap() - funcMap["iniEscape"] = ini.Escape - - tpl, err := template.New("cloud-config").Funcs(funcMap).Parse(cloudConfigTpl) - if err != nil { - return "", fmt.Errorf("failed to parse the cloud config template: %w", err) - } - - buf := &bytes.Buffer{} - if err := tpl.Execute(buf, c); err != nil { - return "", fmt.Errorf("failed to execute cloud config template: %w", err) - } - - return buf.String(), nil -} - -// CloudConfigToString converts CloudConfig into its formatted string representation. -// Deprecated: use struct receiver function String() instead. -func CloudConfigToString(c *CloudConfig) (string, error) { - funcMap := sprig.TxtFuncMap() - funcMap["iniEscape"] = ini.Escape - - tpl, err := template.New("cloud-config").Funcs(funcMap).Parse(cloudConfigTpl) - if err != nil { - return "", fmt.Errorf("failed to parse the cloud config template: %w", err) - } - - buf := &bytes.Buffer{} - if err := tpl.Execute(buf, c); err != nil { - return "", fmt.Errorf("failed to execute cloud config template: %w", err) - } - - return buf.String(), nil -} diff --git a/pkg/cloudprovider/provider/vsphere/types/cloudconfig_test.go b/pkg/cloudprovider/provider/vsphere/types/cloudconfig_test.go deleted file mode 100644 index 0f69bf4db..000000000 --- a/pkg/cloudprovider/provider/vsphere/types/cloudconfig_test.go +++ /dev/null @@ -1,138 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "flag" - "testing" - - "gopkg.in/gcfg.v1" - - testhelper "github.com/kubermatic/machine-controller/pkg/test" -) - -var update = flag.Bool("update", false, "update testdata files") - -func TestCloudConfigToString(t *testing.T) { - tests := []struct { - name string - config *CloudConfig - }{ - { - name: "simple-config", - config: &CloudConfig{ - Global: GlobalOpts{ - User: "admin", - Password: "password", - InsecureFlag: true, - }, - Workspace: WorkspaceOpts{ - VCenterIP: "https://127.0.0.1:8443", - ResourcePoolPath: "/some-resource-pool", - DefaultDatastore: "Datastore", - Folder: "some-folder", - Datacenter: "Datacenter", - }, - Disk: DiskOpts{ - SCSIControllerType: "pvscsi", - }, - VirtualCenter: map[string]*VirtualCenterConfig{}, - }, - }, - { - name: "2-virtual-centers", - config: &CloudConfig{ - Global: GlobalOpts{ - User: "admin", - Password: "password", - InsecureFlag: true, - }, - Workspace: WorkspaceOpts{ - VCenterIP: "https://127.0.0.1:8443", - ResourcePoolPath: "/some-resource-pool", - DefaultDatastore: "Datastore", - Folder: "some-folder", - Datacenter: "Datacenter", - }, - Disk: DiskOpts{ - SCSIControllerType: "pvscsi", - }, - VirtualCenter: map[string]*VirtualCenterConfig{ - "vc1": { - User: "1-some-user", - Password: "1-some-password", - VCenterPort: "443", - Datacenters: "1-foo", - }, - "vc2": { - User: "2-some-user", - Password: "2-some-password", - VCenterPort: "443", - Datacenters: "2-foo", - }, - }, - }, - }, - { - name: "3-dual-stack", - config: &CloudConfig{ - Global: GlobalOpts{ - User: "admin", - Password: "password", - InsecureFlag: true, - IPFamily: "ipv4,ipv6", - }, - Workspace: WorkspaceOpts{ - VCenterIP: "https://127.0.0.1:8443", - ResourcePoolPath: "/some-resource-pool", - DefaultDatastore: "Datastore", - Folder: "some-folder", - Datacenter: "Datacenter", - }, - Disk: DiskOpts{ - SCSIControllerType: "pvscsi", - }, - VirtualCenter: map[string]*VirtualCenterConfig{ - "vc1": { - User: "1-some-user", - Password: "1-some-password", - VCenterPort: "443", - Datacenters: "1-foo", - IPFamily: "ipv4,ipv6", - }, - }, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - s, err := test.config.String() - if err != nil { - t.Fatal(err) - } - t.Logf("\n%s", s) - - nc := &CloudConfig{} - if err := gcfg.ReadStringInto(nc, s); err != nil { - t.Fatalf("failed to load string into config object: %v", err) - } - goldenName := test.name + ".golden" - testhelper.CompareOutput(t, goldenName, s, *update) - }) - } -} diff --git a/pkg/cloudprovider/provider/vsphere/types/testdata/2-virtual-centers.golden b/pkg/cloudprovider/provider/vsphere/types/testdata/2-virtual-centers.golden deleted file mode 100644 index ccf5cb7d0..000000000 --- a/pkg/cloudprovider/provider/vsphere/types/testdata/2-virtual-centers.golden +++ /dev/null @@ -1,33 +0,0 @@ -[Global] -user = "admin" -password = "password" -port = "" -insecure-flag = true -working-dir = "" -datacenter = "" -datastore = "" -server = "" - -[Disk] -scsicontrollertype = "pvscsi" - -[Workspace] -server = "https://127.0.0.1:8443" -datacenter = "Datacenter" -folder = "some-folder" -default-datastore = "Datastore" -resourcepool-path = "/some-resource-pool" - - -[VirtualCenter "vc1"] -user = "1-some-user" -password = "1-some-password" -port = 443 -datacenters = "1-foo" - -[VirtualCenter "vc2"] -user = "2-some-user" -password = "2-some-password" -port = 443 -datacenters = "2-foo" - diff --git a/pkg/cloudprovider/provider/vsphere/types/testdata/3-dual-stack.golden b/pkg/cloudprovider/provider/vsphere/types/testdata/3-dual-stack.golden deleted file mode 100644 index 88343530b..000000000 --- a/pkg/cloudprovider/provider/vsphere/types/testdata/3-dual-stack.golden +++ /dev/null @@ -1,29 +0,0 @@ -[Global] -user = "admin" -password = "password" -port = "" -insecure-flag = true -working-dir = "" -datacenter = "" -datastore = "" -server = "" -ip-family = "ipv4,ipv6" - -[Disk] -scsicontrollertype = "pvscsi" - -[Workspace] -server = "https://127.0.0.1:8443" -datacenter = "Datacenter" -folder = "some-folder" -default-datastore = "Datastore" -resourcepool-path = "/some-resource-pool" - - -[VirtualCenter "vc1"] -user = "1-some-user" -password = "1-some-password" -port = 443 -datacenters = "1-foo" -ip-family = "ipv4,ipv6" - diff --git a/pkg/cloudprovider/provider/vsphere/types/testdata/simple-config.golden b/pkg/cloudprovider/provider/vsphere/types/testdata/simple-config.golden deleted file mode 100644 index 084d188ca..000000000 --- a/pkg/cloudprovider/provider/vsphere/types/testdata/simple-config.golden +++ /dev/null @@ -1,21 +0,0 @@ -[Global] -user = "admin" -password = "password" -port = "" -insecure-flag = true -working-dir = "" -datacenter = "" -datastore = "" -server = "" - -[Disk] -scsicontrollertype = "pvscsi" - -[Workspace] -server = "https://127.0.0.1:8443" -datacenter = "Datacenter" -folder = "some-folder" -default-datastore = "Datastore" -resourcepool-path = "/some-resource-pool" - - diff --git a/pkg/cloudprovider/provider/vsphere/types/types.go b/pkg/cloudprovider/provider/vsphere/types/types.go deleted file mode 100644 index b0112d03c..000000000 --- a/pkg/cloudprovider/provider/vsphere/types/types.go +++ /dev/null @@ -1,66 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" -) - -// RawConfig represents vsphere specific configuration. -type RawConfig struct { - TemplateVMName providerconfigtypes.ConfigVarString `json:"templateVMName"` - // Deprecated: use networks instead. - VMNetName providerconfigtypes.ConfigVarString `json:"vmNetName"` - Networks []providerconfigtypes.ConfigVarString `json:"networks"` - Username providerconfigtypes.ConfigVarString `json:"username"` - Password providerconfigtypes.ConfigVarString `json:"password"` - VSphereURL providerconfigtypes.ConfigVarString `json:"vsphereURL"` - Datacenter providerconfigtypes.ConfigVarString `json:"datacenter"` - - // Cluster defines the cluster to use in vcenter. - // Only needed for vm anti affinity. - Cluster providerconfigtypes.ConfigVarString `json:"cluster"` - - Folder providerconfigtypes.ConfigVarString `json:"folder"` - ResourcePool providerconfigtypes.ConfigVarString `json:"resourcePool"` - - // Either Datastore or DatastoreCluster have to be provided. - DatastoreCluster providerconfigtypes.ConfigVarString `json:"datastoreCluster"` - Datastore providerconfigtypes.ConfigVarString `json:"datastore"` - - CPUs int32 `json:"cpus"` - MemoryMB int64 `json:"memoryMB"` - DiskSizeGB *int64 `json:"diskSizeGB,omitempty"` - Tags []Tag `json:"tags,omitempty"` - AllowInsecure providerconfigtypes.ConfigVarBool `json:"allowInsecure"` - VMAntiAffinity providerconfigtypes.ConfigVarBool `json:"vmAntiAffinity"` -} - -// Tag represents vsphere tag. -type Tag struct { - Description string `json:"description,omitempty"` - ID string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - CategoryID string `json:"categoryID"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/vultr/provider.go b/pkg/cloudprovider/provider/vultr/provider.go deleted file mode 100644 index d7d1fd912..000000000 --- a/pkg/cloudprovider/provider/vultr/provider.go +++ /dev/null @@ -1,656 +0,0 @@ -/* -Copyright 2023 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vultr - -import ( - "context" - "encoding/base64" - "errors" - "fmt" - "net/http" - "strconv" - "time" - - "github.com/vultr/govultr/v3" - "go.uber.org/zap" - "golang.org/x/oauth2" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - vultrtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vultr/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/wait" -) - -const ( - createCheckPeriod = 10 * time.Second - createCheckTimeout = 5 * time.Minute - createCheckFailedWaitPeriod = 10 * time.Second -) - -type ValidVPC struct { - IsAllValid bool - InvalidVpcs []string -} - -type provider struct { - configVarResolver *providerconfig.ConfigVarResolver -} - -// New returns a new vultr provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return &provider{configVarResolver: configVarResolver} -} - -type Config struct { - PhysicalMachine bool - APIKey string - Region string - Plan string - OsID string - Tags []string - VpcID []string - EnableVPC bool - EnableIPv6 bool - EnableVPC2 bool - Vpc2ID []string -} - -func getIDForOS(os providerconfigtypes.OperatingSystem) (int, error) { - switch os { - case providerconfigtypes.OperatingSystemUbuntu: - return 1743, nil - // name: CentOS 7 x64 - case providerconfigtypes.OperatingSystemCentOS: - return 167, nil - // name: Rocky Linux 9 x64 - case providerconfigtypes.OperatingSystemRockyLinux: - return 1869, nil - } - return 0, providerconfigtypes.ErrOSNotSupported -} - -func getClient(ctx context.Context, apiKey string) *govultr.Client { - config := &oauth2.Config{} - ts := config.TokenSource(ctx, &oauth2.Token{AccessToken: apiKey}) - return govultr.NewClient(oauth2.NewClient(ctx, ts)) -} - -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - - pconfig, err := providerconfigtypes.GetConfig(provSpec) - if err != nil { - return nil, nil, err - } - - if pconfig.OperatingSystemSpec.Raw == nil { - return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") - } - - rawConfig, err := vultrtypes.GetConfig(*pconfig) - if err != nil { - return nil, nil, err - } - - c := Config{} - - c.APIKey, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.APIKey, "VULTR_API_KEY") - if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"apiKey\" field, error = %w", err) - } - - c.Plan, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Plan) - if err != nil { - return nil, nil, err - } - - c.Region, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Region) - if err != nil { - return nil, nil, err - } - - c.OsID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.OsID) - if err != nil { - return nil, nil, err - } - - c.Tags = rawConfig.Tags - c.PhysicalMachine = rawConfig.PhysicalMachine - c.EnableIPv6 = rawConfig.EnableIPv6 - c.VpcID = rawConfig.VpcID - c.EnableVPC = rawConfig.EnableVPC - c.EnableVPC2 = rawConfig.EnableVPC2 - c.Vpc2ID = rawConfig.Vpc2ID - - return &c, pconfig, err -} - -func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { - return spec, nil -} - -func (p *provider) validateVpc(ctx context.Context, client *govultr.Client, c *Config, legacyVPC bool) (ValidVPC, error) { - validVpc := ValidVPC{IsAllValid: true} - accountvpcs := []string{} - var requestedvpcs []string - - if legacyVPC { - for { - vpcs, meta, err := func(ctx context.Context, client *govultr.Client) ([]govultr.VPC, *govultr.Meta, error) { - vpcs, meta, resp, err := client.VPC.List(ctx, &govultr.ListOptions{}) - if err != nil { - return nil, nil, vltErrorToTerminalError(resp.StatusCode, err) - } - defer resp.Body.Close() - - return vpcs, meta, nil - }(ctx, client) - if err != nil { - return validVpc, err - } - for _, v := range vpcs { - accountvpcs = append(accountvpcs, v.ID) - } - if meta.Links.Next == "" { - break - } - } - requestedvpcs = c.VpcID - } else { - for { - vpcs, meta, err := func(ctx context.Context, client *govultr.Client) ([]govultr.VPC2, *govultr.Meta, error) { - vpcs, meta, resp, err := client.VPC2.List(ctx, &govultr.ListOptions{}) - if err != nil { - return nil, nil, vltErrorToTerminalError(resp.StatusCode, err) - } - defer resp.Body.Close() - - return vpcs, meta, nil - }(ctx, client) - if err != nil { - return validVpc, err - } - for _, v := range vpcs { - accountvpcs = append(accountvpcs, v.ID) - } - if meta.Links.Next == "" { - break - } - } - requestedvpcs = c.Vpc2ID - } - accountvpcsset := sets.New[string](accountvpcs...) - // Iterator to provide user the exact mismatches - for _, v := range requestedvpcs { - if !accountvpcsset.Has(v) { - validVpc.IsAllValid = false - validVpc.InvalidVpcs = append(validVpc.InvalidVpcs, v) - } - } - - return validVpc, nil -} - -func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { - c, pc, err := p.getConfig(spec.ProviderSpec) - if err != nil { - return fmt.Errorf("failed to parse config: %w", err) - } - - if c.APIKey == "" { - return errors.New("apiKey is missing") - } - - if c.Region == "" { - return errors.New("region is missing") - } - - if c.Plan == "" { - return errors.New("plan is missing") - } - - if c.OsID == "" { - return errors.New("osID is missing") - } - - _, err = getIDForOS(pc.OperatingSystem) - if err != nil { - return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, err) - } - - client := getClient(ctx, c.APIKey) - - plans, resp, err := client.Region.Availability(ctx, c.Region, "") - - // TODO: Validate region separately - if err != nil { - return err - } - resp.Body.Close() - - planFound := false - - // Check if given plan present in the returned list - for _, plan := range plans.AvailablePlans { - if plan == c.Plan { - planFound = true - break - } - } - if !planFound { - return fmt.Errorf("invalid/not supported plan specified %q, available plans are: %q, %w", c.Plan, plans.AvailablePlans, err) - } - - validvpc, err := p.validateVpc(ctx, client, c, false) - if err != nil { - return err - } - if !validvpc.IsAllValid { - return fmt.Errorf("invalid/not supported vpc id specified %v", validvpc.InvalidVpcs) - } - - if c.PhysicalMachine { - // Don't check for validity of legacy VPC as BareMetal doesn't support VPC v1 - return nil - } - - // Verify legacy VPCs - validvpc, err = p.validateVpc(ctx, client, c, true) - if err != nil { - return err - } - - if !validvpc.IsAllValid { - return fmt.Errorf("invalid/not supported vpc id specified %v", validvpc.InvalidVpcs) - } - - return nil -} - -func (p *provider) getPhysicalMachine(ctx context.Context, c *Config, machine *clusterv1alpha1.Machine) (*vultrPhysicalMachine, error) { - client := getClient(ctx, c.APIKey) - // Not looping on metadata assuming that tagged machines won;t cross - // pagination boundary - instances, _, resp, err := client.BareMetalServer.List(ctx, &govultr.ListOptions{ - Tag: string(machine.UID), - }) - if err != nil { - return nil, vltErrorToTerminalError(resp.StatusCode, err) - } - resp.Body.Close() - for _, instance := range instances { - if sets.NewString(instance.Tags...).Has(string(machine.UID)) { - return &vultrPhysicalMachine{instance: &instance}, nil - } - } - return nil, cloudprovidererrors.ErrInstanceNotFound -} - -func (p *provider) getVirtualMachine(ctx context.Context, c *Config, machine *clusterv1alpha1.Machine) (*vultrVirtualMachine, error) { - client := getClient(ctx, c.APIKey) - - instances, _, resp, err := client.Instance.List(ctx, &govultr.ListOptions{ - Tag: string(machine.UID), - }) - if err != nil { - return nil, vltErrorToTerminalError(resp.StatusCode, err) - } - resp.Body.Close() - - for _, instance := range instances { - if sets.NewString(instance.Tags...).Has(string(machine.UID)) && - instance.Label == machine.Name { - return &vultrVirtualMachine{instance: &instance}, nil - } - } - - return nil, cloudprovidererrors.ErrInstanceNotFound -} - -func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - if !c.PhysicalMachine { - return p.getVirtualMachine(ctx, c, machine) - } - - return p.getPhysicalMachine(ctx, c, machine) -} - -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return "", "", nil -} - -func (p *provider) waitForInstanceCreation(ctx context.Context, c *Config, instance instance.Instance, machine *clusterv1alpha1.Machine) error { - return wait.PollUntilContextTimeout(ctx, createCheckPeriod, createCheckTimeout, false, func(ctx context.Context) (bool, error) { - var err error - if !c.PhysicalMachine { - _, err = p.getVirtualMachine(ctx, c, machine) - } else { - _, err = p.getPhysicalMachine(ctx, c, machine) - } - - if err != nil { - if cloudprovidererrors.IsNotFound(err) { - // Continue the loop as the instances was successfully fetched - // just that our instance was not found - return false, nil - } - if isTerminalErr, _, _ := cloudprovidererrors.IsTerminalError(err); isTerminalErr { - return true, err - } - // Wait for some time as instance creation is successful - // just that we are not able to fetch it - time.Sleep(createCheckFailedWaitPeriod) - return false, fmt.Errorf("instance %q created but controller failed to fetch instance details", instance.Name()) - } - return true, nil - }) -} - -func (p *provider) createVirtualMachine(ctx context.Context, client *govultr.Client, c *Config, machine *clusterv1alpha1.Machine, osid int, userdata string) (*vultrVirtualMachine, error) { - tags := sets.List[string](sets.New(c.Tags...).Insert(string(machine.UID))) - - instanceCreateRequest := govultr.InstanceCreateReq{ - Region: c.Region, - Plan: c.Plan, - OsID: osid, - - Label: machine.Spec.Name, - UserData: base64.StdEncoding.EncodeToString([]byte(userdata)), - Tags: tags, - - EnableIPv6: &c.EnableIPv6, - EnableVPC: &c.EnableVPC, - AttachVPC: c.VpcID, - EnableVPC2: &c.EnableVPC2, - AttachVPC2: c.Vpc2ID, - } - instance, resp, err := client.Instance.Create(ctx, &instanceCreateRequest) - if err != nil { - return nil, vltErrorToTerminalError(resp.StatusCode, err) - } - resp.Body.Close() - - return &vultrVirtualMachine{instance: instance}, nil -} - -func (p *provider) createPhysicalMachine(ctx context.Context, client *govultr.Client, c *Config, machine *clusterv1alpha1.Machine, osid int, userdata string) (*vultrPhysicalMachine, error) { - tags := sets.NewString(c.Tags...).Insert(string(machine.UID)).List() - - bareMetalCreateRequest := govultr.BareMetalCreate{ - Region: c.Region, - Plan: c.Plan, - Label: machine.Spec.Name, - UserData: base64.StdEncoding.EncodeToString([]byte(userdata)), - EnableIPv6: &c.EnableIPv6, - Tags: tags, - OsID: osid, - AttachVPC2: c.Vpc2ID, - EnableVPC2: &c.EnableVPC2, - } - instance, resp, err := client.BareMetalServer.Create(ctx, &bareMetalCreateRequest) - if err != nil { - return nil, vltErrorToTerminalError(resp.StatusCode, err) - } - resp.Body.Close() - return &vultrPhysicalMachine{instance: instance}, nil -} - -func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { - c, pc, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - - if c.OsID == "" { - osID, err := getIDForOS(pc.OperatingSystem) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Invalid operating system specified %q, details = %v", pc.OperatingSystem, err), - } - } - c.OsID = strconv.Itoa(osID) - } - strOsID, err := strconv.Atoi(c.OsID) - if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Cannot parse operating system id %q, details = %v", pc.OperatingSystem, err), - } - } - client := getClient(ctx, c.APIKey) - - var instance instance.Instance - if !c.PhysicalMachine { - instance, err = p.createVirtualMachine(ctx, client, c, machine, strOsID, userdata) - if err != nil { - return nil, err - } - } else { - instance, err = p.createPhysicalMachine(ctx, client, c, machine, strOsID, userdata) - if err != nil { - return nil, err - } - } - - err = p.waitForInstanceCreation(ctx, c, instance, machine) - if err != nil { - if !c.PhysicalMachine { - if err := client.Instance.Delete(ctx, instance.ID()); err != nil { - log.Error("Failed to cleanup instance after failed creation: %v", err) - } - } else { - if err := client.BareMetalServer.Delete(ctx, instance.ID()); err != nil { - log.Error("Failed to cleanup bare metal instance after failed creation: %v", err) - } - } - return nil, err - } - return instance, nil -} - -func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { - instance, err := p.Get(ctx, log, machine, data) - if err != nil { - if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { - return true, nil - } - return false, err - } - - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return false, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), - } - } - client := getClient(ctx, c.APIKey) - - if !c.PhysicalMachine { - if err := client.Instance.Delete(ctx, instance.ID()); err != nil { - return false, fmt.Errorf("failed to delete instance: %w", err) - } - } else { - if err := client.BareMetalServer.Delete(ctx, instance.ID()); err != nil { - return false, fmt.Errorf("failed to delete bare metal instance: %w", err) - } - } - - return false, nil -} - -func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { - labels := make(map[string]string) - - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err == nil { - labels["plan"] = c.Plan - labels["region"] = c.Region - } - - return labels, err -} - -func (p *provider) MigrateUID(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { - c, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return fmt.Errorf("failed to decode providerconfig: %w", err) - } - client := getClient(ctx, c.APIKey) - - if !c.PhysicalMachine { - instance, err := p.getVirtualMachine(ctx, c, machine) - if err != nil { - return err - } - _, resp, err := client.Instance.Update(ctx, instance.instance.ID, &govultr.InstanceUpdateReq{ - Tags: sets.NewString(instance.instance.Tags...).Delete(string(machine.UID)).Insert(string(newUID)).List(), - }) - if err != nil { - return vltErrorToTerminalError(resp.StatusCode, err) - } - resp.Body.Close() - return nil - } - instance, err := p.getPhysicalMachine(ctx, c, machine) - if err != nil { - return fmt.Errorf("failed to get instance with UID tag: %w", err) - } - _, resp, err := client.BareMetalServer.Update(ctx, instance.instance.ID, &govultr.BareMetalUpdate{ - Tags: sets.NewString(instance.instance.Tags...).Delete(string(machine.UID)).Insert(string(newUID)).List(), - }) - if err != nil { - return vltErrorToTerminalError(resp.StatusCode, err) - } - resp.Body.Close() - return nil -} - -type vultrVirtualMachine struct { - instance *govultr.Instance -} -type vultrPhysicalMachine struct { - instance *govultr.BareMetalServer -} - -func (v *vultrVirtualMachine) Name() string { - return v.instance.Label -} -func (v *vultrPhysicalMachine) Name() string { - return v.instance.Label -} - -func (v *vultrVirtualMachine) ID() string { - return v.instance.ID -} -func (v *vultrPhysicalMachine) ID() string { - return v.instance.ID -} - -func (v *vultrVirtualMachine) ProviderID() string { - if v.instance == nil || v.instance.ID == "" { - return "" - } - return "vultr://" + v.instance.ID -} -func (v *vultrPhysicalMachine) ProviderID() string { - if v.instance == nil || v.instance.ID == "" { - return "" - } - return "vultr://" + v.instance.ID -} - -func (v *vultrVirtualMachine) HostID() string { - return v.ID() -} - -func (v *vultrPhysicalMachine) HostID() string { - return v.ID() -} - -func (v *vultrVirtualMachine) Addresses() map[string]v1.NodeAddressType { - addresses := map[string]v1.NodeAddressType{} - addresses[v.instance.MainIP] = v1.NodeExternalIP - addresses[v.instance.InternalIP] = v1.NodeInternalIP - return addresses -} -func (v *vultrPhysicalMachine) Addresses() map[string]v1.NodeAddressType { - addresses := map[string]v1.NodeAddressType{} - addresses[v.instance.MainIP] = v1.NodeExternalIP - return addresses -} - -func (v *vultrVirtualMachine) Status() instance.Status { - switch v.instance.Status { - case "active": - return instance.StatusRunning - case "pending": - return instance.StatusCreating - // "suspending" or "resizing" - default: - return instance.StatusUnknown - } -} -func (v *vultrPhysicalMachine) Status() instance.Status { - switch v.instance.Status { - case "active": - return instance.StatusRunning - case "pending": - return instance.StatusCreating - // "suspending" or "resizing" - default: - return instance.StatusUnknown - } -} - -func vltErrorToTerminalError(status int, err error) error { - switch status { - case http.StatusUnauthorized: - return cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: "A request has been rejected due to invalid credentials which were taken from the MachineSpec", - } - default: - return err - } -} - -func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { - return nil -} diff --git a/pkg/cloudprovider/provider/vultr/types/types.go b/pkg/cloudprovider/provider/vultr/types/types.go deleted file mode 100644 index 278ea6066..000000000 --- a/pkg/cloudprovider/provider/vultr/types/types.go +++ /dev/null @@ -1,42 +0,0 @@ -/* -Copyright 2023 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" -) - -type RawConfig struct { - PhysicalMachine bool `json:"physicalMachine,omitempty"` - APIKey providerconfigtypes.ConfigVarString `json:"apiKey,omitempty"` - Region providerconfigtypes.ConfigVarString `json:"region"` - Plan providerconfigtypes.ConfigVarString `json:"plan"` - OsID providerconfigtypes.ConfigVarString `json:"osId"` - Tags []string `json:"tags,omitempty"` - VpcID []string `json:"vpcId,omitempty"` - Vpc2ID []string `json:"vpc2Id,omitempty"` - EnableVPC bool `json:"enableVPC,omitempty"` - EnableVPC2 bool `json:"enableVPC2,omitempty"` - EnableIPv6 bool `json:"enableIPv6,omitempty"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/controller/machine/controller.go b/pkg/controller/machine/controller.go index f59a088d4..7446feb55 100644 --- a/pkg/controller/machine/controller.go +++ b/pkg/controller/machine/controller.go @@ -45,7 +45,6 @@ import ( controllerutil "github.com/kubermatic/machine-controller/pkg/controller/util" kuberneteshelper "github.com/kubermatic/machine-controller/pkg/kubernetes" "github.com/kubermatic/machine-controller/pkg/node/eviction" - "github.com/kubermatic/machine-controller/pkg/node/poddeletion" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" "github.com/kubermatic/machine-controller/pkg/rhsm" @@ -67,7 +66,6 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/tools/reference" "k8s.io/client-go/util/retry" - ccmapi "k8s.io/cloud-provider/api" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" @@ -483,7 +481,7 @@ func (r *Reconciler) reconcile(ctx context.Context, log *zap.SugaredLogger, mach } } else { if r.nodeSettings.ExternalCloudProvider { - return r.handleNodeFailuresWithExternalCCM(ctx, log, prov, providerConfig, node, machine) + return r.handleNodeFailuresWithExternalCCM(ctx, log, prov, machine) } return r.ensureInstanceExistsForMachine(ctx, log, prov, machine, userdataPlugin, providerConfig) } @@ -542,24 +540,6 @@ func (r *Reconciler) machineHasValidNode(ctx context.Context, machine *clusterv1 return true, nil } -func (r *Reconciler) shouldCleanupVolumes(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, providerName providerconfigtypes.CloudProvider) (bool, error) { - // we need to wait for volumeAttachments clean up only for vSphere - if providerName != providerconfigtypes.CloudProviderVsphere { - return false, nil - } - - hasMachine, err := r.machineHasValidNode(ctx, machine) - if err != nil { - return false, err - } - - if !hasMachine { - log.Debug("Skipping eviction since it does not have a node") - } - - return hasMachine, nil -} - // evictIfNecessary checks if the machine has a node and evicts it if necessary. func (r *Reconciler) shouldEvict(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine) (bool, error) { // If the deletion got triggered a few hours ago, skip eviction. @@ -632,10 +612,6 @@ func (r *Reconciler) deleteMachine( return nil, err } } - shouldCleanUpVolumes, err := r.shouldCleanupVolumes(ctx, log, machine, providerName) - if err != nil { - return nil, err - } var evictedSomething, deletedSomething bool var volumesFree = true @@ -645,13 +621,6 @@ func (r *Reconciler) deleteMachine( return nil, fmt.Errorf("failed to evict node %s: %w", machine.Status.NodeRef.Name, err) } } - if shouldCleanUpVolumes { - deletedSomething, volumesFree, err = poddeletion.New(machine.Status.NodeRef.Name, r.client, r.kubeClient).Run(ctx, log) - if err != nil { - return nil, fmt.Errorf("failed to delete pods bound to volumes running on node %s: %w", machine.Status.NodeRef.Name, err) - } - } - if evictedSomething || deletedSomething || !volumesFree { return &reconcile.Result{RequeueAfter: 10 * time.Second}, nil } @@ -864,10 +833,6 @@ func (r *Reconciler) ensureInstanceExistsForMachine( return nil, fmt.Errorf("failed to render cloud config: %w", err) } - if providerConfig.CloudProvider == providerconfigtypes.CloudProviderVsphere && externalCloudProvider { - cloudConfig = "" - } - registryCredentials, err := containerruntime.GetContainerdAuthConfig(ctx, r.client, r.nodeSettings.RegistryCredentialsSecretRef) if err != nil { return nil, fmt.Errorf("failed to get containerd auth config: %w", err) @@ -1210,16 +1175,6 @@ func (r *Reconciler) getNode(ctx context.Context, log *zap.SugaredLogger, instan // then cause cluster stability issues in some cases. for _, nodeAddress := range node.Status.Addresses { for instanceAddress := range instance.Addresses() { - // We observed that the issue described above happens often on Hetzner. - // As we know that the Node and the instance name will always be same - // on Hetzner, we can use it as an additional check to prevent this - // issue. - // TODO: We should do this for other providers, but there are providers where - // the node and the instance names will not match, so it requires further - // investigation (e.g. AWS). - if provider == providerconfigtypes.CloudProviderHetzner && node.Name != instance.Name() { - continue - } if nodeAddress.Address == instanceAddress { log.Debugw("Found node by IP address", "node", node.Name) return node.DeepCopy(), true, nil @@ -1341,15 +1296,8 @@ func (r *Reconciler) handleNodeFailuresWithExternalCCM( ctx context.Context, log *zap.SugaredLogger, prov cloudprovidertypes.Provider, - provConfig *providerconfigtypes.Config, - node *corev1.Node, machine *clusterv1alpha1.Machine, ) (*reconcile.Result, error) { - taintShutdown := corev1.Taint{ - Key: ccmapi.TaintNodeShutdown, - Effect: corev1.TaintEffectNoSchedule, - } - _, err := prov.Get(ctx, log, machine, r.providerData) if err != nil { if cloudprovidererrors.IsNotFound(err) { @@ -1357,13 +1305,6 @@ func (r *Reconciler) handleNodeFailuresWithExternalCCM( return &reconcile.Result{RequeueAfter: deletionRetryWaitPeriod}, nil } return nil, err - } else if taintExists(node, taintShutdown) { - switch provConfig.CloudProvider { - case providerconfigtypes.CloudProviderKubeVirt: - log.Infof("Deleting a shut-down machine %q that cannot recover", machine.Name) - skipEviction := true - return r.deleteMachine(ctx, log, prov, providerconfigtypes.CloudProviderKubeVirt, machine, skipEviction) - } } log.Debug("Waiting for a node to become %q", corev1.NodeReady) diff --git a/pkg/controller/machine/controller_test.go b/pkg/controller/machine/controller_test.go index 09b7dffdd..f8db25b8a 100644 --- a/pkg/controller/machine/controller_test.go +++ b/pkg/controller/machine/controller_test.go @@ -157,38 +157,6 @@ func TestController_GetNode(t *testing.T) { err: nil, instance: &fakeInstance{id: "3", addresses: map[string]corev1.NodeAddressType{"172.16.1.3": corev1.NodeInternalIP}}, }, - { - name: "hetzner node found by internal ip", - provider: "hetzner", - resNode: &node3, - exists: true, - err: nil, - instance: &fakeInstance{id: "3", name: "node3", addresses: map[string]corev1.NodeAddressType{"192.168.1.3": corev1.NodeInternalIP}}, - }, - { - name: "hetzner node found by external ip", - provider: "hetzner", - resNode: &node3, - exists: true, - err: nil, - instance: &fakeInstance{id: "3", name: "node3", addresses: map[string]corev1.NodeAddressType{"172.16.1.3": corev1.NodeExternalIP}}, - }, - { - name: "hetzner node not found - node and instance names mismatch", - provider: "hetzner", - resNode: nil, - exists: false, - err: nil, - instance: &fakeInstance{id: "3", name: "instance3", addresses: map[string]corev1.NodeAddressType{"192.168.1.3": corev1.NodeInternalIP}}, - }, - { - name: "hetzner node found by provider id", - provider: "hetzner", - resNode: &node4, - exists: true, - err: nil, - instance: &fakeInstance{id: "4", addresses: map[string]corev1.NodeAddressType{"": ""}, providerID: "hcloud://123"}, - }, } for _, test := range tests { @@ -691,16 +659,6 @@ func TestControllerFindNodeByProviderID(t *testing.T) { }, expectedNode: true, }, - { - name: "azure providerID", - instance: &fakeInstance{id: "99", providerID: "azure:///test/test"}, - provider: providerconfigtypes.CloudProviderAWS, - nodes: []corev1.Node{ - getTestNode("1", "random"), - getTestNode("2", "azure:///test/test"), - }, - expectedNode: true, - }, } for _, test := range tests { diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 588dc6e91..589db528a 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -48,28 +48,11 @@ const ( type CloudProvider string const ( - CloudProviderAWS CloudProvider = "aws" - CloudProviderAzure CloudProvider = "azure" - CloudProviderDigitalocean CloudProvider = "digitalocean" - CloudProviderGoogle CloudProvider = "gce" - CloudProviderEquinixMetal CloudProvider = "equinixmetal" - CloudProviderPacket CloudProvider = "packet" - CloudProviderHetzner CloudProvider = "hetzner" - CloudProviderKubeVirt CloudProvider = "kubevirt" - CloudProviderLinode CloudProvider = "linode" - CloudProviderNutanix CloudProvider = "nutanix" - CloudProviderOpenstack CloudProvider = "openstack" - CloudProviderVsphere CloudProvider = "vsphere" - CloudProviderVultr CloudProvider = "vultr" - CloudProviderVMwareCloudDirector CloudProvider = "vmware-cloud-director" - CloudProviderFake CloudProvider = "fake" - CloudProviderEdge CloudProvider = "edge" - CloudProviderAlibaba CloudProvider = "alibaba" - CloudProviderAnexia CloudProvider = "anexia" - CloudProviderScaleway CloudProvider = "scaleway" - CloudProviderBaremetal CloudProvider = "baremetal" - CloudProviderExternal CloudProvider = "external" - CloudProviderOpenNebula CloudProvider = "opennebula" + CloudProviderAWS CloudProvider = "aws" + CloudProviderOpenstack CloudProvider = "openstack" + CloudProviderFake CloudProvider = "fake" + CloudProviderBaremetal CloudProvider = "baremetal" + CloudProviderExternal CloudProvider = "external" ) var ( @@ -88,26 +71,9 @@ var ( // AllCloudProviders is a slice containing all supported cloud providers. AllCloudProviders = []CloudProvider{ CloudProviderAWS, - CloudProviderAzure, - CloudProviderDigitalocean, - CloudProviderEquinixMetal, - CloudProviderPacket, - CloudProviderGoogle, - CloudProviderHetzner, - CloudProviderKubeVirt, - CloudProviderLinode, - CloudProviderNutanix, CloudProviderOpenstack, - CloudProviderVsphere, - CloudProviderVMwareCloudDirector, CloudProviderFake, - CloudProviderEdge, - CloudProviderAlibaba, - CloudProviderAnexia, - CloudProviderScaleway, CloudProviderBaremetal, - CloudProviderVultr, - CloudProviderOpenNebula, } ) @@ -118,8 +84,6 @@ func IntreeCloudProviderImplementationSupported(cloudProvider CloudProvider, ver } switch cloudProvider { - case CloudProviderAzure, CloudProviderVsphere, CloudProviderGoogle: - return true, nil case CloudProviderAWS: // In-tree AWS support was removed in Kubernetes 1.27. ltKube127Condition, _ := semver.NewConstraint("< 1.27") diff --git a/pkg/userdata/amzn2/provider.go b/pkg/userdata/amzn2/provider.go index e2b838328..80482c85e 100644 --- a/pkg/userdata/amzn2/provider.go +++ b/pkg/userdata/amzn2/provider.go @@ -216,9 +216,6 @@ write_files: socat \ wget \ curl \ - {{- if or (eq .CloudProviderName "vsphere") (eq .CloudProviderName "vmware-cloud-director") }} - open-vm-tools \ - {{- end }} ipvsadm {{ .ContainerRuntimeScript | indent 4 }} @@ -229,9 +226,6 @@ write_files: /opt/bin/setup_net_env.sh systemctl disable --now firewalld || true - {{ if eq .CloudProviderName "vsphere" }} - systemctl enable --now vmtoolsd.service - {{ end -}} systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service systemctl disable setup.service diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index 6d20a7b22..e6aa02940 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -120,44 +120,6 @@ func TestUserDataGeneration(t *testing.T) { }, externalCloudProvider: true, }, - { - name: "kubelet-v1.29.2-vsphere", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", - }, - }, - cloudProviderName: stringPtr("vsphere"), - }, - { - name: "kubelet-v1.29.2-vsphere-proxy", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", - }, - }, - cloudProviderName: stringPtr("vsphere"), - httpProxy: "http://192.168.100.100:3128", - noProxy: "192.168.1.0", - insecureRegistries: "192.168.100.100:5000, 10.0.0.1:5000", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "kubelet-v1.29.2-vsphere-mirrors", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", - }, - }, - cloudProviderName: stringPtr("vsphere"), - httpProxy: "http://192.168.100.100:3128", - noProxy: "192.168.1.0", - registryMirrors: "https://registry.docker-cn.com", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, { name: "kubelet-v1.28-aws", spec: clusterv1alpha1.MachineSpec{ diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml deleted file mode 100644 index f0c0ea4aa..000000000 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml +++ /dev/null @@ -1,471 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false - -write_files: -- path: "/etc/environment" - content: | - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - hostnamectl set-hostname node1 - - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - open-vm-tools \ - ipvsadm - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry.docker-cn.com"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-proxy.yaml deleted file mode 100644 index b0717e211..000000000 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-proxy.yaml +++ /dev/null @@ -1,478 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false - -write_files: -- path: "/etc/environment" - content: | - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - hostnamectl set-hostname node1 - - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - open-vm-tools \ - ipvsadm - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - [plugins."io.containerd.grpc.v1.cri".registry.configs] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000".tls] - insecure_skip_verify = true - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000".tls] - insecure_skip_verify = true - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere.yaml deleted file mode 100644 index 028a63809..000000000 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere.yaml +++ /dev/null @@ -1,462 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - hostnamectl set-hostname node1 - - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - open-vm-tools \ - ipvsadm - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/provider.go b/pkg/userdata/centos/provider.go index 0f24ee2c8..8fd73026f 100644 --- a/pkg/userdata/centos/provider.go +++ b/pkg/userdata/centos/provider.go @@ -224,18 +224,8 @@ write_files: socat \ wget \ curl \ - {{- if or (eq .CloudProviderName "vsphere") (eq .CloudProviderName "vmware-cloud-director") }} - open-vm-tools \ - {{- end }} - {{- if eq .CloudProviderName "nutanix" }} - iscsi-initiator-utils \ - {{- end }} ipvsadm - {{- /* iscsid service is required on Nutanix machines for CSI driver to attach volumes. */}} - {{- if eq .CloudProviderName "nutanix" }} - systemctl enable --now iscsid - {{ end }} {{ .ContainerRuntimeScript | indent 4 }} {{ safeDownloadBinariesScript .KubeletVersion | indent 4 }} @@ -244,14 +234,8 @@ write_files: /opt/bin/setup_net_env.sh systemctl disable --now firewalld || true - {{ if eq .CloudProviderName "vsphere" }} - systemctl enable --now vmtoolsd.service - {{ end -}} systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service - {{- if eq .CloudProviderName "kubevirt" }} - systemctl enable --now --no-block restart-kubelet.service - {{ end }} systemctl disable setup.service - path: "/opt/bin/supervise.sh" @@ -343,42 +327,6 @@ write_files: append: true {{- end }} -{{- if eq .CloudProviderName "kubevirt" }} -- path: "/opt/bin/restart-kubelet.sh" - permissions: "0744" - content: | - #!/bin/bash - # Needed for Kubevirt provider because if the virt-launcher pod is deleted, - # the VM and DataVolume states are kept and VM is rebooted. We need to restart the kubelet - # with the new config (new IP) and run this at every boot. - set -xeuo pipefail - - # This helps us avoid an unnecessary restart for kubelet on the first boot - if [ -f /etc/kubelet_needs_restart ]; then - # restart kubelet since it's not the first boot - systemctl daemon-reload - systemctl restart kubelet.service - else - touch /etc/kubelet_needs_restart - fi - -- path: "/etc/systemd/system/restart-kubelet.service" - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - Description=Service responsible for restarting kubelet when the machine is rebooted - - [Service] - Type=oneshot - ExecStart=/opt/bin/restart-kubelet.sh - - [Install] - WantedBy=multi-user.target -{{- end }} - runcmd: - systemctl enable --now setup.service ` diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go index d93b87a2f..b31b99bff 100644 --- a/pkg/userdata/centos/provider_test.go +++ b/pkg/userdata/centos/provider_test.go @@ -110,16 +110,6 @@ func TestUserDataGeneration(t *testing.T) { }, }, }, - { - name: "kubelet-v1.29.2-nutanix", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", - }, - }, - cloudProviderName: stringPtr("nutanix"), - }, { name: "kubelet-v1.29.2-aws-external", spec: clusterv1alpha1.MachineSpec{ @@ -130,44 +120,6 @@ func TestUserDataGeneration(t *testing.T) { }, externalCloudProvider: true, }, - { - name: "kubelet-v1.29.2-vsphere", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", - }, - }, - cloudProviderName: stringPtr("vsphere"), - }, - { - name: "kubelet-v1.29.2-vsphere-proxy", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", - }, - }, - cloudProviderName: stringPtr("vsphere"), - httpProxy: "http://192.168.100.100:3128", - noProxy: "192.168.1.0", - insecureRegistries: "192.168.100.100:5000, 10.0.0.1:5000", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "kubelet-v1.29.2-vsphere-mirrors", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", - }, - }, - cloudProviderName: stringPtr("vsphere"), - httpProxy: "http://192.168.100.100:3128", - noProxy: "192.168.1.0", - registryMirrors: "https://registry.docker-cn.com", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, { name: "kubelet-v1.28-aws", spec: clusterv1alpha1.MachineSpec{ diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml deleted file mode 100644 index 5eecedbc5..000000000 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml +++ /dev/null @@ -1,452 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml index 8a5bb7a7b..b58da3338 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml @@ -85,6 +85,7 @@ write_files: curl \ ipvsadm + yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true diff --git a/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml index d57407477..8d0535df8 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml @@ -85,6 +85,7 @@ write_files: curl \ ipvsadm + yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true diff --git a/pkg/userdata/centos/testdata/kubelet-v1.29-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29-aws.yaml index 29b7cb2bb..f89e34c3e 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.29-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.29-aws.yaml @@ -85,6 +85,7 @@ write_files: curl \ ipvsadm + yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true diff --git a/pkg/userdata/centos/testdata/kubelet-v1.29.2-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29.2-aws-external.yaml index 5ba93232e..0be73d1dd 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.29.2-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.29.2-aws-external.yaml @@ -85,6 +85,7 @@ write_files: curl \ ipvsadm + yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true diff --git a/pkg/userdata/centos/testdata/kubelet-v1.29.2-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29.2-aws.yaml index c18f74664..73531f2ab 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.29.2-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.29.2-aws.yaml @@ -85,6 +85,7 @@ write_files: curl \ ipvsadm + yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true diff --git a/pkg/userdata/centos/testdata/kubelet-v1.29.2-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29.2-nutanix.yaml deleted file mode 100644 index 963494efe..000000000 --- a/pkg/userdata/centos/testdata/kubelet-v1.29.2-nutanix.yaml +++ /dev/null @@ -1,468 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - iscsi-initiator-utils \ - ipvsadm - systemctl enable --now iscsid - - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=nutanix \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml deleted file mode 100644 index 1061299b4..000000000 --- a/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml +++ /dev/null @@ -1,477 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false - -write_files: -- path: "/etc/environment" - content: | - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry.docker-cn.com"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-proxy.yaml deleted file mode 100644 index 93fe63ce9..000000000 --- a/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-proxy.yaml +++ /dev/null @@ -1,484 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false - -write_files: -- path: "/etc/environment" - content: | - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - [plugins."io.containerd.grpc.v1.cri".registry.configs] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000".tls] - insecure_skip_verify = true - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000".tls] - insecure_skip_verify = true - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere.yaml deleted file mode 100644 index 3a8004604..000000000 --- a/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere.yaml +++ /dev/null @@ -1,468 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go index c83f4cf3e..0038c4095 100644 --- a/pkg/userdata/flatcar/provider.go +++ b/pkg/userdata/flatcar/provider.go @@ -504,12 +504,6 @@ storage: {{- end }} systemctl disable setup.service - # Creates iscsi InitiatorName on Nutanix machines for CSI driver to attach volumes. - {{- if eq .CloudProviderName "nutanix" }} - systemctl start iscsi-init.service - systemctl enable --now iscsid.service - {{- end }} - - path: /opt/bin/download.sh filesystem: root mode: 0755 diff --git a/pkg/userdata/flatcar/provider_test.go b/pkg/userdata/flatcar/provider_test.go index 91d64a2d7..188860641 100644 --- a/pkg/userdata/flatcar/provider_test.go +++ b/pkg/userdata/flatcar/provider_test.go @@ -128,7 +128,7 @@ func TestUserDataGeneration(t *testing.T) { { name: "ignition_v1.29.0", providerSpec: &providerconfigtypes.Config{ - CloudProvider: "vsphere", + CloudProvider: "openstack", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, CAPublicKey: pointer.String("ssh-rsa AAABBB"), Network: &providerconfigtypes.NetworkConfig{ @@ -146,8 +146,8 @@ func TestUserDataGeneration(t *testing.T) { }, }, ccProvider: &fakeCloudConfigProvider{ - name: "vsphere", - config: "{vsphere-config:true}", + name: "openstack", + config: "{openstack-config:true}", err: nil, }, DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, @@ -159,7 +159,7 @@ func TestUserDataGeneration(t *testing.T) { { name: "ignition_v1.28.5", providerSpec: &providerconfigtypes.Config{ - CloudProvider: "vsphere", + CloudProvider: "openstack", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, CAPublicKey: pointer.String("ssh-rsa AAABBB"), Network: &providerconfigtypes.NetworkConfig{ @@ -177,8 +177,8 @@ func TestUserDataGeneration(t *testing.T) { }, }, ccProvider: &fakeCloudConfigProvider{ - name: "vsphere", - config: "{vsphere-config:true}", + name: "openstack", + config: "{openstack-config:true}", err: nil, }, DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, @@ -190,7 +190,7 @@ func TestUserDataGeneration(t *testing.T) { { name: "ignition_v1.28.5", providerSpec: &providerconfigtypes.Config{ - CloudProvider: "vsphere", + CloudProvider: "openstack", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, CAPublicKey: pointer.String("ssh-rsa AAABBB"), Network: &providerconfigtypes.NetworkConfig{ @@ -208,8 +208,8 @@ func TestUserDataGeneration(t *testing.T) { }, }, ccProvider: &fakeCloudConfigProvider{ - name: "vsphere", - config: "{vsphere-config:true}", + name: "openstack", + config: "{openstack-config:true}", err: nil, }, DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, @@ -221,7 +221,7 @@ func TestUserDataGeneration(t *testing.T) { { name: "ignition_v1.29.2", providerSpec: &providerconfigtypes.Config{ - CloudProvider: "vsphere", + CloudProvider: "openstack", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, CAPublicKey: pointer.String("ssh-rsa AAABBB"), Network: &providerconfigtypes.NetworkConfig{ @@ -239,8 +239,8 @@ func TestUserDataGeneration(t *testing.T) { }, }, ccProvider: &fakeCloudConfigProvider{ - name: "vsphere", - config: "{vsphere-config:true}", + name: "openstack", + config: "{openstack-config:true}", err: nil, }, DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, @@ -252,7 +252,7 @@ func TestUserDataGeneration(t *testing.T) { { name: "ignition_v1.29.2", providerSpec: &providerconfigtypes.Config{ - CloudProvider: "vsphere", + CloudProvider: "openstack", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, CAPublicKey: pointer.String("ssh-rsa AAABBB"), Network: &providerconfigtypes.NetworkConfig{ @@ -270,8 +270,8 @@ func TestUserDataGeneration(t *testing.T) { }, }, ccProvider: &fakeCloudConfigProvider{ - name: "vsphere", - config: "{vsphere-config:true}", + name: "openstack", + config: "{openstack-config:true}", err: nil, }, DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, @@ -283,7 +283,7 @@ func TestUserDataGeneration(t *testing.T) { { name: "cloud-init_v1.29.0", providerSpec: &providerconfigtypes.Config{ - CloudProvider: "anexia", + CloudProvider: "aws", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, CAPublicKey: pointer.String("ssh-rsa AAABBB"), Network: &providerconfigtypes.NetworkConfig{ @@ -301,8 +301,8 @@ func TestUserDataGeneration(t *testing.T) { }, }, ccProvider: &fakeCloudConfigProvider{ - name: "anexia", - config: "{anexia-config:true}", + name: "aws", + config: "{aws-config:true}", err: nil, }, DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, @@ -314,7 +314,7 @@ func TestUserDataGeneration(t *testing.T) { { name: "cloud-init_v1.28.5", providerSpec: &providerconfigtypes.Config{ - CloudProvider: "anexia", + CloudProvider: "aws", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, CAPublicKey: pointer.String("ssh-rsa AAABBB"), Network: &providerconfigtypes.NetworkConfig{ @@ -332,8 +332,8 @@ func TestUserDataGeneration(t *testing.T) { }, }, ccProvider: &fakeCloudConfigProvider{ - name: "anexia", - config: "{anexia-config:true}", + name: "aws", + config: "{aws-config:true}", err: nil, }, DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, @@ -345,7 +345,7 @@ func TestUserDataGeneration(t *testing.T) { { name: "cloud-init_v1.28.5", providerSpec: &providerconfigtypes.Config{ - CloudProvider: "anexia", + CloudProvider: "aws", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, CAPublicKey: pointer.String("ssh-rsa AAABBB"), Network: &providerconfigtypes.NetworkConfig{ @@ -363,8 +363,8 @@ func TestUserDataGeneration(t *testing.T) { }, }, ccProvider: &fakeCloudConfigProvider{ - name: "anexia", - config: "{anexia-config:true}", + name: "aws", + config: "{aws-config:true}", err: nil, }, DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, @@ -376,7 +376,7 @@ func TestUserDataGeneration(t *testing.T) { { name: "cloud-init_v1.29.2", providerSpec: &providerconfigtypes.Config{ - CloudProvider: "anexia", + CloudProvider: "aws", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, CAPublicKey: pointer.String("ssh-rsa AAABBB"), Network: &providerconfigtypes.NetworkConfig{ @@ -394,8 +394,8 @@ func TestUserDataGeneration(t *testing.T) { }, }, ccProvider: &fakeCloudConfigProvider{ - name: "anexia", - config: "{anexia-config:true}", + name: "aws", + config: "{aws-config:true}", err: nil, }, DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, @@ -407,7 +407,7 @@ func TestUserDataGeneration(t *testing.T) { { name: "cloud-init_v1.29.2", providerSpec: &providerconfigtypes.Config{ - CloudProvider: "anexia", + CloudProvider: "aws", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, CAPublicKey: pointer.String("ssh-rsa AAABBB"), Network: &providerconfigtypes.NetworkConfig{ @@ -425,8 +425,8 @@ func TestUserDataGeneration(t *testing.T) { }, }, ccProvider: &fakeCloudConfigProvider{ - name: "anexia", - config: "{anexia-config:true}", + name: "aws", + config: "{aws-config:true}", err: nil, }, DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.28.5.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.28.5.yaml index 8a665ce20..380484df0 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.28.5.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.28.5.yaml @@ -97,10 +97,6 @@ coreos: [Unit] Requires=download-script.service After=download-script.service - - name: 50-rpc-statd.conf - content: | - [Unit] - Wants=rpc-statd.service content: | [Unit] After=containerd.service @@ -128,9 +124,8 @@ coreos: --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=anexia \ + --cloud-provider=aws \ --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ @@ -334,7 +329,7 @@ write_files: - path: /etc/kubernetes/cloud-config permissions: "0400" content: | - {anexia-config:true} + {aws-config:true} - path: /etc/kubernetes/pki/ca.crt permissions: "0644" @@ -367,9 +362,6 @@ write_files: -----END CERTIFICATE----- -- path: /etc/hostname - permissions: "0600" - content: 'node1' - path: /etc/ssh/sshd_config permissions: "0600" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.29.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.29.0.yaml index 83c546dea..97c1120cb 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.29.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.29.0.yaml @@ -97,10 +97,6 @@ coreos: [Unit] Requires=download-script.service After=download-script.service - - name: 50-rpc-statd.conf - content: | - [Unit] - Wants=rpc-statd.service content: | [Unit] After=containerd.service @@ -128,9 +124,8 @@ coreos: --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=anexia \ + --cloud-provider=aws \ --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ @@ -334,7 +329,7 @@ write_files: - path: /etc/kubernetes/cloud-config permissions: "0400" content: | - {anexia-config:true} + {aws-config:true} - path: /etc/kubernetes/pki/ca.crt permissions: "0644" @@ -367,9 +362,6 @@ write_files: -----END CERTIFICATE----- -- path: /etc/hostname - permissions: "0600" - content: 'node1' - path: /etc/ssh/sshd_config permissions: "0600" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.29.2.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.29.2.yaml index 242457c88..e0cf14bfb 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.29.2.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.29.2.yaml @@ -97,10 +97,6 @@ coreos: [Unit] Requires=download-script.service After=download-script.service - - name: 50-rpc-statd.conf - content: | - [Unit] - Wants=rpc-statd.service content: | [Unit] After=containerd.service @@ -128,9 +124,8 @@ coreos: --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=anexia \ + --cloud-provider=aws \ --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ @@ -334,7 +329,7 @@ write_files: - path: /etc/kubernetes/cloud-config permissions: "0400" content: | - {anexia-config:true} + {aws-config:true} - path: /etc/kubernetes/pki/ca.crt permissions: "0644" @@ -367,9 +362,6 @@ write_files: -----END CERTIFICATE----- -- path: /etc/hostname - permissions: "0600" - content: 'node1' - path: /etc/ssh/sshd_config permissions: "0600" diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.28.5.json b/pkg/userdata/flatcar/testdata/ignition_v1.28.5.json index 43276fd70..925a2ee46 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.28.5.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.28.5.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMaximumGCAge%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20300Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20500Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0A%0A%23%20sys11%3A%20we%20use%20the%20user-ssh-keys-agent%20to%20deploy%20ssh%20keys%0Asystemctl%20stop%20coreos-metadata-sshkeys%40core.service%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.28.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fdl.k8s.io%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20containerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/disable-download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Asystemctl%20stop%20download-script.service%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"mask":true,"name":"coreos-metadata-sshkeys@core.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nAfter=download-script.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/disable-download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"disable-download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMaximumGCAge%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20300Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20500Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bopenstack-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0A%0A%23%20sys11%3A%20we%20use%20the%20user-ssh-keys-agent%20to%20deploy%20ssh%20keys%0Asystemctl%20stop%20coreos-metadata-sshkeys%40core.service%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.28.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fdl.k8s.io%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20containerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/disable-download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Asystemctl%20stop%20download-script.service%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"mask":true,"name":"coreos-metadata-sshkeys@core.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nAfter=download-script.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/disable-download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"disable-download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=openstack \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.29.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.29.0.json index 8c586d86c..ed46629c3 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.29.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.29.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMaximumGCAge%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20300Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20500Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0A%0A%23%20sys11%3A%20we%20use%20the%20user-ssh-keys-agent%20to%20deploy%20ssh%20keys%0Asystemctl%20stop%20coreos-metadata-sshkeys%40core.service%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.29.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fdl.k8s.io%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20containerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/disable-download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Asystemctl%20stop%20download-script.service%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"mask":true,"name":"coreos-metadata-sshkeys@core.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nAfter=download-script.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/disable-download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"disable-download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMaximumGCAge%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20300Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20500Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bopenstack-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0A%0A%23%20sys11%3A%20we%20use%20the%20user-ssh-keys-agent%20to%20deploy%20ssh%20keys%0Asystemctl%20stop%20coreos-metadata-sshkeys%40core.service%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.29.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fdl.k8s.io%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20containerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/disable-download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Asystemctl%20stop%20download-script.service%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"mask":true,"name":"coreos-metadata-sshkeys@core.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nAfter=download-script.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/disable-download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"disable-download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=openstack \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.29.2.json b/pkg/userdata/flatcar/testdata/ignition_v1.29.2.json index 85e754b4c..624b45ea0 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.29.2.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.29.2.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMaximumGCAge%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20300Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20500Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0A%0A%23%20sys11%3A%20we%20use%20the%20user-ssh-keys-agent%20to%20deploy%20ssh%20keys%0Asystemctl%20stop%20coreos-metadata-sshkeys%40core.service%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.29.2%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fdl.k8s.io%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20containerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/disable-download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Asystemctl%20stop%20download-script.service%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"mask":true,"name":"coreos-metadata-sshkeys@core.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nAfter=download-script.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/disable-download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"disable-download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMaximumGCAge%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20300Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20500Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bopenstack-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0A%0A%23%20sys11%3A%20we%20use%20the%20user-ssh-keys-agent%20to%20deploy%20ssh%20keys%0Asystemctl%20stop%20coreos-metadata-sshkeys%40core.service%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.29.2%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fdl.k8s.io%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20containerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/disable-download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Asystemctl%20stop%20download-script.service%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"mask":true,"name":"coreos-metadata-sshkeys@core.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nAfter=download-script.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/disable-download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"disable-download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=openstack \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/rhel/provider_test.go b/pkg/userdata/rhel/provider_test.go index e972a47e5..2337a1899 100644 --- a/pkg/userdata/rhel/provider_test.go +++ b/pkg/userdata/rhel/provider_test.go @@ -138,74 +138,6 @@ func TestUserDataGeneration(t *testing.T) { }, externalCloudProvider: true, }, - { - name: "kubelet-v1.29.2-vsphere", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", - }, - }, - cloudProviderName: stringPtr("vsphere"), - }, - { - name: "kubelet-v1.29.2-vsphere-proxy", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", - }, - }, - cloudProviderName: stringPtr("vsphere"), - httpProxy: "http://192.168.100.100:3128", - noProxy: "192.168.1.0", - insecureRegistries: "192.168.100.100:5000, 10.0.0.1:5000", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "kubelet-v1.29.2-vsphere-mirrors", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", - }, - }, - cloudProviderName: stringPtr("vsphere"), - httpProxy: "http://192.168.100.100:3128", - noProxy: "192.168.1.0", - registryMirrors: "https://registry.docker-cn.com", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "kubelet-v1.28-nutanix", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.5", - }, - }, - cloudProviderName: stringPtr("nutanix"), - }, - { - name: "pod-cidr-azure-rhel", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.0", - }, - }, - cloudProviderName: stringPtr("azure"), - }, - { - name: "kubelet-v1.29-nutanix", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.0", - }, - }, - cloudProviderName: stringPtr("nutanix"), - }, } defaultCloudProvider := &fakeCloudConfigProvider{ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.28-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.28-nutanix.yaml deleted file mode 100644 index b9fc1d821..000000000 --- a/pkg/userdata/rhel/testdata/kubelet-v1.28-nutanix.yaml +++ /dev/null @@ -1,541 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 -fqdn: node1 - - -ssh_pwauth: false - -write_files: -- path: "/opt/restart-kubelet.sh" - permissions: "0755" - content: | - #!/bin/bash - - while true; do - output=$(journalctl -u kubelet -n 1 | grep "use of closed network connection") - if [[ $? != 0 ]]; then - echo "Error not found in logs" - elif [[ $output ]]; then - echo "Restart kubelet" - systemctl restart kubelet - fi - - # Runs every 10 minutes - sleep 600 - done - -- path: "/etc/systemd/system/kubelet-restart.service" - permissions: "0644" - content: | - [Unit] - Description=Restarts kubelet on use of closed network connection error - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/kubelet-restart.sh - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - hostnamectl set-hostname node1 - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - iscsi-initiator-utils \ - ipvsadm - systemctl enable --now iscsid - - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=nutanix \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.29-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.29-nutanix.yaml deleted file mode 100644 index 224e419cf..000000000 --- a/pkg/userdata/rhel/testdata/kubelet-v1.29-nutanix.yaml +++ /dev/null @@ -1,541 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 -fqdn: node1 - - -ssh_pwauth: false - -write_files: -- path: "/opt/restart-kubelet.sh" - permissions: "0755" - content: | - #!/bin/bash - - while true; do - output=$(journalctl -u kubelet -n 1 | grep "use of closed network connection") - if [[ $? != 0 ]]; then - echo "Error not found in logs" - elif [[ $output ]]; then - echo "Restart kubelet" - systemctl restart kubelet - fi - - # Runs every 10 minutes - sleep 600 - done - -- path: "/etc/systemd/system/kubelet-restart.service" - permissions: "0644" - content: | - [Unit] - Description=Restarts kubelet on use of closed network connection error - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/kubelet-restart.sh - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - hostnamectl set-hostname node1 - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - iscsi-initiator-utils \ - ipvsadm - systemctl enable --now iscsid - - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=nutanix \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml deleted file mode 100644 index d3f74c179..000000000 --- a/pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml +++ /dev/null @@ -1,550 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 -fqdn: node1 - - -ssh_pwauth: false - -write_files: -- path: "/opt/restart-kubelet.sh" - permissions: "0755" - content: | - #!/bin/bash - - while true; do - output=$(journalctl -u kubelet -n 1 | grep "use of closed network connection") - if [[ $? != 0 ]]; then - echo "Error not found in logs" - elif [[ $output ]]; then - echo "Restart kubelet" - systemctl restart kubelet - fi - - # Runs every 10 minutes - sleep 600 - done - -- path: "/etc/systemd/system/kubelet-restart.service" - permissions: "0644" - content: | - [Unit] - Description=Restarts kubelet on use of closed network connection error - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/kubelet-restart.sh - - [Install] - WantedBy=multi-user.target -- path: "/etc/environment" - content: | - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - hostnamectl set-hostname node1 - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry.docker-cn.com"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere-proxy.yaml deleted file mode 100644 index 9fe688cf1..000000000 --- a/pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere-proxy.yaml +++ /dev/null @@ -1,557 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 -fqdn: node1 - - -ssh_pwauth: false - -write_files: -- path: "/opt/restart-kubelet.sh" - permissions: "0755" - content: | - #!/bin/bash - - while true; do - output=$(journalctl -u kubelet -n 1 | grep "use of closed network connection") - if [[ $? != 0 ]]; then - echo "Error not found in logs" - elif [[ $output ]]; then - echo "Restart kubelet" - systemctl restart kubelet - fi - - # Runs every 10 minutes - sleep 600 - done - -- path: "/etc/systemd/system/kubelet-restart.service" - permissions: "0644" - content: | - [Unit] - Description=Restarts kubelet on use of closed network connection error - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/kubelet-restart.sh - - [Install] - WantedBy=multi-user.target -- path: "/etc/environment" - content: | - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - hostnamectl set-hostname node1 - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - [plugins."io.containerd.grpc.v1.cri".registry.configs] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000".tls] - insecure_skip_verify = true - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000".tls] - insecure_skip_verify = true - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere.yaml deleted file mode 100644 index 2221a9b2b..000000000 --- a/pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere.yaml +++ /dev/null @@ -1,541 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 -fqdn: node1 - - -ssh_pwauth: false - -write_files: -- path: "/opt/restart-kubelet.sh" - permissions: "0755" - content: | - #!/bin/bash - - while true; do - output=$(journalctl -u kubelet -n 1 | grep "use of closed network connection") - if [[ $? != 0 ]]; then - echo "Error not found in logs" - elif [[ $output ]]; then - echo "Restart kubelet" - systemctl restart kubelet - fi - - # Runs every 10 minutes - sleep 600 - done - -- path: "/etc/systemd/system/kubelet-restart.service" - permissions: "0644" - content: | - [Unit] - Description=Restarts kubelet on use of closed network connection error - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/kubelet-restart.sh - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - hostnamectl set-hostname node1 - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml deleted file mode 100644 index cc0db60c7..000000000 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ /dev/null @@ -1,538 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 -fqdn: node1 - - -ssh_pwauth: false - -write_files: -- path: "/opt/restart-kubelet.sh" - permissions: "0755" - content: | - #!/bin/bash - - while true; do - output=$(journalctl -u kubelet -n 1 | grep "use of closed network connection") - if [[ $? != 0 ]]; then - echo "Error not found in logs" - elif [[ $output ]]; then - echo "Restart kubelet" - systemctl restart kubelet - fi - - # Runs every 10 minutes - sleep 600 - done - -- path: "/etc/systemd/system/kubelet-restart.service" - permissions: "0644" - content: | - [Unit] - Description=Restarts kubelet on use of closed network connection error - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/kubelet-restart.sh - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - hostnamectl set-hostname node1 - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=azure \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rockylinux/provider_test.go b/pkg/userdata/rockylinux/provider_test.go index f5c883cb1..259b9ca1b 100644 --- a/pkg/userdata/rockylinux/provider_test.go +++ b/pkg/userdata/rockylinux/provider_test.go @@ -129,54 +129,6 @@ func TestUserDataGeneration(t *testing.T) { }, externalCloudProvider: true, }, - { - name: "kubelet-v1.29.2-vsphere", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", - }, - }, - cloudProviderName: stringPtr("vsphere"), - }, - { - name: "kubelet-v1.29.2-vsphere-proxy", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", - }, - }, - cloudProviderName: stringPtr("vsphere"), - httpProxy: "http://192.168.100.100:3128", - noProxy: "192.168.1.0", - insecureRegistries: "192.168.100.100:5000, 10.0.0.1:5000", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "kubelet-v1.29.2-vsphere-mirrors", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", - }, - }, - cloudProviderName: stringPtr("vsphere"), - httpProxy: "http://192.168.100.100:3128", - noProxy: "192.168.1.0", - registryMirrors: "https://registry.docker-cn.com", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "kubelet-v1.29.2-nutanix", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", - }, - }, - cloudProviderName: stringPtr("nutanix"), - }, } defaultCloudProvider := &fakeCloudConfigProvider{ diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml deleted file mode 100644 index 1eb421418..000000000 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ /dev/null @@ -1,456 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 - - -ssh_pwauth: false - -write_files: -- path: "/etc/environment" - content: | - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - tar \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"},"insecure-registries":["192.168.100.100:5000","10.0.0.1:5000"]} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml deleted file mode 100644 index 60f826744..000000000 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml +++ /dev/null @@ -1,447 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - tar \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-nutanix.yaml deleted file mode 100644 index 9e35bb4f3..000000000 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-nutanix.yaml +++ /dev/null @@ -1,475 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - tar \ - iscsi-initiator-utils \ - ipvsadm - systemctl enable --now iscsid - - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=nutanix \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml deleted file mode 100644 index ae8c9d41c..000000000 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml +++ /dev/null @@ -1,484 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 - - -ssh_pwauth: false - -write_files: -- path: "/etc/environment" - content: | - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - tar \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry.docker-cn.com"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-proxy.yaml deleted file mode 100644 index 88f42cd35..000000000 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-proxy.yaml +++ /dev/null @@ -1,491 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 - - -ssh_pwauth: false - -write_files: -- path: "/etc/environment" - content: | - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - tar \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - [plugins."io.containerd.grpc.v1.cri".registry.configs] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000".tls] - insecure_skip_verify = true - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000".tls] - insecure_skip_verify = true - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere.yaml deleted file mode 100644 index 65fe363ec..000000000 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere.yaml +++ /dev/null @@ -1,475 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - tar \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index 492c9fcd4..8d65fa3f2 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -327,33 +327,6 @@ func TestUserDataGeneration(t *testing.T) { }, externalCloudProvider: true, }, - { - name: "digitalocean-dualstack", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "digitalocean", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - Network: &providerconfigtypes.NetworkConfig{ - IPFamily: util.IPFamilyIPv4IPv6, - }, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - config: "{digitalocean-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), net.ParseIP("10.10.10.12")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, { name: "openstack-dualstack-IPv6+IPv4", providerSpec: &providerconfigtypes.Config{ @@ -383,33 +356,6 @@ func TestUserDataGeneration(t *testing.T) { }, externalCloudProvider: true, }, - { - name: "digitalocean-dualstack-IPv6+IPv4", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "digitalocean", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - Network: &providerconfigtypes.NetworkConfig{ - IPFamily: util.IPFamilyIPv6IPv4, - }, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - config: "{digitalocean-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), net.ParseIP("10.10.10.12")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, { name: "openstack-overwrite-cloud-config", providerSpec: &providerconfigtypes.Config{ @@ -436,92 +382,6 @@ func TestUserDataGeneration(t *testing.T) { DistUpgradeOnBoot: false, }, }, - { - name: "vsphere", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "vsphere", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - OverwriteCloudConfig: stringPtr("custom\ncloud\nconfig"), - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.27.0", - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "vsphere", - config: "{vsphere-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - { - name: "vsphere-proxy", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "vsphere", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - OverwriteCloudConfig: stringPtr("custom\ncloud\nconfig"), - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.27.0", - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "vsphere", - config: "{vsphere-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - httpProxy: "http://192.168.100.100:3128", - noProxy: "192.168.1.0", - insecureRegistries: "192.168.100.100:5000, 10.0.0.1:5000", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "vsphere-mirrors", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "vsphere", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - OverwriteCloudConfig: stringPtr("custom\ncloud\nconfig"), - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.27.0", - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "vsphere", - config: "{vsphere-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - httpProxy: "http://192.168.100.100:3128", - noProxy: "192.168.1.0", - registryMirrors: "https://registry.docker-cn.com", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, { name: "containerd", containerruntime: "containerd", @@ -590,32 +450,6 @@ func TestUserDataGeneration(t *testing.T) { DistUpgradeOnBoot: true, }, }, - { - name: "nutanix", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "nutanix", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - OverwriteCloudConfig: stringPtr("custom\ncloud\nconfig"), - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.0", - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "nutanix", - config: "{nutanix-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, }...) for _, test := range tests { diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml deleted file mode 100644 index 8ced434e6..000000000 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml +++ /dev/null @@ -1,472 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {digitalocean-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - DEFAULT_IFC_IP6=$(ip -o -6 route get 1:: | grep -oP "src \K\S+") - if [ -z "${DEFAULT_IFC_IP6}" ] - then - echodate "Failed to get IPv6 address for the default route interface" - exit 1 - fi - DEFAULT_IFC_IP=$DEFAULT_IFC_IP6,$DEFAULT_IFC_IP - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - - 10.10.10.11 - - 10.10.10.12 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml deleted file mode 100644 index 8fbbf463a..000000000 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml +++ /dev/null @@ -1,472 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {digitalocean-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - DEFAULT_IFC_IP6=$(ip -o -6 route get 1:: | grep -oP "src \K\S+") - if [ -z "${DEFAULT_IFC_IP6}" ] - then - echodate "Failed to get IPv6 address for the default route interface" - exit 1 - fi - DEFAULT_IFC_IP=$DEFAULT_IFC_IP,$DEFAULT_IFC_IP6 - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - - 10.10.10.11 - - 10.10.10.12 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml deleted file mode 100644 index 10cb29335..000000000 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ /dev/null @@ -1,470 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - open-iscsi \ - ipvsadm - systemctl enable --now iscsid - - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=nutanix \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - custom - cloud - config - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml deleted file mode 100644 index e64e6f726..000000000 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ /dev/null @@ -1,478 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: -- path: "/etc/environment" - content: | - PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - open-vm-tools \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - custom - cloud - config - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry.docker-cn.com"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml deleted file mode 100644 index cf2ca8b6f..000000000 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ /dev/null @@ -1,485 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: -- path: "/etc/environment" - content: | - PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - open-vm-tools \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - custom - cloud - config - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - [plugins."io.containerd.grpc.v1.cri".registry.configs] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000".tls] - insecure_skip_verify = true - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000".tls] - insecure_skip_verify = true - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml deleted file mode 100644 index 880c25728..000000000 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ /dev/null @@ -1,468 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - open-vm-tools \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - custom - cloud - config - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 4d205b01b..7c62754c3 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -52,36 +52,15 @@ func init() { } const ( - DOManifest = "./testdata/machinedeployment-digitalocean.yaml" - AWSManifest = "./testdata/machinedeployment-aws.yaml" - AWSSpotInstanceManifest = "./testdata/machinedeployment-aws-spot-instances.yaml" - AWSManifestARM = "./testdata/machinedeployment-aws-arm-machines.yaml" - AWSEBSEncryptedManifest = "./testdata/machinedeployment-aws-ebs-encryption-enabled.yaml" - AzureManifest = "./testdata/machinedeployment-azure.yaml" - AzureRedhatSatelliteManifest = "./testdata/machinedeployment-azure.yaml" - AzureCustomImageReferenceManifest = "./testdata/machinedeployment-azure-custom-image-reference.yaml" - EquinixMetalManifest = "./testdata/machinedeployment-equinixmetal.yaml" - GCEManifest = "./testdata/machinedeployment-gce.yaml" - HZManifest = "./testdata/machinedeployment-hetzner.yaml" - LinodeManifest = "./testdata/machinedeployment-linode.yaml" - VMwareCloudDirectorManifest = "./testdata/machinedeployment-vmware-cloud-director.yaml" - VSPhereManifest = "./testdata/machinedeployment-vsphere.yaml" - VSPhereAntiAffinityManifest = "./testdata/machinedeployment-vsphere-anti-affinity.yaml" - VSPhereMultipleNICManifest = "./testdata/machinedeployment-vsphere-multiple-nic.yaml" - VSPhereDSCManifest = "./testdata/machinedeployment-vsphere-datastore-cluster.yaml" - VSPhereResourcePoolManifest = "./testdata/machinedeployment-vsphere-resource-pool.yaml" - ScalewayManifest = "./testdata/machinedeployment-scaleway.yaml" - OSMachineManifest = "./testdata/machine-openstack.yaml" - OSManifest = "./testdata/machinedeployment-openstack.yaml" - OSManifestProjectAuth = "./testdata/machinedeployment-openstack-project-auth.yaml" - OSUpgradeManifest = "./testdata/machinedeployment-openstack-upgrade.yml" - invalidMachineManifest = "./testdata/machine-invalid.yaml" - kubevirtManifest = "./testdata/machinedeployment-kubevirt.yaml" - alibabaManifest = "./testdata/machinedeployment-alibaba.yaml" - anexiaManifest = "./testdata/machinedeployment-anexia.yaml" - nutanixManifest = "./testdata/machinedeployment-nutanix.yaml" - vultrManifest = "./testdata/machinedeployment-vultr.yaml" - openNebulaManifest = "./testdata/machinedeployment-opennebula.yaml" + AWSManifest = "./testdata/machinedeployment-aws.yaml" + AWSSpotInstanceManifest = "./testdata/machinedeployment-aws-spot-instances.yaml" + AWSManifestARM = "./testdata/machinedeployment-aws-arm-machines.yaml" + AWSEBSEncryptedManifest = "./testdata/machinedeployment-aws-ebs-encryption-enabled.yaml" + OSMachineManifest = "./testdata/machine-openstack.yaml" + OSManifest = "./testdata/machinedeployment-openstack.yaml" + OSManifestProjectAuth = "./testdata/machinedeployment-openstack-project-auth.yaml" + OSUpgradeManifest = "./testdata/machinedeployment-openstack-upgrade.yml" + invalidMachineManifest = "./testdata/machine-invalid.yaml" ) const ( @@ -582,437 +561,6 @@ func TestAWSEbsEncryptionEnabledProvisioningE2E(t *testing.T) { testScenario(t, scenario, fmt.Sprintf("aws-%s", *testRunIdentifier), params, AWSEBSEncryptedManifest, false) } -// TestAzureProvisioningE2E - a test suite that exercises Azure provider -// by requesting nodes with different combination of container runtime type, container runtime version and the OS flavour. -func TestAzureProvisioningE2E(t *testing.T) { - t.Parallel() - - // test data - azureTenantID := os.Getenv("AZURE_E2E_TESTS_TENANT_ID") - azureSubscriptionID := os.Getenv("AZURE_E2E_TESTS_SUBSCRIPTION_ID") - azureClientID := os.Getenv("AZURE_E2E_TESTS_CLIENT_ID") - azureClientSecret := os.Getenv("AZURE_E2E_TESTS_CLIENT_SECRET") - if len(azureTenantID) == 0 || len(azureSubscriptionID) == 0 || len(azureClientID) == 0 || len(azureClientSecret) == 0 { - t.Fatal("Unable to run the test suite, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID, AZURE_CLIENT_ID and AZURE_CLIENT_SECRET environment variables cannot be empty") - } - - selector := Not(OsSelector("amzn2")) - - // act - params := []string{ - fmt.Sprintf("<< AZURE_TENANT_ID >>=%s", azureTenantID), - fmt.Sprintf("<< AZURE_SUBSCRIPTION_ID >>=%s", azureSubscriptionID), - fmt.Sprintf("<< AZURE_CLIENT_ID >>=%s", azureClientID), - fmt.Sprintf("<< AZURE_CLIENT_SECRET >>=%s", azureClientSecret), - fmt.Sprintf("<< AZURE_OS_DISK_SKU >>=%s", "Standard_LRS"), - fmt.Sprintf("<< AZURE_DATA_DISK_SKU >>=%s", "Standard_LRS"), - } - runScenarios(t, selector, params, AzureManifest, fmt.Sprintf("azure-%s", *testRunIdentifier)) -} - -// TestAzureCustomImageReferenceProvisioningE2E - a test suite that exercises Azure provider -// by requesting nodes with different combination of container runtime type, container runtime version and custom Image reference. -func TestAzureCustomImageReferenceProvisioningE2E(t *testing.T) { - t.Parallel() - - // test data - azureTenantID := os.Getenv("AZURE_E2E_TESTS_TENANT_ID") - azureSubscriptionID := os.Getenv("AZURE_E2E_TESTS_SUBSCRIPTION_ID") - azureClientID := os.Getenv("AZURE_E2E_TESTS_CLIENT_ID") - azureClientSecret := os.Getenv("AZURE_E2E_TESTS_CLIENT_SECRET") - if len(azureTenantID) == 0 || len(azureSubscriptionID) == 0 || len(azureClientID) == 0 || len(azureClientSecret) == 0 { - t.Fatal("Unable to run the test suite, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID, AZURE_CLIENT_ID and AZURE_CLIENT_SECRET environment variables cannot be empty") - } - - selector := OsSelector("ubuntu") - // act - params := []string{ - fmt.Sprintf("<< AZURE_TENANT_ID >>=%s", azureTenantID), - fmt.Sprintf("<< AZURE_SUBSCRIPTION_ID >>=%s", azureSubscriptionID), - fmt.Sprintf("<< AZURE_CLIENT_ID >>=%s", azureClientID), - fmt.Sprintf("<< AZURE_CLIENT_SECRET >>=%s", azureClientSecret), - fmt.Sprintf("<< AZURE_OS_DISK_SKU >>=%s", "Standard_LRS"), - fmt.Sprintf("<< AZURE_DATA_DISK_SKU >>=%s", "Standard_LRS"), - } - runScenarios(t, selector, params, AzureCustomImageReferenceManifest, fmt.Sprintf("azure-%s", *testRunIdentifier)) -} - -// TestAzureRedhatSatelliteProvisioningE2E - a test suite that exercises Azure provider -// by requesting rhel node and subscribe to redhat satellite server. -func TestAzureRedhatSatelliteProvisioningE2E(t *testing.T) { - t.Parallel() - t.Skip() - - // test data - azureTenantID := os.Getenv("AZURE_E2E_TESTS_TENANT_ID") - azureSubscriptionID := os.Getenv("AZURE_E2E_TESTS_SUBSCRIPTION_ID") - azureClientID := os.Getenv("AZURE_E2E_TESTS_CLIENT_ID") - azureClientSecret := os.Getenv("AZURE_E2E_TESTS_CLIENT_SECRET") - if len(azureTenantID) == 0 || len(azureSubscriptionID) == 0 || len(azureClientID) == 0 || len(azureClientSecret) == 0 { - t.Fatal("Unable to run the test suite, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID, AZURE_CLIENT_ID and AZURE_CLIENT_SECRET environment variables cannot be empty") - } - - // act - params := []string{ - fmt.Sprintf("<< AZURE_TENANT_ID >>=%s", azureTenantID), - fmt.Sprintf("<< AZURE_SUBSCRIPTION_ID >>=%s", azureSubscriptionID), - fmt.Sprintf("<< AZURE_CLIENT_ID >>=%s", azureClientID), - fmt.Sprintf("<< AZURE_CLIENT_SECRET >>=%s", azureClientSecret), - fmt.Sprintf("<< AZURE_OS_DISK_SKU >>=%s", "Standard_LRS"), - fmt.Sprintf("<< AZURE_DATA_DISK_SKU >>=%s", "Standard_LRS"), - } - - scenario := scenario{ - name: "Azure redhat satellite server subscription", - osName: "rhel", - containerRuntime: defaultContainerRuntime, - kubernetesVersion: defaultKubernetesVersion, - executor: verifyCreateAndDelete, - } - - testScenario(t, scenario, *testRunIdentifier, params, AzureRedhatSatelliteManifest, false) -} - -// TestGCEProvisioningE2E - a test suite that exercises Google Cloud provider -// by requesting nodes with different combination of container runtime type, -// container runtime version and the OS flavour. -func TestGCEProvisioningE2E(t *testing.T) { - t.Parallel() - - // Test data. - googleServiceAccount := os.Getenv("GOOGLE_SERVICE_ACCOUNT") - if len(googleServiceAccount) == 0 { - t.Fatal("Unable to run the test suite, GOOGLE_SERVICE_ACCOUNT environment variable cannot be empty") - } - - // Act. GCE does not support CentOS. - selector := And(OsSelector("ubuntu", "flatcar"), Not(VersionSelector("1.29.2"))) - params := []string{ - fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>=%s", safeBase64Encoding(googleServiceAccount)), - } - - runScenarios(t, selector, params, GCEManifest, fmt.Sprintf("gce-%s", *testRunIdentifier)) -} - -// TestHetznerProvisioning - a test suite that exercises Hetzner provider -// by requesting nodes with different combination of container runtime type, container runtime version and the OS flavour. -func TestHetznerProvisioningE2E(t *testing.T) { - t.Parallel() - - // test data - hzToken := os.Getenv("HZ_E2E_TOKEN") - if len(hzToken) == 0 { - t.Fatal("Unable to run the test suite, HZ_E2E_TOKEN environment variable cannot be empty") - } - - selector := OsSelector("ubuntu", "rockylinux") - - // act - params := []string{fmt.Sprintf("<< HETZNER_TOKEN >>=%s", hzToken)} - runScenarios(t, selector, params, HZManifest, fmt.Sprintf("hz-%s", *testRunIdentifier)) -} - -// TestEquinixMetalProvisioningE2E - a test suite that exercises Equinix Metal provider -// by requesting nodes with different combination of container runtime type, container runtime version and the OS flavour. -func TestEquinixMetalProvisioningE2E(t *testing.T) { - t.Parallel() - - // test data - token := os.Getenv("METAL_AUTH_TOKEN") - if len(token) == 0 { - t.Fatal("Unable to run the test suite, METAL_AUTH_TOKEN environment variable cannot be empty") - } - - projectID := os.Getenv("METAL_PROJECT_ID") - if len(projectID) == 0 { - t.Fatal("Unable to run the test suite, METAL_PROJECT_ID environment variable cannot be empty") - } - - selector := And(OsSelector("ubuntu", "centos", "rockylinux", "flatcar"), Not(NameSelector("migrateUID"))) - - // act - params := []string{ - fmt.Sprintf("<< METAL_AUTH_TOKEN >>=%s", token), - fmt.Sprintf("<< METAL_PROJECT_ID >>=%s", projectID), - } - runScenarios(t, selector, params, EquinixMetalManifest, fmt.Sprintf("equinixmetal-%s", *testRunIdentifier)) -} - -func TestAlibabaProvisioningE2E(t *testing.T) { - t.Parallel() - - // test data - accessKeyID := os.Getenv("ALIBABA_ACCESS_KEY_ID") - if len(accessKeyID) == 0 { - t.Fatal("Unable to run the test suite, ALIBABA_ACCESS_KEY_ID environment variable cannot be empty") - } - - accessKeySecret := os.Getenv("ALIBABA_ACCESS_KEY_SECRET") - if len(accessKeySecret) == 0 { - t.Fatal("Unable to run the test suite, ALIBABA_ACCESS_KEY_SECRET environment variable cannot be empty") - } - - selector := OsSelector("ubuntu") - - // act - params := []string{ - fmt.Sprintf("<< ALIBABA_ACCESS_KEY_ID >>=%s", accessKeyID), - fmt.Sprintf("<< ALIBABA_ACCESS_KEY_SECRET >>=%s", accessKeySecret), - } - runScenarios(t, selector, params, alibabaManifest, fmt.Sprintf("alibaba-%s", *testRunIdentifier)) -} - -// TestLinodeProvisioning - a test suite that exercises Linode provider -// by requesting nodes with different combination of container runtime type, container runtime version and the OS flavour. -// -// note that tests require a valid API Token that is read from the LINODE_E2E_TEST_TOKEN environmental variable. -func TestLinodeProvisioningE2E(t *testing.T) { - t.Parallel() - - // test data - linodeToken := os.Getenv("LINODE_E2E_TESTS_TOKEN") - if len(linodeToken) == 0 { - t.Fatal("Unable to run the test suite, LINODE_E2E_TESTS_TOKEN environment variable cannot be empty") - } - - // we're shimming userdata through Linode stackscripts and the stackscript hasn't been verified for use with centos - selector := OsSelector("ubuntu") - - // act - params := []string{fmt.Sprintf("<< LINODE_TOKEN >>=%s", linodeToken)} - runScenarios(t, selector, params, LinodeManifest, fmt.Sprintf("linode-%s", *testRunIdentifier)) -} - -func getVMwareCloudDirectorTestParams(t *testing.T) []string { - // test data - password := os.Getenv("VCD_PASSWORD") - username := os.Getenv("VCD_USER") - organization := os.Getenv("VCD_ORG") - url := os.Getenv("VCD_URL") - vdc := os.Getenv("VCD_VDC") - - if password == "" || username == "" || organization == "" || url == "" || vdc == "" { - t.Fatal("Unable to run the test suite, VCD_PASSWORD, VCD_USER, VCD_ORG, " + - "VCD_URL, or VCD_VDC environment variables cannot be empty") - } - - // set up parameters - params := []string{fmt.Sprintf("<< VCD_PASSWORD >>=%s", password), - fmt.Sprintf("<< VCD_USER >>=%s", username), - fmt.Sprintf("<< VCD_ORG >>=%s", organization), - fmt.Sprintf("<< VCD_URL >>=%s", url), - fmt.Sprintf("<< VCD_VDC >>=%s", vdc), - } - return params -} - -func TestVMwareCloudDirectorProvisioningE2E(t *testing.T) { - t.Parallel() - - selector := OsSelector("ubuntu") - params := getVMwareCloudDirectorTestParams(t) - - runScenarios(t, selector, params, VMwareCloudDirectorManifest, fmt.Sprintf("vcd-%s", *testRunIdentifier)) -} - -func getVSphereTestParams(t *testing.T) []string { - // test data - vsPassword := os.Getenv("VSPHERE_E2E_PASSWORD") - vsUsername := os.Getenv("VSPHERE_E2E_USERNAME") - vsAddress := os.Getenv("VSPHERE_E2E_ADDRESS") - - if vsPassword == "" || vsUsername == "" || vsAddress == "" { - t.Fatal("Unable to run the test suite, VSPHERE_E2E_PASSWORD, VSPHERE_E2E_USERNAME" + - "or VSPHERE_E2E_ADDRESS environment variables cannot be empty") - } - - // act - params := []string{fmt.Sprintf("<< VSPHERE_PASSWORD >>=%s", vsPassword), - fmt.Sprintf("<< VSPHERE_USERNAME >>=%s", vsUsername), - fmt.Sprintf("<< VSPHERE_ADDRESS >>=%s", vsAddress), - } - return params -} - -// TestVsphereProvisioning - a test suite that exercises vsphere provider -// by requesting nodes with different combination of container runtime type, container runtime version and the OS flavour. -func TestVsphereProvisioningE2E(t *testing.T) { - t.Parallel() - - selector := Not(OsSelector("amzn2", "centos")) - params := getVSphereTestParams(t) - - runScenarios(t, selector, params, VSPhereManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) -} - -// TestVsphereMultipleNICProvisioning - is the same as the TestVsphereProvisioning suit but has multiple networks attached to the VMs. -// by requesting nodes with different combination of container runtime type, container runtime version and the OS flavour. -func TestVsphereMultipleNICProvisioningE2E(t *testing.T) { - t.Parallel() - - selector := OsSelector("ubuntu") - params := getVSphereTestParams(t) - - runScenarios(t, selector, params, VSPhereMultipleNICManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) -} - -// TestVsphereAntiAffinityProvisioningE2E - is the same as the TestVsphereProvisioning suit but has anti-affinity rules applied to the VMs. -func TestVsphereAntiAffinityProvisioningE2E(t *testing.T) { - t.Parallel() - - params := getVSphereTestParams(t) - - scenario := scenario{ - name: "VSphere Anti-Affinity provisioning", - osName: "ubuntu", - containerRuntime: defaultContainerRuntime, - kubernetesVersion: defaultKubernetesVersion, - executor: verifyCreateAndDelete, - } - - testScenario(t, scenario, *testRunIdentifier, params, VSPhereAntiAffinityManifest, false) -} - -// TestVsphereDatastoreClusterProvisioning - is the same as the TestVsphereProvisioning suite but specifies a DatastoreCluster -// instead of the Datastore in the provider specs. -func TestVsphereDatastoreClusterProvisioningE2E(t *testing.T) { - t.Parallel() - - selector := OsSelector("ubuntu", "centos", "rhel", "flatcar") - - params := getVSphereTestParams(t) - runScenarios(t, selector, params, VSPhereDSCManifest, fmt.Sprintf("vs-dsc-%s", *testRunIdentifier)) -} - -// TestVsphereResourcePoolProvisioning - creates a machine deployment using a -// resource pool. -func TestVsphereResourcePoolProvisioningE2E(t *testing.T) { - t.Parallel() - - params := getVSphereTestParams(t) - // We do not need to test all combinations. - scenario := scenario{ - name: "vSphere resource pool provisioning", - osName: "flatcar", - containerRuntime: defaultContainerRuntime, - kubernetesVersion: defaultKubernetesVersion, - executor: verifyCreateAndDelete, - } - - testScenario(t, scenario, *testRunIdentifier, params, VSPhereResourcePoolManifest, false) -} - -// TestScalewayProvisioning - a test suite that exercises scaleway provider -// by requesting nodes with different combination of container runtime type, container runtime version and the OS flavour. -// -// note that tests require the following environment variable: -// - SCW_ACCESS_KEY -> the Scaleway Access Key -// - SCW_SECRET_KEY -> the Scaleway Secret Key -// - SCW_DEFAULT_PROJECT_ID -> the Scaleway Project ID. -func TestScalewayProvisioningE2E(t *testing.T) { - t.Parallel() - - // test data - scwAccessKey := os.Getenv("SCW_ACCESS_KEY") - if len(scwAccessKey) == 0 { - t.Fatal("Unable to run the test suite, SCW_E2E_TEST_ACCESS_KEY environment variable cannot be empty") - } - - scwSecretKey := os.Getenv("SCW_SECRET_KEY") - if len(scwSecretKey) == 0 { - t.Fatal("Unable to run the test suite, SCW_E2E_TEST_SECRET_KEY environment variable cannot be empty") - } - - scwProjectID := os.Getenv("SCW_DEFAULT_PROJECT_ID") - if len(scwProjectID) == 0 { - t.Fatal("Unable to run the test suite, SCW_E2E_TEST_PROJECT_ID environment variable cannot be empty") - } - - selector := Not(OsSelector("rhel", "flatcar", "rockylinux")) - // act - params := []string{ - fmt.Sprintf("<< SCW_ACCESS_KEY >>=%s", scwAccessKey), - fmt.Sprintf("<< SCW_SECRET_KEY >>=%s", scwSecretKey), - fmt.Sprintf("<< SCW_DEFAULT_PROJECT_ID >>=%s", scwProjectID), - } - runScenarios(t, selector, params, ScalewayManifest, fmt.Sprintf("scw-%s", *testRunIdentifier)) -} - -func getNutanixTestParams(t *testing.T) []string { - // test data - password := os.Getenv("NUTANIX_E2E_PASSWORD") - username := os.Getenv("NUTANIX_E2E_USERNAME") - cluster := os.Getenv("NUTANIX_E2E_CLUSTER_NAME") - project := os.Getenv("NUTANIX_E2E_PROJECT_NAME") - subnet := os.Getenv("NUTANIX_E2E_SUBNET_NAME") - additionalSubnetNames := os.Getenv("NUTANIX_E2E_ADDITIONAL_SUBNET_NAMES") - endpoint := os.Getenv("NUTANIX_E2E_ENDPOINT") - - if password == "" || username == "" || endpoint == "" || cluster == "" || project == "" || subnet == "" { - t.Fatal("Unable to run the test suite, NUTANIX_E2E_PASSWORD, NUTANIX_E2E_USERNAME, NUTANIX_E2E_CLUSTER_NAME, " + - "NUTANIX_E2E_ENDPOINT, NUTANIX_E2E_PROJECT_NAME or NUTANIX_E2E_SUBNET_NAME environment variables cannot be empty") - } - - // set up parameters - params := []string{fmt.Sprintf("<< NUTANIX_PASSWORD >>=%s", password), - fmt.Sprintf("<< NUTANIX_USERNAME >>=%s", username), - fmt.Sprintf("<< NUTANIX_ENDPOINT >>=%s", endpoint), - fmt.Sprintf("<< NUTANIX_CLUSTER >>=%s", cluster), - fmt.Sprintf("<< NUTANIX_PROJECT >>=%s", project), - fmt.Sprintf("<< NUTANIX_SUBNET >>=%s", subnet), - fmt.Sprintf("<< NUTANIX_ADDITIONAL_SUBNETS >>=%s", additionalSubnetNames), - } - return params -} - -// TestNutanixProvisioningE2E tests provisioning on Nutanix as cloud provider. -func TestNutanixProvisioningE2E(t *testing.T) { - t.Parallel() - - // exclude migrateUID test case because it's a no-op for Nutanix and runs from a different - // location, thus possibly blocking access a HTTP proxy if it is configured. - selector := And(OsSelector("ubuntu", "centos"), Not(NameSelector("migrateUID"))) - params := getNutanixTestParams(t) - runScenarios(t, selector, params, nutanixManifest, fmt.Sprintf("nx-%s", *testRunIdentifier)) -} - -func TestOpenNebulaProvisioningE2E(t *testing.T) { - t.Parallel() - - oneEndpoint := os.Getenv("ONE_ENDPOINT") - oneUsername := os.Getenv("ONE_USERNAME") - onePassword := os.Getenv("ONE_PASSWORD") - - // required parameters - if oneEndpoint == "" || oneUsername == "" || onePassword == "" { - t.Fatal("unable to run test suite, all of ONE_ENDPOINT, ONE_USERNAME, and ONE_PASSWORD must be set!") - } - - // optional parameters - oneDatastore := os.Getenv("ONE_DATASTORE") - oneNetwork := os.Getenv("ONE_NETWORK") - - // set defaults for minione deployments - if oneDatastore == "" { - oneDatastore = "default" - } - - if oneNetwork == "" { - oneNetwork = "vnet" - } - - params := []string{ - fmt.Sprintf("<< ONE_ENDPOINT >>=%s", oneEndpoint), - fmt.Sprintf("<< ONE_USERNAME >>=%s", oneUsername), - fmt.Sprintf("<< ONE_PASSWORD >>=%s", onePassword), - fmt.Sprintf("<< ONE_DATASTORE_NAME >>=%s", oneDatastore), - fmt.Sprintf("<< ONE_NETWORK_NAME >>=%s", oneNetwork), - } - - selector := OsSelector("rockylinux", "flatcar") - runScenarios(t, selector, params, openNebulaManifest, fmt.Sprintf("one-%s", *testRunIdentifier)) -} - // TestUbuntuProvisioningWithUpgradeE2E will create an instance from an old Ubuntu 1604 // image and upgrade it prior to joining the cluster. func TestUbuntuProvisioningWithUpgradeE2E(t *testing.T) { @@ -1050,68 +598,3 @@ func TestUbuntuProvisioningWithUpgradeE2E(t *testing.T) { testScenario(t, scenario, *testRunIdentifier, params, OSUpgradeManifest, false) } - -// TestDeploymentControllerUpgradesMachineE2E verifies the machineDeployment controller correctly -// rolls over machines on changes in the machineDeployment. -func TestDeploymentControllerUpgradesMachineE2E(t *testing.T) { - t.Parallel() - - // test data - hzToken := os.Getenv("HZ_E2E_TOKEN") - if len(hzToken) == 0 { - t.Fatal("Unable to run the test suite, HZ_E2E_TOKEN environment variable cannot be empty") - } - - // act - params := []string{fmt.Sprintf("<< HETZNER_TOKEN >>=%s", hzToken)} - - scenario := scenario{ - name: "MachineDeployment upgrade", - osName: "ubuntu", - containerRuntime: defaultContainerRuntime, - kubernetesVersion: defaultKubernetesVersion, - executor: verifyCreateUpdateAndDelete, - } - testScenario(t, scenario, *testRunIdentifier, params, HZManifest, false) -} - -func TestAnexiaProvisioningE2E(t *testing.T) { - t.Parallel() - - token := os.Getenv("ANEXIA_TOKEN") - vlanID := os.Getenv("ANEXIA_VLAN_ID") - templateID := os.Getenv("ANEXIA_TEMPLATE_ID") - locationID := os.Getenv("ANEXIA_LOCATION_ID") - - if token == "" || vlanID == "" || templateID == "" || locationID == "" { - t.Fatal("Unable to run test suite, all of ANEXIA_TOKEN, ANEXIA_VLAN_ID, ANEXIA_TEMPLATE_ID, and ANEXIA_LOCATION_ID must be set!") - } - - selector := OsSelector("flatcar") - params := []string{ - fmt.Sprintf("<< ANEXIA_TOKEN >>=%s", token), - fmt.Sprintf("<< ANEXIA_VLAN_ID >>=%s", vlanID), - fmt.Sprintf("<< ANEXIA_TEMPLATE_ID >>=%s", templateID), - fmt.Sprintf("<< ANEXIA_LOCATION_ID >>=%s", locationID), - } - - runScenarios(t, selector, params, anexiaManifest, fmt.Sprintf("anexia-%s", *testRunIdentifier)) -} - -// TestVultrProvisioning - a test suite that exercises Vultr provider -// by requesting nodes with different combination of container runtime type, container runtime version and the OS flavour. -func TestVultrProvisioningE2E(t *testing.T) { - t.Parallel() - - // test data - apiKey := os.Getenv("VULTR_API_KEY") - if len(apiKey) == 0 { - t.Fatal("Unable to run the test suite, VULTR_API_KEY environment variable cannot be empty") - } - - selector := OsSelector("ubuntu", "centos", "rockylinux") - - // act - params := []string{fmt.Sprintf("<< VULTR_API_KEY >>=%s", apiKey)} - runScenarios(t, selector, params, vultrManifest, fmt.Sprintf("vlt-%s", *testRunIdentifier)) -} diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index aff05972f..c6315829d 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -54,27 +54,6 @@ var ( string(providerconfigtypes.OperatingSystemFlatcar): "kubermatic-e2e-flatcar", string(providerconfigtypes.OperatingSystemRockyLinux): "machine-controller-e2e-rockylinux", } - - openNebulaImages = map[string]string{ - string(providerconfigtypes.OperatingSystemFlatcar): "machine-controller-e2e-flatcar", - string(providerconfigtypes.OperatingSystemRockyLinux): "machine-controller-e2e-rockylinux", - } - - vSphereOSImageTemplates = map[string]string{ - string(providerconfigtypes.OperatingSystemCentOS): "kkp-centos-7", - string(providerconfigtypes.OperatingSystemFlatcar): "kkp-flatcar-3139.2.0", - string(providerconfigtypes.OperatingSystemRHEL): "kkp-rhel-8.6", - string(providerconfigtypes.OperatingSystemRockyLinux): "kkp-rockylinux-8", - string(providerconfigtypes.OperatingSystemUbuntu): "kkp-ubuntu-22.04", - } - - kubevirtImages = map[string]string{ - string(providerconfigtypes.OperatingSystemCentOS): "centos", - string(providerconfigtypes.OperatingSystemFlatcar): "flatcar", - string(providerconfigtypes.OperatingSystemRHEL): "rhel", - string(providerconfigtypes.OperatingSystemRockyLinux): "rockylinux", - string(providerconfigtypes.OperatingSystemUbuntu): "ubuntu-22.04", - } ) type scenario struct { @@ -241,23 +220,6 @@ func testScenario(t *testing.T, testCase scenario, cloudProvider string, testPar scenarioParams = append(scenarioParams, fmt.Sprintf("<< MAX_PRICE >>=%s", "0.023")) } - if strings.Contains(cloudProvider, string(providerconfigtypes.CloudProviderEquinixMetal)) { - switch testCase.osName { - case string(providerconfigtypes.OperatingSystemCentOS): - scenarioParams = append(scenarioParams, fmt.Sprintf("<< INSTANCE_TYPE >>=%s", "m3.small.x86")) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< METRO_CODE >>=%s", "AM")) - case string(providerconfigtypes.OperatingSystemFlatcar): - scenarioParams = append(scenarioParams, fmt.Sprintf("<< INSTANCE_TYPE >>=%s", "c3.small.x86")) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< METRO_CODE >>=%s", "NY")) - case string(providerconfigtypes.OperatingSystemRockyLinux): - scenarioParams = append(scenarioParams, fmt.Sprintf("<< INSTANCE_TYPE >>=%s", "m3.small.x86")) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< METRO_CODE >>=%s", "AM")) - case string(providerconfigtypes.OperatingSystemUbuntu): - scenarioParams = append(scenarioParams, fmt.Sprintf("<< INSTANCE_TYPE >>=%s", "m3.small.x86")) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< METRO_CODE >>=%s", "TY")) - } - } - // only used by assume role scenario, otherwise empty (disabled) scenarioParams = append(scenarioParams, fmt.Sprintf("<< AWS_ASSUME_ROLE_ARN >>=%s", os.Getenv("AWS_ASSUME_ROLE_ARN"))) scenarioParams = append(scenarioParams, fmt.Sprintf("<< AWS_ASSUME_ROLE_EXTERNAL_ID >>=%s", os.Getenv("AWS_ASSUME_ROLE_EXTERNAL_ID"))) @@ -265,15 +227,6 @@ func testScenario(t *testing.T, testCase scenario, cloudProvider string, testPar // only used by OpenStack scenarios scenarioParams = append(scenarioParams, fmt.Sprintf("<< OS_IMAGE >>=%s", openStackImages[testCase.osName])) - // only used by OpenNebula scenarios - scenarioParams = append(scenarioParams, fmt.Sprintf("<< ONE_IMAGE >>=%s", openNebulaImages[testCase.osName])) - - // only use by vSphere scenarios - scenarioParams = append(scenarioParams, fmt.Sprintf("<< OS_Image_Template >>=%s", vSphereOSImageTemplates[testCase.osName])) - - // only use by KubeVirt scenarios - scenarioParams = append(scenarioParams, fmt.Sprintf("<< KUBEVIRT_OS_IMAGE >>=%s", kubevirtImages[testCase.osName])) - // default kubeconfig to the hardcoded path at which `make e2e-cluster` creates its new kubeconfig gopath := os.Getenv("GOPATH") projectDir := filepath.Join(gopath, "src/github.com/kubermatic/machine-controller") diff --git a/test/e2e/provisioning/testdata/machine-invalid.yaml b/test/e2e/provisioning/testdata/machine-invalid.yaml index 17a100569..8876b4680 100644 --- a/test/e2e/provisioning/testdata/machine-invalid.yaml +++ b/test/e2e/provisioning/testdata/machine-invalid.yaml @@ -9,9 +9,9 @@ spec: value: sshPublicKeys: - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "hetzner" + cloudProvider: "openstack" cloudProviderSpec: - token: << HETZNER_TOKEN >> + token: << OS_TOKEN >> serverType: "cx11" datacenter: "" location: "fsn1" diff --git a/test/e2e/provisioning/testdata/machinedeployment-alibaba.yaml b/test/e2e/provisioning/testdata/machinedeployment-alibaba.yaml deleted file mode 100644 index 63e9637ee..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-alibaba.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system - annotations: - k8c.io/operating-system-profile: osp-<< OS_NAME >> -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "alibaba" - cloudProviderSpec: - accessKeyID: << ALIBABA_ACCESS_KEY_ID >> - accessKeySecret: << ALIBABA_ACCESS_KEY_SECRET >> - instanceType: "ecs.c6.large" - instanceName: "alibaba-instance" - regionID: eu-central-1 - vSwitchID: "vsw-gw8g8mn4ohmj483hsylmn" - internetMaxBandwidthOut: 10 - zoneID: eu-central-1a - diskType: "cloud_efficiency" - diskSize: "40" - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-anexia.yaml b/test/e2e/provisioning/testdata/machinedeployment-anexia.yaml deleted file mode 100644 index 2507cb58d..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-anexia.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system - annotations: - k8c.io/operating-system-profile: osp-flatcar-cloud-init -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: anexia - cloudProviderSpec: - token: "<< ANEXIA_TOKEN >>" - vlanID: "<< ANEXIA_VLAN_ID >>" - templateID: "<< ANEXIA_TEMPLATE_ID >>" - locationID: "<< ANEXIA_LOCATION_ID >>" - cpus: 2 - memory: 2048 - diskSize: 60 - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - provisioningUtility: "cloud-init" - distUpgradeOnBoot: false - disableAutoUpdate: true - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-azure-custom-image-reference.yaml b/test/e2e/provisioning/testdata/machinedeployment-azure-custom-image-reference.yaml deleted file mode 100644 index 18eb88c32..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-azure-custom-image-reference.yaml +++ /dev/null @@ -1,61 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system - annotations: - k8c.io/operating-system-profile: osp-<< OS_NAME >> -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "azure" - cloudProviderSpec: - tenantID: "<< AZURE_TENANT_ID >>" - clientID: "<< AZURE_CLIENT_ID >>" - clientSecret: "<< AZURE_CLIENT_SECRET >>" - subscriptionID: "<< AZURE_SUBSCRIPTION_ID >>" - location: "westeurope" - resourceGroup: "machine-controller-e2e" - vnetResourceGroup: "" - vmSize: "Standard_F2" - # optional disk size values in GB. If not set, the defaults for the vmSize will be used. - osDiskSize: << OS_DISK_SIZE >> - dataDiskSize: << DATA_DISK_SIZE >> - vnetName: "machine-controller-e2e" - subnetName: "machine-controller-e2e" - routeTableName: "machine-controller-e2e" - imageReference: - publisher: "Canonical" - offer: "0001-com-ubuntu-server-focal" - sku: "20_04-lts" - version: "latest" - assignPublicIP: false - zones: - - "1" - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` - rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" - # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` - rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml deleted file mode 100644 index d92a6a034..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system - annotations: - k8c.io/operating-system-profile: osp-<< OS_NAME >> -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "azure" - cloudProviderSpec: - tenantID: "<< AZURE_TENANT_ID >>" - clientID: "<< AZURE_CLIENT_ID >>" - clientSecret: "<< AZURE_CLIENT_SECRET >>" - subscriptionID: "<< AZURE_SUBSCRIPTION_ID >>" - location: "westeurope" - resourceGroup: "machine-controller-e2e" - vmSize: "Standard_F2" - # optional disk size values in GB. If not set, the defaults for the vmSize will be used. - vnetName: "machine-controller-e2e" - vnetResourceGroup: "" - subnetName: "machine-controller-e2e" - imageID: "<< IMAGE_ID >>" - assignPublicIP: true - zones: - - "1" - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` - rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" - # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` - rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" - rhelSatelliteServer: "" - rhelUseSatelliteServer: false - rhelOrganizationName: "" - rhelActivationKey: "" - versions: - kubelet: 1.28.5 diff --git a/test/e2e/provisioning/testdata/machinedeployment-azure.yaml b/test/e2e/provisioning/testdata/machinedeployment-azure.yaml deleted file mode 100644 index 3b6ed09d4..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-azure.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system - annotations: - k8c.io/operating-system-profile: osp-<< OS_NAME >> -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "azure" - cloudProviderSpec: - tenantID: "<< AZURE_TENANT_ID >>" - clientID: "<< AZURE_CLIENT_ID >>" - clientSecret: "<< AZURE_CLIENT_SECRET >>" - subscriptionID: "<< AZURE_SUBSCRIPTION_ID >>" - location: "westeurope" - resourceGroup: "machine-controller-e2e" - vnetResourceGroup: "" - vmSize: "Standard_F2" - # optional disk size values in GB. If not set, the defaults for the vmSize will be used. - osDiskSize: << OS_DISK_SIZE >> - osDiskSKU: << AZURE_OS_DISK_SKU >> - dataDiskSize: << DATA_DISK_SIZE >> - dataDiskSKU: << AZURE_DATA_DISK_SKU >> - vnetName: "machine-controller-e2e" - subnetName: "machine-controller-e2e" - routeTableName: "machine-controller-e2e" - assignPublicIP: false - zones: - - "1" - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` - rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" - # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` - rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-digitalocean.yaml b/test/e2e/provisioning/testdata/machinedeployment-digitalocean.yaml deleted file mode 100644 index 19479c97d..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-digitalocean.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system - annotations: - k8c.io/operating-system-profile: osp-<< OS_NAME >> -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "digitalocean" - cloudProviderSpec: - token: << DIGITALOCEAN_TOKEN >> - region: nyc3 - size: c-2 - backups: false - ipv6: false - private_networking: true - monitoring: false - tags: - - "machine-controller" - # Can be 'ubuntu' or 'centos' - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml b/test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml deleted file mode 100644 index 398240b8d..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system - annotations: - k8c.io/operating-system-profile: osp-<< OS_NAME >> -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "equinixmetal" - cloudProviderSpec: - token: << METAL_AUTH_TOKEN >> - projectID: << METAL_PROJECT_ID >> - instanceType: << INSTANCE_TYPE >> - metro: << METRO_CODE >> - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-gce.yaml b/test/e2e/provisioning/testdata/machinedeployment-gce.yaml deleted file mode 100644 index 6b318f8b9..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-gce.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system - annotations: - k8c.io/operating-system-profile: osp-<< OS_NAME >> -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "gce" - cloudProviderSpec: - # If empty, can be set via GOOGLE_SERVICE_ACCOUNT env var. The environment variable - # should be plaintext. The value in the cloudProviderSpec however must be base64-encoded. - serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>" - # See https://cloud.google.com/compute/docs/regions-zones/ - zone: "europe-west3-a" - # See https://cloud.google.com/compute/docs/machine-types - machineType: "n1-standard-1" - # In GB - diskSize: 25 - # Can be 'pd-standard' or 'pd-ssd' - diskType: "pd-standard" - labels: - "kubernetes_cluster": "gce-test-cluster" - assignPublicIPAddress: true - customImage: "<< CUSTOM-IMAGE >>" - disableMachineServiceAccount: false - # Can be 'ubuntu' or 'rhel' - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` - rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" - # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` - rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-hetzner.yaml b/test/e2e/provisioning/testdata/machinedeployment-hetzner.yaml deleted file mode 100644 index 66a5cc2ee..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-hetzner.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system - annotations: - k8c.io/operating-system-profile: osp-<< OS_NAME >> -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "hetzner" - cloudProviderSpec: - token: << HETZNER_TOKEN >> - serverType: "cx11" - datacenter: "" - location: "nbg1" - networks: - - "machine-controller-e2e" - firewalls: - - "machine-controller-e2e" - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml deleted file mode 100644 index d97169adf..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system - annotations: - k8c.io/operating-system-profile: osp-<< OS_NAME >> -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "kubevirt" - cloudProviderSpec: - auth: - kubeconfig: - value: '<< KUBECONFIG_BASE64 >>' - virtualMachine: - template: - cpus: "1" - memory: "4096M" - primaryDisk: - osImage: http://image-repo.kube-system.svc/images/<< KUBEVIRT_OS_IMAGE >>.img - size: "25Gi" - storageClassName: rook-ceph-block - dnsPolicy: "None" - dnsConfig: - nameservers: - - 8.8.8.8 - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` - rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" - # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` - rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-linode.yaml b/test/e2e/provisioning/testdata/machinedeployment-linode.yaml deleted file mode 100644 index 055b94175..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-linode.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system - annotations: - k8c.io/operating-system-profile: osp-<< OS_NAME >> -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "linode" - cloudProviderSpec: - backups: false - private_networking: true - region: eu-west - tags: - - "machine-controller" - token: << LINODE_TOKEN >> - type: g6-standard-2 - # Can be 'ubuntu' - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml b/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml deleted file mode 100644 index 2ac36dd59..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml +++ /dev/null @@ -1,46 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system - annotations: - k8c.io/operating-system-profile: osp-<< OS_NAME >> -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "nutanix" - cloudProviderSpec: - username: '<< NUTANIX_USERNAME >>' - password: '<< NUTANIX_PASSWORD >>' - endpoint: '<< NUTANIX_ENDPOINT >>' - allowInsecure: true - clusterName: '<< NUTANIX_CLUSTER >>' - projectName: '<< NUTANIX_PROJECT >>' - subnetName: '<< NUTANIX_SUBNET >>' - additionalSubnetNames: [] - imageName: 'machine-controller-e2e-<< OS_NAME >>' - cpus: 2 - memoryMB: 2048 - diskSize: 20 - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-opennebula.yaml b/test/e2e/provisioning/testdata/machinedeployment-opennebula.yaml deleted file mode 100644 index ba1a73d84..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-opennebula.yaml +++ /dev/null @@ -1,55 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "opennebula" - cloudProviderSpec: - endpoint: "<< ONE_ENDPOINT >>" - username: "<< ONE_USERNAME >>" - password: "<< ONE_PASSWORD >>" - - cpu: 1 - vcpu: 2 - memory: 1024 - - image: "<< ONE_IMAGE >>" - datastore: "<< ONE_DATASTORE_NAME >>" - diskSize: 51200 # MB - - network: "<< ONE_NETWORK_NAME >>" - - enableVNC: true - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` - rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" - # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` - rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" - - # use cloud-init for flatcar as ignition doesn't know anything about OpenNebula yet - provisioningUtility: "cloud-init" - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-scaleway.yaml b/test/e2e/provisioning/testdata/machinedeployment-scaleway.yaml deleted file mode 100644 index 3927e59bd..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-scaleway.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system - annotations: - k8c.io/operating-system-profile: osp-<< OS_NAME >> -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "scaleway" - cloudProviderSpec: - accessKey: << SCW_ACCESS_KEY >> - secretKey: << SCW_SECRET_KEY >> - projectId: << SCW_DEFAULT_PROJECT_ID >> - commercialType: "DEV1-M" - zone: "fr-par-1" - tags: - - foo - - bar - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml b/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml deleted file mode 100644 index 8ce2a4fc0..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml +++ /dev/null @@ -1,60 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system - annotations: - k8c.io/operating-system-profile: osp-<< OS_NAME >> -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "vmware-cloud-director" - cloudProviderSpec: - username: "<< VCD_USER >>" - url: "<< VCD_URL >>" - password: "<< VCD_PASSWORD >>" - organization: "<< VCD_ORG >>" - vdc: "<< VCD_VDC >>" - allowInsecure: false - vapp: "kubermatic-e2e" - catalog: "kubermatic" - template: "machine-controller-<< OS_NAME >>" - network: "kubermatic-e2e-routed-network" - ipAllocationMode: "DHCP" - cpus: 2 - cpuCores: 1 - memoryMB: 2048 - diskSizeGB: << DISK_SIZE >> - diskBusType: "paravirtual" - diskIOPS: 0 - metadata: - key: value - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - attachSubscription: false - # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` - rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" - # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` - rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-anti-affinity.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-anti-affinity.yaml deleted file mode 100644 index 8f74c8465..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-anti-affinity.yaml +++ /dev/null @@ -1,54 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system - annotations: - k8c.io/operating-system-profile: osp-<< OS_NAME >> -spec: - replicas: 3 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "vsphere" - cloudProviderSpec: - templateVMName: '<< OS_Image_Template >>' - username: '<< VSPHERE_USERNAME >>' - vsphereURL: '<< VSPHERE_ADDRESS >>' - datacenter: 'Hamburg' - folder: '/Hamburg/vm/Kubermatic-ci' - password: << VSPHERE_PASSWORD >> - # example: 'https://your-vcenter:8443'. '/sdk' gets appended automatically - cluster: Kubermatic - vmAntiAffinity: true - datastore: vsan - cpus: 2 - MemoryMB: 4096 - diskSizeGB: << DISK_SIZE >> - allowInsecure: true - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - attachSubscription: false - # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` - rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" - # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` - rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml deleted file mode 100644 index b85102700..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system - annotations: - k8c.io/operating-system-profile: osp-<< OS_NAME >> -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "vsphere" - cloudProviderSpec: - templateVMName: '<< OS_Image_Template >>' - username: '<< VSPHERE_USERNAME >>' - vsphereURL: '<< VSPHERE_ADDRESS >>' - datacenter: 'Hamburg' - folder: '/Hamburg/vm/Kubermatic-ci' - password: << VSPHERE_PASSWORD >> - # example: 'https://your-vcenter:8443'. '/sdk' gets appended automatically - cluster: Kubermatic - vmAntiAffinity: true - datastoreCluster: 'dsc-1' - cpus: 2 - MemoryMB: 2048 - diskSizeGB: << DISK_SIZE >> - allowInsecure: true - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - attachSubscription: false - # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` - rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" - # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` - rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" - # 'rhsmOfflineToken' if it was provided red hat systems subscriptions will be removed upon machines deletions, and if wasn't - # provided the rhsm will be disabled and any created subscription won't be removed automatically - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-multiple-nic.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-multiple-nic.yaml deleted file mode 100644 index 4be08e39c..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-multiple-nic.yaml +++ /dev/null @@ -1,57 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system - annotations: - k8c.io/operating-system-profile: osp-<< OS_NAME >> -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "vsphere" - cloudProviderSpec: - templateVMName: "<< OS_Image_Template >>" - username: "<< VSPHERE_USERNAME >>" - vsphereURL: "<< VSPHERE_ADDRESS >>" - datacenter: "Hamburg" - folder: "/Hamburg/vm/Kubermatic-ci" - password: << VSPHERE_PASSWORD >> - # example: 'https://your-vcenter:8443'. '/sdk' gets appended automatically - cluster: Kubermatic - vmAntiAffinity: true - networks: - - /Hamburg/network/Default Network - - /Hamburg/network/Management - datastore: vsan - cpus: 2 - MemoryMB: 4096 - diskSizeGB: << DISK_SIZE >> - allowInsecure: true - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - attachSubscription: false - # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` - rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" - # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` - rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml deleted file mode 100644 index 54a7a345c..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml +++ /dev/null @@ -1,48 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system - annotations: - k8c.io/operating-system-profile: osp-<< OS_NAME >> -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "vsphere" - cloudProviderSpec: - templateVMName: '<< OS_Image_Template >>' - username: '<< VSPHERE_USERNAME >>' - vsphereURL: '<< VSPHERE_ADDRESS >>' - datacenter: 'Hamburg' - folder: '/Hamburg/vm/Kubermatic-ci' - password: << VSPHERE_PASSWORD >> - datastore: 'vsan' - resourcePool: 'e2e-resource-pool' - cluster: Kubermatic - vmAntiAffinity: true - cpus: 2 - MemoryMB: 2048 - diskSizeGB: << DISK_SIZE >> - allowInsecure: true - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml deleted file mode 100644 index ad4dcfda1..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml +++ /dev/null @@ -1,55 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system - annotations: - k8c.io/operating-system-profile: osp-<< OS_NAME >> -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "vsphere" - cloudProviderSpec: - templateVMName: '<< OS_Image_Template >>' - username: '<< VSPHERE_USERNAME >>' - vsphereURL: '<< VSPHERE_ADDRESS >>' - datacenter: 'Hamburg' - folder: '/Hamburg/vm/Kubermatic-ci' - password: << VSPHERE_PASSWORD >> - # example: 'https://your-vcenter:8443'. '/sdk' gets appended automatically - cluster: Kubermatic - vmAntiAffinity: true - datastore: vsan - cpus: 2 - MemoryMB: 2048 - allowInsecure: true - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" - network: - cidr: "192.168.44.<< IP_OCTET >>/20" - gateway: "192.168.32.1" - dns: - servers: - - "192.168.32.1" - - "8.8.8.8" - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml deleted file mode 100644 index c53ba3f43..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml +++ /dev/null @@ -1,54 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system - annotations: - k8c.io/operating-system-profile: osp-<< OS_NAME >> -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "vsphere" - cloudProviderSpec: - templateVMName: '<< OS_Image_Template >>' - username: '<< VSPHERE_USERNAME >>' - vsphereURL: '<< VSPHERE_ADDRESS >>' - datacenter: 'Hamburg' - folder: '/Hamburg/vm/Kubermatic-ci' - password: << VSPHERE_PASSWORD >> - # example: 'https://your-vcenter:8443'. '/sdk' gets appended automatically - cluster: Kubermatic - vmAntiAffinity: true - datastore: vsan - cpus: 2 - MemoryMB: 4096 - diskSizeGB: << DISK_SIZE >> - allowInsecure: true - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - attachSubscription: false - # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` - rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" - # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` - rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-vultr.yaml b/test/e2e/provisioning/testdata/machinedeployment-vultr.yaml deleted file mode 100644 index e7baddeb7..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-vultr.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system - annotations: - k8c.io/operating-system-profile: osp-<< OS_NAME >> -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "vultr" - cloudProviderSpec: - apiKey: << VULTR_API_KEY >> - region: blr - plan: 'vhf-8c-32gb' - osId: 127 - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/verify.go b/test/e2e/provisioning/verify.go index 5fdd7813c..c1a433d8e 100644 --- a/test/e2e/provisioning/verify.go +++ b/test/e2e/provisioning/verify.go @@ -26,7 +26,6 @@ import ( clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" machinecontroller "github.com/kubermatic/machine-controller/pkg/controller/machine" evictiontypes "github.com/kubermatic/machine-controller/pkg/node/eviction/types" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -149,7 +148,7 @@ func createAndAssure(ctx context.Context, machineDeployment *clusterv1alpha1.Mac klog.Infof("Creating a new %q MachineDeployment", machineDeployment.Name) - // Some cloud provider API's are slow (e.g. hetzner), and it can happen that our webhook + // Some cloud provider API's are slow (e.g. openstack), and it can happen that our webhook // needs longer to validate a MachineDeployment than the kube-apiserver is willing to wait. // In real world scenarios this is not that critical, but for tests we need to pay closer // attention and retry the creation a few times. @@ -283,15 +282,6 @@ func assureNodeForMachineDeployment(machineDeployment *clusterv1alpha1.MachineDe } for _, machine := range machines { - // Azure doesn't seem to easily expose the private IP address, there is only a PublicIPAddressClient in the sdk - providerConfig, err := providerconfigtypes.GetConfig(machine.Spec.ProviderSpec) - if err != nil { - return fmt.Errorf("failed to get provider config: %w", err) - } - if providerConfig.CloudProvider == providerconfigtypes.CloudProviderAzure { - continue - } - if len(machine.Status.Addresses) == 0 { return fmt.Errorf("expected to find a node for MachineDeployment %q but Machine %q has no address yet, indicating instance creation at the provider failed", machineDeployment.Name, machine.Name) } From 19bbe89e0aa8890a10596cc4f6526c0ee0214a64 Mon Sep 17 00:00:00 2001 From: Phillip Stagnet Date: Mon, 27 May 2024 17:59:24 +0200 Subject: [PATCH 2/2] Update go.mod and go.sum to remove cloud providers --- go.mod | 66 -------- go.sum | 527 +-------------------------------------------------------- 2 files changed, 2 insertions(+), 591 deletions(-) diff --git a/go.mod b/go.mod index 23feb6dbd..9ebbd1479 100644 --- a/go.mod +++ b/go.mod @@ -5,16 +5,9 @@ go 1.21 toolchain go1.21.5 require ( - cloud.google.com/go/logging v1.9.0 - cloud.google.com/go/monitoring v1.17.0 - github.com/Azure/azure-sdk-for-go v65.0.0+incompatible - github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 - github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/BurntSushi/toml v1.3.2 github.com/Masterminds/semver/v3 v3.2.1 github.com/Masterminds/sprig/v3 v3.2.3 - github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240104224209-592a2a64f555 - github.com/aliyun/alibaba-cloud-sdk-go v1.62.654 github.com/aws/aws-sdk-go-v2 v1.24.1 github.com/aws/aws-sdk-go-v2/config v1.26.3 github.com/aws/aws-sdk-go-v2/credentials v1.16.14 @@ -22,7 +15,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 github.com/aws/smithy-go v1.19.0 github.com/davecgh/go-spew v1.1.1 - github.com/digitalocean/godo v1.107.0 github.com/flatcar/container-linux-config-transpiler v0.9.4 github.com/go-logr/logr v1.4.1 github.com/go-logr/zapr v1.3.0 @@ -31,28 +23,18 @@ require ( github.com/gophercloud/gophercloud v1.8.0 github.com/hashicorp/go-multierror v1.1.1 github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb - github.com/hetznercloud/hcloud-go v1.39.0 - github.com/linode/linodego v1.26.0 - github.com/nutanix-cloud-native/prism-go-client v0.3.4 - github.com/packethost/packngo v0.31.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pborman/uuid v1.2.1 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 github.com/prometheus/client_golang v1.18.0 - github.com/scaleway/scaleway-sdk-go v1.0.0-beta.22 github.com/sethvargo/go-password v0.2.0 github.com/spf13/pflag v1.0.5 github.com/tinkerbell/tink v0.8.0 - github.com/vmware/go-vcloud-director/v2 v2.21.0 - github.com/vmware/govmomi v0.34.2 - github.com/vultr/govultr/v3 v3.6.0 - go.anx.io/go-anxcloud v0.6.2 go.uber.org/zap v1.26.0 golang.org/x/crypto v0.17.0 golang.org/x/oauth2 v0.15.0 gomodules.xyz/jsonpatch/v2 v2.4.0 - google.golang.org/api v0.155.0 google.golang.org/grpc v1.60.1 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.1 @@ -60,35 +42,17 @@ require ( k8s.io/apiextensions-apiserver v0.29.1 k8s.io/apimachinery v0.29.1 k8s.io/client-go v0.29.1 - k8s.io/cloud-provider v0.29.1 k8s.io/klog v1.0.0 k8s.io/kubelet v0.29.1 k8s.io/utils v0.0.0-20240102154912-e7106e64919e - kubevirt.io/api v1.1.1 - kubevirt.io/containerized-data-importer-api v1.58.0 sigs.k8s.io/controller-runtime v0.17.0 sigs.k8s.io/yaml v1.4.0 ) require ( - cloud.google.com/go v0.111.0 // indirect - cloud.google.com/go/compute v1.23.3 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/longrunning v0.5.4 // indirect - github.com/Azure/go-autorest v14.2.0+incompatible // indirect - github.com/Azure/go-autorest/autorest v0.11.29 // indirect - github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect - github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect - github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect - github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect - github.com/Azure/go-autorest/logger v0.2.1 // indirect - github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect - github.com/PaesslerAG/gval v1.2.2 // indirect - github.com/PaesslerAG/jsonpath v0.1.1 // indirect github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 // indirect github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect - github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect @@ -102,62 +66,39 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect - github.com/dimchansky/utfbom v1.1.1 // indirect github.com/distribution/reference v0.5.0 // indirect github.com/docker/distribution v2.8.3+incompatible // indirect github.com/emicklei/go-restful/v3 v3.11.1 // indirect github.com/evanphx/json-patch v5.7.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.8.0 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect github.com/flatcar/ignition v0.36.2 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.20.2 // indirect github.com/go-openapi/jsonreference v0.20.4 // indirect github.com/go-openapi/swag v0.22.7 // indirect - github.com/go-resty/resty/v2 v2.11.0 // indirect - github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect - github.com/google/s2a-go v0.1.7 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-retryablehttp v0.7.5 // indirect - github.com/hashicorp/go-version v1.6.0 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b // indirect - github.com/kr/pretty v0.3.1 // indirect - github.com/kr/text v0.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/onsi/ginkgo/v2 v2.14.0 // indirect - github.com/onsi/gomega v1.30.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/openshift/api v0.0.0-20240104110125-c7a2d3b41e1f // indirect - github.com/openshift/custom-resource-status v1.1.2 // indirect - github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect github.com/packethost/pkg v0.0.0-20230710142318-f8a288cd3046 // indirect - github.com/peterhellberg/link v1.2.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect @@ -166,23 +107,18 @@ require ( github.com/spf13/cast v1.6.0 // indirect github.com/spf13/cobra v1.8.0 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect - go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect go.opentelemetry.io/otel v1.21.0 // indirect go.opentelemetry.io/otel/metric v1.21.0 // indirect go.opentelemetry.io/otel/trace v1.21.0 // indirect - go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect golang.org/x/net v0.19.0 // indirect - golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/term v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.16.1 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 // indirect @@ -190,13 +126,11 @@ require ( google.golang.org/protobuf v1.32.0 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/component-base v0.29.1 // indirect k8s.io/klog/v2 v2.110.1 // indirect k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910 // indirect - kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/go.sum b/go.sum index 004b2e405..4261540ee 100644 --- a/go.sum +++ b/go.sum @@ -13,13 +13,7 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go v0.111.0 h1:YHLKNupSD1KqjDbQ3+LVdQ81h/UJbJyZG203cEfnQgM= -cloud.google.com/go v0.111.0/go.mod h1:0mibmpKP1TyOOFYQY5izo0LnT+ecvOQ0Sg3OdmMiNRU= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -32,14 +26,6 @@ cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGB cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= -cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= -cloud.google.com/go/logging v1.9.0 h1:iEIOXFO9EmSiTjDmfpbRjOxECO7R8C7b8IXUGOj7xZw= -cloud.google.com/go/logging v1.9.0/go.mod h1:1Io0vnZv4onoUnsVUQY3HZ3Igb1nBchky0A0y7BBBhE= -cloud.google.com/go/longrunning v0.5.4 h1:w8xEcbZodnA2BbW6sVirkkoC+1gP8wS57EUUgGS0GVg= -cloud.google.com/go/longrunning v0.5.4/go.mod h1:zqNVncI0BOP8ST6XQD1+VcvuShMmq7+xFSzOL++V0dI= -cloud.google.com/go/monitoring v1.17.0 h1:blrdvF0MkPPivSO041ihul7rFMhXdVp8Uq7F59DKXTU= -cloud.google.com/go/monitoring v1.17.0/go.mod h1:KwSsX5+8PnXv5NJnICZzW2R8pWTis8ypC4zmdRD63Tw= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -50,44 +36,10 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/99designs/gqlgen v0.15.1 h1:48bRXecwlCNTa/n2bMSp2rQsXNxwZ54QHbiULNf78ec= -github.com/99designs/gqlgen v0.15.1/go.mod h1:nbeSjFkqphIqpZsYe1ULVz0yfH8hjpJdJIQoX/e0G2I= -github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9EbjRYTySD/t8i1Ee/W5EGXw= -github.com/Azure/azure-sdk-for-go v65.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= -github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= -github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= -github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= -github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8= -github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c= -github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 h1:wkAZRgT/pn8HhFyzfe9UnqOjJYqlembgCTi72Bm/xKk= -github.com/Azure/go-autorest/autorest/azure/auth v0.5.12/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg= -github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg= -github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 h1:w77/uPk80ZET2F+AfQExZyEWtn+0Rk/uw17m9fv5Ajc= -github.com/Azure/go-autorest/autorest/azure/cli v0.4.6/go.mod h1:piCfgPho7BiIDdEQ1+g4VmKyD5y+p/XtSNqE6Hc4QD0= -github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= -github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= -github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= -github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= -github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= -github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= -github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= @@ -95,25 +47,10 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240104224209-592a2a64f555 h1:kWb9OISprBC94fTeagHWzz+TQOx5IrwQOY88JyEVNjc= -github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240104224209-592a2a64f555/go.mod h1:dvAwZi1Aol7eu6BENzHtl8ztGBkacB9t/fJj+fYk+Xg= -github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I= -github.com/PaesslerAG/gval v1.2.2 h1:Y7iBzhgE09IGTt5QgGQ2IdaYYYOU134YGHBThD+wm9E= -github.com/PaesslerAG/gval v1.2.2/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac= -github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8= -github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEsylIk= -github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= -github.com/agnivade/levenshtein v1.1.0 h1:n6qGwyHG61v3ABce1rPVZklEYRT8NFpCMrpZdBUbYGM= -github.com/agnivade/levenshtein v1.1.0/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= github.com/ajeddeloh/go-json v0.0.0-20160803184958-73d058cf8437/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 h1:4SPQljF/GJ8Q+QlCWMWxRBepub4DresnOm4eI2ebFGc= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= -github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -122,18 +59,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4L0zgAOR8lTQK9VlyBVVd7G4omaOQs= github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/aliyun/alibaba-cloud-sdk-go v1.62.654 h1:UpBbuyd0eqDkIfiuRmBGqdjXWd4Q7YwD9entykxwlnI= -github.com/aliyun/alibaba-cloud-sdk-go v1.62.654/go.mod h1:CJJYa1ZMxjlN/NbXEwmejEnBkhi0DV+Yb3B2lxf+74o= -github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= -github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= -github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.8.39/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= -github.com/aws/aws-sdk-go v1.42.23/go.mod h1:gyRszuZ/icHmHAVE4gc/r+cfCmhA1AD+vqfWbgI+eHs= github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU= github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= github.com/aws/aws-sdk-go-v2/config v1.26.3 h1:dKuc2jdp10y13dEEvPqWxqLoc0vF3Z9FC45MvuQSxOA= @@ -169,8 +96,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/bnkamalesh/webgo/v4 v4.1.11/go.mod h1:taIAonQTzao8G5rnB22WgKmQuIOWHpQ0n/YLAidBXlM= -github.com/bnkamalesh/webgo/v6 v6.2.2/go.mod h1:2Y+dEdTp1xC/ra+3PAVZV6hh4sCI+iPK7mcHt+t9bfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -182,13 +107,9 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/go-semver v0.1.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -198,74 +119,38 @@ github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03V github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creasty/defaults v1.5.2 h1:/VfB6uxpyp6h0fr7SPp7n8WJBoV8jfxQXPCnkVSjyls= -github.com/creasty/defaults v1.5.2/go.mod h1:FPZ+Y0WNrbqOVw+c6av63eyHUAl6pMHZwqLPvXUZGfY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= -github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= -github.com/digitalocean/godo v1.107.0 h1:P72IbmGFQvKOvyjVLyT59bmHxilA4E5hWi40rF4zNQc= -github.com/digitalocean/godo v1.107.0/go.mod h1:R6EmmWI8CT1+fCtjWY9UCB+L5uufuZH13wk3YhxycCs= -github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= -github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful/v3 v3.11.1 h1:S+9bSbua1z3FgCnV0KKOSSZ3mDthb5NyEPL5gEpCvyk= github.com/emicklei/go-restful/v3 v3.11.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= -github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flatcar/container-linux-config-transpiler v0.9.4 h1:yXQ0NB8PeNrKJPrZvbv5/DV63PNhTqt8vaf8YxmX/RA= github.com/flatcar/container-linux-config-transpiler v0.9.4/go.mod h1:LxanhPvXkWgHG9PrkT4rX/p7YhUPdDGGsUdkNpV3L5U= github.com/flatcar/ignition v0.36.2 h1:xGHgScUe0P4Fkprjqv7L2CE58emiQgP833OCCn9z2v4= github.com/flatcar/ignition v0.36.2/go.mod h1:uk1tpzLFRXus4RrvzgMI+IqmmB8a/RGFSBlI+tMTbbA= -github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= -github.com/go-chi/chi v1.5.4 h1:QHdzF2szwjqVV4wmByUnTcsbIg7UGaQ0tPF2t5GcAIs= -github.com/go-chi/chi v1.5.4/go.mod h1:uaf8YgoFazUOkPBG7fxPftUylNumIev9awIWOENIuEg= -github.com/go-chi/cors v1.2.0/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= -github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8= -github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -277,63 +162,30 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU= github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.22.7 h1:JWrc1uc/P9cSomxfnsFSVWoE1FW6bNbrVPmpQYpCcR8= github.com/go-openapi/swag v0.22.7/go.mod h1:Gl91UqO+btAM0plGGxHqJcQZ1ZTy6jbmridBTsDy8A0= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= -github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= -github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= -github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= -github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= -github.com/go-playground/validator/v10 v10.10.1 h1:uA0+amWMiglNZKZ9FJRKUAe9U3RX91eVn1JYXMWt7ig= -github.com/go-playground/validator/v10 v10.10.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= -github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8= -github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/goccy/go-json v0.9.4/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= -github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -347,7 +199,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -363,15 +214,11 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -381,24 +228,17 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= -github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -406,72 +246,42 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= -github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= -github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= -github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= -github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gophercloud/gophercloud v1.8.0 h1:TM3Jawprb2NrdOnvcHhWJalmKmAmOGgfZElM/3oBYCk= github.com/gophercloud/gophercloud v1.8.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= -github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= -github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= -github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= -github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= -github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb h1:tsEKRC3PU9rMw18w/uAptoijhgG4EvlA5kfJPtwrMDk= github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= -github.com/hetznercloud/hcloud-go v1.39.0 h1:RUlzI458nGnPR6dlcZlrsGXYC1hQlFbKdm8tVtEQQB0= -github.com/hetznercloud/hcloud-go v1.39.0/go.mod h1:mepQwR6va27S3UQthaEPGS86jtzSY9xWL1e9dyxXpgA= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -479,9 +289,7 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -491,75 +299,30 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= -github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= -github.com/keploy/go-sdk v0.4.3 h1:dCsmfANlZH94It+JKWx8/JEEC6dn8W7KIRRKRZwCPZQ= -github.com/keploy/go-sdk v0.4.3/go.mod h1:tn62gQ8a/AD7mY51DvQfhudiBPTlD+w3XtXemDcbON4= -github.com/kevinmbeaulieu/eq-go v1.0.0/go.mod h1:G3S8ajA56gKBZm4UB9AOyoOS37JO3roToPzKNM8dtdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/kolo/xmlrpc v0.0.0-20190717152603-07c4ee3fd181/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ= -github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJis0QP7YMxobob6zhzq6Yre00= -github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/labstack/echo/v4 v4.6.1/go.mod h1:RnjgMWNDB9g/HucVWhQYNQP9PvbYf6adqftqryo7s9k= -github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= -github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= -github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y= -github.com/lestrrat-go/blackmagic v1.0.0/go.mod h1:TNgH//0vYSs8VXDCfkZLgIrVTTXQELZffUV0tz3MtdQ= -github.com/lestrrat-go/httpcc v1.0.0/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++0Gf8MBnAvE= -github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc= -github.com/lestrrat-go/jwx v1.2.20/go.mod h1:tLE1XszaFgd7zaS5wHe4NxA+XVhu7xgdRvDpNyi3kNM= -github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/linode/linodego v1.26.0 h1:2tOZ3Wxn4YvGBRgZi3Vz6dab+L16XUntJ9sJxh3ZBio= -github.com/linode/linodego v1.26.0/go.mod h1:kD7Bf1piWg/AXb9TA0ThAVwzR+GPf6r2PvbTbVk7PMA= -github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matryer/moq v0.2.3/go.mod h1:9RtPYjTnH1bSBIkpvtHkFN7nbWAnO7oRpdJkEIn6UtE= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.2.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -567,49 +330,17 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nutanix-cloud-native/prism-go-client v0.3.4 h1:bHY3VPrHHYnbRtkpGaKK+2ZmvUjNVRC55CYZbXIfnOk= -github.com/nutanix-cloud-native/prism-go-client v0.3.4/go.mod h1:tTIH02E6o6AWSShr98QChoxuZl+jBhkXFixom9+fd1Y= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY= github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/openshift/api v0.0.0-20240104110125-c7a2d3b41e1f h1:3BMVfQpz1xe8MmJprp1+NL8hrpl9I04JVP9EczdCOqE= -github.com/openshift/api v0.0.0-20240104110125-c7a2d3b41e1f/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4= -github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= -github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= -github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU= -github.com/packethost/packngo v0.31.0 h1:LLH90ardhULWbagBIc3I3nl2uU75io0a7AwY6hyi0S4= -github.com/packethost/packngo v0.31.0/go.mod h1:Io6VJqzkiqmIEQbpOjeIw9v8q9PfcTEq8TEY/tMQsfw= github.com/packethost/pkg v0.0.0-20230710142318-f8a288cd3046 h1:zF+CUhv8LMpqTFFpECX6WF+yUWS2Bd1Nc1W+AczzqbY= github.com/packethost/pkg v0.0.0-20230710142318-f8a288cd3046/go.mod h1:W/xTaqgJ2kJCwayvm3BF3bOj9ku0F5DjjYnZaioxnOk= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= @@ -617,11 +348,7 @@ github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTK github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/peterhellberg/link v1.2.0 h1:UA5pg3Gp/E0F2WdX7GERiNrPQrM1K6CVJUUWfHa4t6c= -github.com/peterhellberg/link v1.2.0/go.mod h1:gYfAh+oJgQu2SrZHg5hROVRQe1ICoK0/HHJTcE0edxc= github.com/pin/tftp v2.1.0+incompatible/go.mod h1:xVpZOMCXTy+A5QMjEVN0Glwa1sUvaJhFXbr/aAxuxGY= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -653,32 +380,20 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= -github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rollbar/rollbar-go v1.4.2/go.mod h1:kLQ9gP3WCRGrvJmF0ueO3wK9xWocej8GRX98D8sa39w= github.com/rollbar/rollbar-go/errors v0.0.0-20210929193720-32947096267e/go.mod h1:Ie0xEc1Cyj+T4XMO8s0Vf7pMfvSAAy1sb4AYc8aJsao= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= -github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= -github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.22 h1:wJrcTdddKOI8TFxs8cemnhKP2EmKy3yfUKHj3ZdfzYo= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.22/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= -github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc7i/UHI= github.com/sethvargo/go-password v0.2.0/go.mod h1:Ym4Mr9JXLBycr02MFuVQ/0JHidNetSgbzutTr3zsYXE= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sigma/bdoor v0.0.0-20160202064022-babf2a4017b0/go.mod h1:WBu7REWbxC/s/J06jsk//d+9DOz9BbsmcIrimuGRFbs= github.com/sigma/vmw-guestinfo v0.0.0-20160204083807-95dd4126d6e8/go.mod h1:JrRFFC0veyh0cibh0DAhriSY7/gV3kDdNaVUOmfx01U= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -687,7 +402,6 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= @@ -695,91 +409,41 @@ github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tinkerbell/lint-install v0.0.0-20211012174934-5ee5ab01db76/go.mod h1:0h2KsALaQLNkoVeV+G+HjBWWCnp0COFYhJdRd5WCQPM= github.com/tinkerbell/tink v0.8.0 h1:qgl/rglpO5Rvq6UKZd29O6X9mDgZZYgf841+Y0IYWak= github.com/tinkerbell/tink v0.8.0/go.mod h1:bfAkSH7J/QQYIyqZRR6IQp8w78aac6l8Z2Lws5uXz6A= -github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= -github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= -github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= -github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= -github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.35.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= -github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= -github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= -github.com/vektah/gqlparser/v2 v2.2.0 h1:bAc3slekAAJW6sZTi07aGq0OrfaCjj4jxARAaC7g2EM= -github.com/vektah/gqlparser/v2 v2.2.0/go.mod h1:i3mQIGIrbK2PD1RrCeMTlVbkF2FJ6WkU1KJlJlC+3F4= github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= -github.com/vmware/go-vcloud-director/v2 v2.21.0 h1:zIONrJpM+Fj+rDyXmsRfMAn1sP5WAP87USL0T9GS4DY= -github.com/vmware/go-vcloud-director/v2 v2.21.0/go.mod h1:QPxGFgrUcSyzy9IlpwDE4UNT3tsOy2047tJOPEJ4nlw= -github.com/vmware/govmomi v0.34.2 h1:o6ydkTVITOkpQU6HAf6tP5GvHFCNJlNUNlMsvFK77X4= -github.com/vmware/govmomi v0.34.2/go.mod h1:qWWT6n9mdCr/T9vySsoUqcI04sSEj4CqHXxtk/Y+Los= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= -github.com/vultr/govultr/v3 v3.6.0 h1:WCXQwgdiZnGxG4CI+TTohE14V3jV6ikg/64fhDVdbIs= -github.com/vultr/govultr/v3 v3.6.0/go.mod h1:rt9v2x114jZmmLAE/h5N5jnxTmsK9ewwS2oQZ0UBQzM= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= -github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.anx.io/go-anxcloud v0.6.2 h1:4FV9xtjilRny/TEBeqsqOPxm1i9UPlPjHRDK86fhFjc= -go.anx.io/go-anxcloud v0.6.2/go.mod h1:TW0KcKa1hlYEwCQ2YAFec07xtfX60psI/dmjJqRdmjY= -go.keploy.io/server v0.1.8 h1:b50vAt1+WKMscYVP5Bm8gx/iSaR7mpHox8VpaxjrQ88= -go.keploy.io/server v0.1.8/go.mod h1:ZqhwTZOBb+dzx5t30Wt6eUGI6kO5QizvPg6coNPtbow= -go.mongodb.org/mongo-driver v1.8.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= -go.mongodb.org/mongo-driver v1.8.1/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= -go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= -go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= @@ -787,11 +451,7 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= -go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -800,9 +460,7 @@ go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95a go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= -go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= go4.org v0.0.0-20160314031811-03efcb870d84/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= @@ -814,27 +472,13 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= @@ -846,7 +490,6 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= -golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -859,7 +502,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -869,15 +511,10 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -891,7 +528,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -902,35 +538,19 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210913180222-943fd674d43e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -938,14 +558,7 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -960,15 +573,10 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -977,17 +585,11 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -998,55 +600,29 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211015200801-69063c4bb744/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1054,30 +630,21 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -1086,14 +653,12 @@ golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1113,30 +678,17 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200815165600-90abf76919f3/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= -golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1145,10 +697,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1165,20 +713,12 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.155.0 h1:vBmGhCYs0djJttDNynWo44zosHlPvHmA0XiN2zP2DtA= -google.golang.org/api v0.155.0/go.mod h1:GI5qK5f40kCpHfPn6+YzGAByIKWv8ujFnmoWm7Igduk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -1213,17 +753,6 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917 h1:nz5NESFLZbJGPFxDT/HCn+V1mZ8JGNoY4nUpmW/Y2eg= google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917/go.mod h1:pZqR+glSb11aJ+JQcczCvgf47+duRuzNSKqE8YAQnV0= @@ -1243,16 +772,10 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/grpc/examples v0.0.0-20210728214646-ad0a2a847cdf/go.mod h1:bF8wuZSAZTcbF7ZPKrDI/qY52toTP/yxLpRRY4Eu9Js= @@ -1269,7 +792,6 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= @@ -1278,19 +800,13 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1303,13 +819,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1317,71 +829,36 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= -k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= k8s.io/api v0.29.1 h1:DAjwWX/9YT7NQD4INu49ROJuZAAAP/Ijki48GUPzxqw= k8s.io/api v0.29.1/go.mod h1:7Kl10vBRUXhnQQI8YR/R327zXC8eJ7887/+Ybta+RoQ= k8s.io/apiextensions-apiserver v0.29.1 h1:S9xOtyk9M3Sk1tIpQMu9wXHm5O2MX6Y1kIpPMimZBZw= k8s.io/apiextensions-apiserver v0.29.1/go.mod h1:zZECpujY5yTW58co8V2EQR4BD6A9pktVgHhvc0uLfeU= -k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= -k8s.io/apimachinery v0.24.3/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= k8s.io/apimachinery v0.29.1 h1:KY4/E6km/wLBguvCZv8cKTeOwwOBqFNjwJIdMkMbbRc= k8s.io/apimachinery v0.29.1/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= -k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30= k8s.io/client-go v0.29.1 h1:19B/+2NGEwnFLzt0uB5kNJnfTsbV8w6TgQRz9l7ti7A= k8s.io/client-go v0.29.1/go.mod h1:TDG/psL9hdet0TI9mGyHJSgRkW3H9JZk2dNEUS7bRks= -k8s.io/cloud-provider v0.29.1 h1:bDLpOSpysWrtU2PCkvyP2sUTwRBa6MGCmxt68CRRW/8= -k8s.io/cloud-provider v0.29.1/go.mod h1:u50Drm6AbuoKpsVbAstNiFHGgbSVHuJV4TWN5imdM2w= -k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= k8s.io/component-base v0.29.1 h1:MUimqJPCRnnHsskTTjKD+IC1EHBbRCVyi37IoFBrkYw= k8s.io/component-base v0.29.1/go.mod h1:fP9GFjxYrLERq1GcWWZAE3bqbNcDKDytn2srWuHTtKc= -k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.10.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= -k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910 h1:1Rp/XEKP5uxPs6QrsngEHAxBjaAR78iJRiJq5Fi7LSU= k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910/go.mod h1:Pa1PvrP7ACSkuX6I7KYomY6cmMA0Tx86waBhDUgoKPw= k8s.io/kubelet v0.29.1 h1:cso8Dk8dymkj8q+EvW/aCbIYU2aOkH27gho48tYza/8= k8s.io/kubelet v0.29.1/go.mod h1:hTl/naFcCVG1Ku17fMgj/krbheBwBkf3gnFhaboMx7E= -k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -kubevirt.io/api v1.1.1 h1:vt5bOpACArNFIudx1bcE1VeejQdh5wCd7Oz/uFBIkH8= -kubevirt.io/api v1.1.1/go.mod h1:CJ4vZsaWhVN3jNbyc9y3lIZhw8nUHbWjap0xHABQiqc= -kubevirt.io/containerized-data-importer-api v1.58.0 h1:l6bH2SrCUi14QAi1Mv1vzcrqZI0XYzrV1KLK6hiC0QI= -kubevirt.io/containerized-data-importer-api v1.58.0/go.mod h1:Y/8ETgHS1GjO89bl682DPtQOYEU/1ctPFBz6Sjxm4DM= -kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 h1:fZYvD3/Vnitfkx6IJxjLAk8ugnZQ7CXVYcRfkSKmuZY= -kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s= sigs.k8s.io/controller-runtime v0.17.0/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= -sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= -sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=