From 6f1543ff69cea320322b9ef410f889c2803ea7d6 Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Thu, 1 Aug 2024 15:12:42 +0200 Subject: [PATCH] bootstrap: fix useExperimentalRetryJoin for kubernetes v1.31 This adds a new script for useExperimentalRetryJoin for kubernetes >= v1.31 due to the following changes: * control-plane update-status phase got removed * new phases got added for the ControlPlaneKubeletLocalMode feature gate --- .../kubeadm/internal/cloudinit/cloudinit.go | 24 ++- .../internal/cloudinit/cloudinit_test.go | 37 +++++ .../kubeadm-bootstrap-script-pre-k8s-1-31.sh | 137 ++++++++++++++++++ .../cloudinit/kubeadm-bootstrap-script.sh | 10 +- .../controllers/kubeadmconfig_controller.go | 3 + 5 files changed, 202 insertions(+), 9 deletions(-) create mode 100644 bootstrap/kubeadm/internal/cloudinit/kubeadm-bootstrap-script-pre-k8s-1-31.sh diff --git a/bootstrap/kubeadm/internal/cloudinit/cloudinit.go b/bootstrap/kubeadm/internal/cloudinit/cloudinit.go index c2f158f7c5c7..048ad085728e 100644 --- a/bootstrap/kubeadm/internal/cloudinit/cloudinit.go +++ b/bootstrap/kubeadm/internal/cloudinit/cloudinit.go @@ -22,9 +22,11 @@ import ( "fmt" "text/template" + "github.com/blang/semver/v4" "github.com/pkg/errors" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + "sigs.k8s.io/cluster-api/util/version" ) const ( @@ -56,6 +58,7 @@ type BaseUserData struct { KubeadmCommand string KubeadmVerbosity string SentinelFileCommand string + KubernetesVersion semver.Version } func (input *BaseUserData) prepare() error { @@ -64,7 +67,7 @@ func (input *BaseUserData) prepare() error { input.KubeadmCommand = fmt.Sprintf(standardJoinCommand, input.KubeadmVerbosity) if input.UseExperimentalRetry { input.KubeadmCommand = retriableJoinScriptName - joinScriptFile, err := generateBootstrapScript(input) + joinScriptFile, err := generateBootstrapScript(input, input.KubernetesVersion) if err != nil { return errors.Wrap(err, "failed to generate user data for machine joining control plane") } @@ -118,12 +121,23 @@ func generate(kind string, tpl string, data interface{}) ([]byte, error) { } var ( + //go:embed kubeadm-bootstrap-script-pre-k8s-1-31.sh + kubeadmBootstrapScriptPre1_31 string //go:embed kubeadm-bootstrap-script.sh kubeadmBootstrapScript string + + // kubernetesVersion1_31 is the version where kubeadm removed the update-status phase + // and introduced new phases with the ControlPlaneKubeletLocalMode feature gate. + kubernetesVersion1_31 = semver.MustParse("1.31.0") ) -func generateBootstrapScript(input interface{}) (*bootstrapv1.File, error) { - joinScript, err := generate("JoinScript", kubeadmBootstrapScript, input) +func generateBootstrapScript(input interface{}, parsedversion semver.Version) (*bootstrapv1.File, error) { + bootstrapScript := kubeadmBootstrapScript + if useKubeadmBootstrapScriptPre1_31(parsedversion) { + bootstrapScript = kubeadmBootstrapScriptPre1_31 + } + + joinScript, err := generate("JoinScript", bootstrapScript, input) if err != nil { return nil, errors.Wrap(err, "failed to bootstrap script for machine joins") } @@ -134,3 +148,7 @@ func generateBootstrapScript(input interface{}) (*bootstrapv1.File, error) { Content: string(joinScript), }, nil } + +func useKubeadmBootstrapScriptPre1_31(parsedversion semver.Version) bool { + return version.Compare(parsedversion, kubernetesVersion1_31, version.WithoutPreReleases()) < 0 +} diff --git a/bootstrap/kubeadm/internal/cloudinit/cloudinit_test.go b/bootstrap/kubeadm/internal/cloudinit/cloudinit_test.go index 626d2479888d..789e9b7e8bcd 100644 --- a/bootstrap/kubeadm/internal/cloudinit/cloudinit_test.go +++ b/bootstrap/kubeadm/internal/cloudinit/cloudinit_test.go @@ -19,6 +19,7 @@ package cloudinit import ( "testing" + "github.com/blang/semver/v4" . "github.com/onsi/gomega" "k8s.io/utils/ptr" @@ -278,3 +279,39 @@ func TestNewJoinControlPlaneExperimentalRetry(t *testing.T) { g.Expect(out).To(ContainSubstring(f)) } } + +func Test_useKubeadmBootstrapScriptPre1_31(t *testing.T) { + tests := []struct { + name string + parsedversion semver.Version + want bool + }{ + { + name: "true for version for v1.30", + parsedversion: semver.MustParse("1.30.99"), + want: true, + }, + { + name: "true for version for v1.28", + parsedversion: semver.MustParse("1.28.0"), + want: true, + }, + { + name: "false for v1.31.0", + parsedversion: semver.MustParse("1.31.0"), + want: false, + }, + { + name: "false for v1.31.0-beta.0", + parsedversion: semver.MustParse("1.31.0-beta.0"), + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := useKubeadmBootstrapScriptPre1_31(tt.parsedversion); got != tt.want { + t.Errorf("useKubeadmBootstrapScriptPre1_31() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/bootstrap/kubeadm/internal/cloudinit/kubeadm-bootstrap-script-pre-k8s-1-31.sh b/bootstrap/kubeadm/internal/cloudinit/kubeadm-bootstrap-script-pre-k8s-1-31.sh new file mode 100644 index 000000000000..5c78238ce0ac --- /dev/null +++ b/bootstrap/kubeadm/internal/cloudinit/kubeadm-bootstrap-script-pre-k8s-1-31.sh @@ -0,0 +1,137 @@ +#!/bin/bash +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Log an error and exit. +# Args: +# $1 Message to log with the error +# $2 The error code to return +log::error_exit() { + local message="${1}" + local code="${2}" + + log::error "${message}" + # {{ if .ControlPlane }} + log::info "Removing member from cluster status" + kubeadm reset -f update-cluster-status || true + log::info "Removing etcd member" + kubeadm reset -f remove-etcd-member || true + # {{ end }} + log::info "Resetting kubeadm" + kubeadm reset -f || true + log::error "cluster.x-k8s.io kubeadm bootstrap script $0 exiting with status ${code}" + exit "${code}" +} + +log::success_exit() { + log::info "cluster.x-k8s.io kubeadm bootstrap script $0 finished" + exit 0 +} + +# Log an error but keep going. +log::error() { + local message="${1}" + timestamp=$(date --iso-8601=seconds) + echo "!!! [${timestamp}] ${1}" >&2 + shift + for message; do + echo " ${message}" >&2 + done +} + +# Print a status line. Formatted to show up in a stream of output. +log::info() { + timestamp=$(date --iso-8601=seconds) + echo "+++ [${timestamp}] ${1}" + shift + for message; do + echo " ${message}" + done +} + +check_kubeadm_command() { + local command="${1}" + local code="${2}" + case ${code} in + "0") + log::info "kubeadm reported successful execution for ${command}" + ;; + "1") + log::error "kubeadm reported failed action(s) for ${command}" + ;; + "2") + log::error "kubeadm reported preflight check error during ${command}" + ;; + "3") + log::error_exit "kubeadm reported validation error for ${command}" "${code}" + ;; + *) + log::error "kubeadm reported unknown error ${code} for ${command}" + ;; + esac +} + +function retry-command() { + n=0 + local kubeadm_return + until [ $n -ge 5 ]; do + log::info "running '$*'" + # shellcheck disable=SC1083 + "$@" --config=/run/kubeadm/kubeadm-join-config.yaml {{.KubeadmVerbosity}} + kubeadm_return=$? + check_kubeadm_command "'$*'" "${kubeadm_return}" + if [ ${kubeadm_return} -eq 0 ]; then + break + fi + # We allow preflight errors to pass + if [ ${kubeadm_return} -eq 2 ]; then + break + fi + n=$((n + 1)) + sleep 15 + done + if [ ${kubeadm_return} -ne 0 ]; then + log::error_exit "too many errors, exiting" "${kubeadm_return}" + fi +} + +# {{ if .ControlPlane }} +function try-or-die-command() { + local kubeadm_return + log::info "running '$*'" + # shellcheck disable=SC1083 + "$@" --config=/run/kubeadm/kubeadm-join-config.yaml {{.KubeadmVerbosity}} + kubeadm_return=$? + check_kubeadm_command "'$*'" "${kubeadm_return}" + if [ ${kubeadm_return} -ne 0 ]; then + log::error_exit "fatal error, exiting" "${kubeadm_return}" + fi +} +# {{ end }} + +retry-command kubeadm join phase preflight --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests +# {{ if .ControlPlane }} +retry-command kubeadm join phase control-plane-prepare download-certs +retry-command kubeadm join phase control-plane-prepare certs +retry-command kubeadm join phase control-plane-prepare kubeconfig +retry-command kubeadm join phase control-plane-prepare control-plane +# {{ end }} +retry-command kubeadm join phase kubelet-start +# {{ if .ControlPlane }} +try-or-die-command kubeadm join phase control-plane-join etcd +retry-command kubeadm join phase control-plane-join update-status +retry-command kubeadm join phase control-plane-join mark-control-plane +# {{ end }} + +log::success_exit diff --git a/bootstrap/kubeadm/internal/cloudinit/kubeadm-bootstrap-script.sh b/bootstrap/kubeadm/internal/cloudinit/kubeadm-bootstrap-script.sh index 5c78238ce0ac..f338f3a88746 100644 --- a/bootstrap/kubeadm/internal/cloudinit/kubeadm-bootstrap-script.sh +++ b/bootstrap/kubeadm/internal/cloudinit/kubeadm-bootstrap-script.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020 The Kubernetes Authors. +# Copyright 2024 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -128,10 +128,8 @@ retry-command kubeadm join phase control-plane-prepare kubeconfig retry-command kubeadm join phase control-plane-prepare control-plane # {{ end }} retry-command kubeadm join phase kubelet-start -# {{ if .ControlPlane }} -try-or-die-command kubeadm join phase control-plane-join etcd -retry-command kubeadm join phase control-plane-join update-status -retry-command kubeadm join phase control-plane-join mark-control-plane -# {{ end }} + +# Run kubeadm join and skip all already executed phases. +try-or-die-command kubeadm join --skip-phases preflight,control-plane-prepare,kubelet-start log::success_exit diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go index 435bf03f644d..af698cad812f 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go @@ -535,6 +535,7 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex Mounts: scope.Config.Spec.Mounts, DiskSetup: scope.Config.Spec.DiskSetup, KubeadmVerbosity: verbosityFlag, + KubernetesVersion: parsedVersion, }, InitConfiguration: initdata, ClusterConfiguration: clusterdata, @@ -656,6 +657,7 @@ func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope) DiskSetup: scope.Config.Spec.DiskSetup, KubeadmVerbosity: verbosityFlag, UseExperimentalRetry: scope.Config.Spec.UseExperimentalRetryJoin, + KubernetesVersion: parsedVersion, }, JoinConfiguration: joinData, } @@ -774,6 +776,7 @@ func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *S DiskSetup: scope.Config.Spec.DiskSetup, KubeadmVerbosity: verbosityFlag, UseExperimentalRetry: scope.Config.Spec.UseExperimentalRetryJoin, + KubernetesVersion: parsedVersion, }, }