refine install step

Signed-off-by: pixiake <[email protected]>
stoneshi-yunify · Aug 28, 2020 · 459eedb · 459eedb
1 parent a49f8eb
commit 459eedb
Show file tree

Hide file tree

Showing 8 changed files with 90 additions and 54 deletions.
diff --git a/pkg/addons/install.go b/pkg/addons/install.go
@@ -55,7 +55,6 @@ func installAddon(mgr *manager.Manager, addon *kubekeyapi.Addon, kubeconfig stri
 
 	// install yaml
 	if len(addon.Sources.Yaml.Path) != 0 {
-		var yamlPaths []string
 		var settings = cli.New()
 		p := getter.All(settings)
 		for _, yaml := range addon.Sources.Yaml.Path {
@@ -66,16 +65,17 @@ func installAddon(mgr *manager.Manager, addon *kubekeyapi.Addon, kubeconfig stri
 				if err != nil {
 					return errors.Wrap(err, "Failed to look up current directory")
 				}
-				yamlPaths = append(yamlPaths, fp)
+				yamlPaths := []string{fp}
+				if err := manifests.InstallYaml(yamlPaths, addon.Namespace, "", mgr.Cluster.Kubernetes.Version); err != nil {
+					return err
+				}
 			} else {
-				yamlPaths = append(yamlPaths, yaml)
+				yamlPaths := []string{yaml}
+				if err := manifests.InstallYaml(yamlPaths, addon.Namespace, "", mgr.Cluster.Kubernetes.Version); err != nil {
+					return err
+				}
 			}
 		}
-
-		if err := manifests.InstallYaml(yamlPaths, addon.Namespace, ""); err != nil {
-			return err
-		}
-
 	}
 
 	return nil

diff --git a/pkg/addons/manifests/install.go b/pkg/addons/manifests/install.go
@@ -19,12 +19,14 @@ package manifests
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/serializer/yaml"
 	"k8s.io/apimachinery/pkg/types"
+	versionutil "k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"k8s.io/cli-runtime/pkg/printers"
 	"k8s.io/cli-runtime/pkg/resource"
@@ -45,11 +47,10 @@ var (
 	defaultCacheDir = filepath.Join(homedir.HomeDir(), ".kube", "cache")
 )
 
-func InstallYaml(manifests []string, namespace, kubeconfig string) error {
+func InstallYaml(manifests []string, namespace, kubeconfig, version string) error {
 
 	configFlags := NewConfigFlags(kubeconfig, namespace)
-
-	o, err := CreateApplyOptions(configFlags, manifests)
+	o, err := CreateApplyOptions(configFlags, manifests, version)
 	if err != nil {
 		return err
 	}
@@ -61,14 +62,24 @@ func InstallYaml(manifests []string, namespace, kubeconfig string) error {
 	return nil
 }
 
-func CreateApplyOptions(configFlags *genericclioptions.ConfigFlags, manifests []string) (*apply.ApplyOptions, error) {
+func CreateApplyOptions(configFlags *genericclioptions.ConfigFlags, manifests []string, version string) (*apply.ApplyOptions, error) {
 	var err error
 	matchVersionKubeConfigFlags := NewMatchVersionFlags(configFlags)
 	f := cmdutil.NewFactory(matchVersionKubeConfigFlags)
 	ioStreams := genericclioptions.IOStreams{In: nil, Out: os.Stdout, ErrOut: os.Stderr}
 
 	o := apply.NewApplyOptions(ioStreams)
-	o.ServerSideApply = true
+
+	cmp, err := versionutil.MustParseSemantic(version).Compare("v1.16.0")
+	if err != nil {
+		return nil, errors.New(fmt.Sprintf("Failed to compare version: %v", err))
+	}
+	if cmp == 0 || cmp == 1 {
+		o.ServerSideApply = true
+	} else {
+		o.ServerSideApply = false
+	}
+
 	o.ForceConflicts = true
 	o.DryRunStrategy = cmdutil.DryRunNone
 
@@ -83,7 +94,7 @@ func CreateApplyOptions(configFlags *genericclioptions.ConfigFlags, manifests []
 	}
 
 	o.DryRunVerifier = resource.NewDryRunVerifier(o.DynamicClient, discoveryClient)
-	o.FieldManager = "kubectl-client-side-apply"
+	o.FieldManager = "client-side-apply"
 
 	o.ToPrinter = func(operation string) (printers.ResourcePrinter, error) {
 		o.PrintFlags.NamePrintFlags.Operation = operation

diff --git a/pkg/cluster/kubernetes/master.go b/pkg/cluster/kubernetes/master.go
@@ -119,8 +119,8 @@ func initKubernetesCluster(mgr *manager.Manager, node *kubekeyapi.HostCfg) error
 			}
 		}
 
-		if err3 := GetKubeConfig(mgr); err3 != nil {
-			return err3
+		if err := GetKubeConfig(mgr); err != nil {
+			return err
 		}
 		if err := removeMasterTaint(mgr, node); err != nil {
 			return err
@@ -315,14 +315,12 @@ func addWorker(mgr *manager.Manager) error {
 
 	createConfigDirCmd := "mkdir -p /root/.kube && mkdir -p $HOME/.kube"
 	chownKubeConfig := "chown $(id -u):$(id -g) $HOME/.kube/config"
-	_, err1 := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", createConfigDirCmd), 1, false)
-	if err1 != nil {
-		return errors.Wrap(errors.WithStack(err1), "Failed to create kube dir")
+	if _, err := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", createConfigDirCmd), 1, false); err != nil {
+		return errors.Wrap(errors.WithStack(err), "Failed to create kube dir")
 	}
 	syncKubeconfigCmd := fmt.Sprintf("echo %s | base64 -d > %s && echo %s | base64 -d > %s && %s", clusterStatus["kubeConfig"], "/root/.kube/config", clusterStatus["kubeConfig"], "$HOME/.kube/config", chownKubeConfig)
-	_, err2 := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", syncKubeconfigCmd), 1, false)
-	if err2 != nil {
-		return errors.Wrap(errors.WithStack(err2), "Failed to sync kube config")
+	if _, err := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", syncKubeconfigCmd), 1, false); err != nil {
+		return errors.Wrap(errors.WithStack(err), "Failed to sync kube config")
 	}
 	return nil
 }

diff --git a/pkg/deploy/kubesphere.go b/pkg/deploy/kubesphere.go
@@ -40,14 +40,14 @@ func DeployKubeSphere(version, repo, kubeconfig string) error {
 	switch version {
 	case "":
 		kubesphereConfig = kubesphere.V3_0_0
-		str, err := kubesphere.GenerateKubeSphereYaml(repo, "latest")
+		str, err := kubesphere.GenerateKubeSphereYaml(repo, "v3.0.0")
 		if err != nil {
 			return err
 		}
 		installerYaml = str
 	case "v3.0.0":
 		kubesphereConfig = kubesphere.V3_0_0
-		str, err := kubesphere.GenerateKubeSphereYaml(repo, "latest")
+		str, err := kubesphere.GenerateKubeSphereYaml(repo, "v3.0.0")
 		if err != nil {
 			return err
 		}

diff --git a/pkg/kubesphere/kubesphere.go b/pkg/kubesphere/kubesphere.go
@@ -117,7 +117,7 @@ EOF
 			return errors.Wrap(errors.WithStack(err), "Failed to generate kubesphere manifests")
 		}
 	case "v3.0.0":
-		kubesphereYaml, err := GenerateKubeSphereYaml(mgr.Cluster.Registry.PrivateRegistry, "latest")
+		kubesphereYaml, err := GenerateKubeSphereYaml(mgr.Cluster.Registry.PrivateRegistry, "v3.0.0")
 		if err != nil {
 			return err
 		}

diff --git a/pkg/plugins/dns/dns.go b/pkg/plugins/dns/dns.go
@@ -30,18 +30,13 @@ func OverrideCorednsService(mgr *manager.Manager) error {
 		return err
 	}
 	corednsSvcgBase64 := base64.StdEncoding.EncodeToString([]byte(corednsSvc))
-	_, err1 := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"echo %s | base64 -d > /etc/kubernetes/coredns-svc.yaml\"", corednsSvcgBase64), 1, false)
-	if err1 != nil {
-		return errors.Wrap(errors.WithStack(err1), "Failed to generate kubeadm config")
+	if _, err := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"echo %s | base64 -d > /etc/kubernetes/coredns-svc.yaml\"", corednsSvcgBase64), 1, false); err != nil {
+		return errors.Wrap(errors.WithStack(err), "Failed to generate kubeadm config")
 	}
 	deleteKubednsSvcCmd := "/usr/local/bin/kubectl delete -n kube-system svc kube-dns"
-	_, err2 := mgr.Runner.ExecuteCmd(deleteKubednsSvcCmd, 5, true)
-	if err2 != nil {
-		return errors.Wrap(errors.WithStack(err2), "Failed to delete kubeadm Kube-DNS service")
-	}
-	_, err3 := mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"/usr/local/bin/kubectl apply -f /etc/kubernetes/coredns-svc.yaml\"", 5, true)
-	if err3 != nil {
-		return errors.Wrap(errors.WithStack(err3), "Failed to create coredns service")
+	_, _ = mgr.Runner.ExecuteCmd(deleteKubednsSvcCmd, 1, true)
+	if _, err := mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"/usr/local/bin/kubectl apply -f /etc/kubernetes/coredns-svc.yaml\"", 2, true); err != nil {
+		return errors.Wrap(errors.WithStack(err), "Failed to create coredns service")
 	}
 	return nil
 }

diff --git a/pkg/upgrade/kubernetes.go b/pkg/upgrade/kubernetes.go
@@ -26,6 +26,7 @@ var (
 	currentVersionStr string
 	nextVersionStr    string
 	mu                sync.Mutex
+	kubeConfig        string
 )
 
 func GetCurrentVersions(mgr *manager.Manager) error {
@@ -151,6 +152,10 @@ func upgradeKubeMasters(mgr *manager.Manager, node *kubekeyapi.HostCfg) error {
 			}
 		}
 
+		if err := kubernetes.GetKubeConfig(mgr); err != nil {
+			return err
+		}
+
 		if _, err := mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"systemctl stop kubelet\"", 2, true); err != nil {
 			return err
 		}
@@ -162,25 +167,13 @@ func upgradeKubeMasters(mgr *manager.Manager, node *kubekeyapi.HostCfg) error {
 		if _, err := mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"systemctl daemon-reload && systemctl restart kubelet\"", 2, true); err != nil {
 			return err
 		}
-	}
 
-	patchCorednsCmd := `sudo -E /bin/sh -c "/usr/local/bin/kubectl patch deploy -n kube-system coredns -p \" 
-spec:
-    template:
-       spec:
-           volumes:
-           - name: config-volume
-             configMap:
-                 name: coredns
-                 items:
-                 - key: Corefile
-                   path: Corefile\""`
-
-	_, _ = mgr.Runner.ExecuteCmd(patchCorednsCmd, 2, false)
-	if mgr.Runner.Index == 0 {
-		if err := dns.CreateClusterDns(mgr); err != nil {
-			return err
+		kubeCfgBase64Cmd := "cat /etc/kubernetes/admin.conf | base64 --wrap=0"
+		output, err2 := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", kubeCfgBase64Cmd), 1, false)
+		if err2 != nil {
+			return errors.Wrap(errors.WithStack(err2), "Failed to get new kubeconfig")
 		}
+		kubeConfig = output
 	}
 
 	time.Sleep(30 * time.Second)
@@ -213,6 +206,16 @@ func upgradeKubeWorkers(mgr *manager.Manager, node *kubekeyapi.HostCfg) error {
 		}
 	}
 
+	createConfigDirCmd := "mkdir -p /root/.kube && mkdir -p $HOME/.kube"
+	chownKubeConfig := "chown $(id -u):$(id -g) $HOME/.kube/config"
+	if _, err := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", createConfigDirCmd), 1, false); err != nil {
+		return errors.Wrap(errors.WithStack(err), "Failed to create kube dir")
+	}
+	syncKubeconfigCmd := fmt.Sprintf("echo %s | base64 -d > %s && echo %s | base64 -d > %s && %s", kubeConfig, "/root/.kube/config", kubeConfig, "$HOME/.kube/config", chownKubeConfig)
+	if _, err := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", syncKubeconfigCmd), 1, false); err != nil {
+		return errors.Wrap(errors.WithStack(err), "Failed to sync kube config")
+	}
+
 	return nil
 }
 
@@ -276,6 +279,9 @@ Loop:
 				return err
 			}
 
+			if err := mgr.RunTaskOnMasterNodes(reconfigDns, false); err != nil {
+				return err
+			}
 			currentVersionStr = nextVersionStr
 		} else {
 			break Loop
@@ -299,3 +305,29 @@ func upgradeCluster(mgr *manager.Manager, node *kubekeyapi.HostCfg) error {
 
 	return nil
 }
+
+func reconfigDns(mgr *manager.Manager, _ *kubekeyapi.HostCfg) error {
+	if mgr.Runner.Index == 0 {
+		patchCorednsCmd := `sudo -E /bin/sh -c "/usr/local/bin/kubectl patch deploy -n kube-system coredns -p \" 
+spec:
+    template:
+       spec:
+           volumes:
+           - name: config-volume
+             configMap:
+                 name: coredns
+                 items:
+                 - key: Corefile
+                   path: Corefile\""`
+
+		_, _ = mgr.Runner.ExecuteCmd(patchCorednsCmd, 2, true)
+
+		if err := dns.OverrideCorednsService(mgr); err != nil {
+			return err
+		}
+		if err := dns.CreateClusterDns(mgr); err != nil {
+			return err
+		}
+	}
+	return nil
+}
diff --git a/pkg/upgrade/kubesphere.go b/pkg/upgrade/kubesphere.go
@@ -62,14 +62,14 @@ func KsToV3(version, repo, kubeconfig string) error {
 	switch version {
 	case "":
 		kubesphereConfig = configV3
-		str, err := kubesphere.GenerateKubeSphereYaml(repo, "latest")
+		str, err := kubesphere.GenerateKubeSphereYaml(repo, "v3.0.0")
 		if err != nil {
 			return err
 		}
 		installerYaml = str
 	case "v3.0.0":
 		kubesphereConfig = configV3
-		str, err := kubesphere.GenerateKubeSphereYaml(repo, "latest")
+		str, err := kubesphere.GenerateKubeSphereYaml(repo, "v3.0.0")
 		if err != nil {
 			return err
 		}