support multi container-runtime

Signed-off-by: pixiake <[email protected]>

apis/kubekey/v1alpha1/cluster_types.go

@@ -145,7 +145,6 @@ type HostGroups struct {
 	Master []HostCfg
 	Worker []HostCfg
 	K8s    []HostCfg
-	Client []HostCfg
 }
 
 type ControlPlaneEndpoint struct {
@@ -276,7 +275,7 @@ func (cfg *ClusterSpec) GroupHosts(logger *log.Logger) (*HostGroups, error) {
 	if len(workerGroup) != len(clusterHostsGroups.Worker) {
 		return nil, errors.New("Incorrect nodeName under roleGroups/work in the configuration file, Please check before installing.")
 	}
-	clusterHostsGroups.Client = append(clusterHostsGroups.Client, clusterHostsGroups.Master[0])
+
 	return &clusterHostsGroups, nil
 }
 

apis/kubekey/v1alpha1/default.go

@@ -33,7 +33,7 @@ const (
 	DefaultNetworkPlugin       = "calico"
 	DefaultPodsCIDR            = "10.233.64.0/18"
 	DefaultServiceCIDR         = "10.233.0.0/18"
-	DefaultKubeImageRepo       = "kubesphere"
+	DefaultKubeImageNamespace  = "kubesphere"
 	DefaultClusterName         = "cluster.local"
 	DefaultArch                = "amd64"
 	DefaultEtcdVersion         = "v3.4.13"
@@ -52,6 +52,9 @@ const (
 	DefaultVethMTU             = 1440
 	DefaultBackendMode         = "vxlan"
 	DefaultProxyMode           = "ipvs"
+	DefaultCrioEndpoint        = "/var/run/crio/crio.sock"
+	DefaultContainerdEndpoint  = "/run/containerd/containerd.sock"
+	DefaultIsulaEndpoint       = "/var/run/isulad.sock"
 	Etcd                       = "etcd"
 	Master                     = "master"
 	Worker                     = "worker"
@@ -83,9 +86,7 @@ func (cfg *ClusterSpec) SetDefaultClusterSpec(incluster bool, logger *log.Logger
 	clusterCfg.Registry = cfg.Registry
 	clusterCfg.Addons = cfg.Addons
 	clusterCfg.KubeSphere = cfg.KubeSphere
-	if cfg.Kubernetes.ImageRepo == "" {
-		clusterCfg.Kubernetes.ImageRepo = DefaultKubeImageRepo
-	}
+
 	if cfg.Kubernetes.ClusterName == "" {
 		clusterCfg.Kubernetes.ClusterName = DefaultClusterName
 	}
@@ -218,9 +219,6 @@ func SetDefaultClusterCfg(cfg *ClusterSpec) Kubernetes {
 	if cfg.Kubernetes.Version == "" {
 		cfg.Kubernetes.Version = DefaultKubeVersion
 	}
-	if cfg.Kubernetes.ImageRepo == "" {
-		cfg.Kubernetes.ImageRepo = DefaultKubeImageRepo
-	}
 	if cfg.Kubernetes.ClusterName == "" {
 		cfg.Kubernetes.ClusterName = DefaultClusterName
 	}
@@ -236,7 +234,6 @@ func SetDefaultClusterCfg(cfg *ClusterSpec) Kubernetes {
 	if cfg.Kubernetes.EtcdBackupScriptDir == "" {
 		cfg.Kubernetes.EtcdBackupScriptDir = DefaultEtcdBackupScriptDir
 	}
-
 	defaultClusterCfg := cfg.Kubernetes
 
 	return defaultClusterCfg

apis/kubekey/v1alpha1/kubernetes_types.go

@@ -17,16 +17,17 @@ limitations under the License.
 package v1alpha1
 
 type Kubernetes struct {
-	Version                string   `yaml:"version" json:"version,omitempty"`
-	ImageRepo              string   `yaml:"imageRepo" json:"imageRepo,omitempty"`
-	ClusterName            string   `yaml:"clusterName" json:"clusterName,omitempty"`
-	MasqueradeAll          bool     `yaml:"masqueradeAll" json:"masqueradeAll,omitempty"`
-	MaxPods                int      `yaml:"maxPods" json:"maxPods,omitempty"`
-	NodeCidrMaskSize       int      `yaml:"nodeCidrMaskSize" json:"nodeCidrMaskSize,omitempty"`
-	ApiserverCertExtraSans []string `yaml:"apiserverCertExtraSans" json:"apiserverCertExtraSans,omitempty"`
-	ProxyMode              string   `yaml:"proxyMode" json:"proxyMode,omitempty"`
-	EtcdBackupDir          string   `yaml:"etcdBackupDir" json:"etcdBackupDir,omitempty"`
-	EtcdBackupPeriod       int      `yaml:"etcdBackupPeriod" json:"etcdBackupPeriod,omitempty"`
-	KeepBackupNumber       int      `yaml:"keepBackupNumber" json:"keepBackupNumber,omitempty"`
-	EtcdBackupScriptDir    string   `yaml:"etcdBackupScript" json:"etcdBackupScript,omitempty"`
+	Version                  string   `yaml:"version" json:"version,omitempty"`
+	ClusterName              string   `yaml:"clusterName" json:"clusterName,omitempty"`
+	MasqueradeAll            bool     `yaml:"masqueradeAll" json:"masqueradeAll,omitempty"`
+	MaxPods                  int      `yaml:"maxPods" json:"maxPods,omitempty"`
+	NodeCidrMaskSize         int      `yaml:"nodeCidrMaskSize" json:"nodeCidrMaskSize,omitempty"`
+	ApiserverCertExtraSans   []string `yaml:"apiserverCertExtraSans" json:"apiserverCertExtraSans,omitempty"`
+	ProxyMode                string   `yaml:"proxyMode" json:"proxyMode,omitempty"`
+	EtcdBackupDir            string   `yaml:"etcdBackupDir" json:"etcdBackupDir,omitempty"`
+	EtcdBackupPeriod         int      `yaml:"etcdBackupPeriod" json:"etcdBackupPeriod,omitempty"`
+	KeepBackupNumber         int      `yaml:"keepBackupNumber" json:"keepBackupNumber,omitempty"`
+	EtcdBackupScriptDir      string   `yaml:"etcdBackupScript" json:"etcdBackupScript,omitempty"`
+	ContainerManager         string   `yaml:"containerManager" json:"containerManager,omitempty"`
+	ContainerRuntimeEndpoint string   `yaml:"containerRuntimeEndpoint" json:"containerRuntimeEndpoint,omitempty"`
 }

pkg/cluster/etcd/etcd.go

@@ -26,6 +26,8 @@ import (
 	"github.com/kubesphere/kubekey/pkg/util/manager"
 	"github.com/pkg/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"os"
+	"path/filepath"
 	"strings"
 	"time"
 )
@@ -151,8 +153,15 @@ func GenerateEtcdService(mgr *manager.Manager) error {
 	return mgr.RunTaskOnEtcdNodes(generateEtcdService, true)
 }
 
-func generateEtcdService(mgr *manager.Manager, _ *kubekeyapiv1alpha1.HostCfg) error {
-	etcdService, err := tmpl.GenerateEtcdService(mgr.Runner.Index)
+// Install etcd and etcdctl.
+// Starting etcd using binary, when container manager is not docker.
+// Starting etcd using docker container, when container manager is docker.
+func generateEtcdService(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) error {
+	if err := installEtcdBinaries(mgr, node); err != nil {
+		return err
+	}
+
+	etcdService, err := tmpl.GenerateEtcdService(mgr.Runner.Index, mgr.EtcdContainer)
 	if err != nil {
 		return err
 	}
@@ -162,25 +171,21 @@ func generateEtcdService(mgr *manager.Manager, _ *kubekeyapiv1alpha1.HostCfg) er
 		return errors.Wrap(errors.WithStack(err1), "Failed to generate etcd service")
 	}
 
-	etcdBin, err := tmpl.GenerateEtcdBinary(mgr, mgr.Runner.Index)
-	if err != nil {
-		return err
-	}
-	etcdBinBase64 := base64.StdEncoding.EncodeToString([]byte(etcdBin))
-	_, err3 := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"echo %s | base64 -d > /usr/local/bin/etcd && chmod +x /usr/local/bin/etcd\"", etcdBinBase64), 1, false)
-	if err3 != nil {
-		return errors.Wrap(errors.WithStack(err3), "Failed to generate etcd bin")
-	}
-
-	getEtcdCtlCmd := fmt.Sprintf("docker run --rm -v /usr/local/bin:/systembindir %s /bin/cp -f /usr/local/bin/etcdctl /systembindir/etcdctl", preinstall.GetImage(mgr, "etcd").ImageName())
-	_, err4 := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", getEtcdCtlCmd), 2, false)
-	if err4 != nil {
-		return errors.Wrap(errors.WithStack(err4), "Failed to get etcdctl")
+	if mgr.EtcdContainer {
+		etcdBin, err := tmpl.GenerateEtcdBinary(mgr, mgr.Runner.Index)
+		if err != nil {
+			return err
+		}
+		etcdBinBase64 := base64.StdEncoding.EncodeToString([]byte(etcdBin))
+		_, err3 := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"echo %s | base64 -d > /usr/local/bin/etcd && chmod +x /usr/local/bin/etcd\"", etcdBinBase64), 1, false)
+		if err3 != nil {
+			return errors.Wrap(errors.WithStack(err3), "Failed to generate etcd bin")
+		}
 	}
 
-	if err := restartEtcd(mgr); err != nil {
-		return err
-	}
+	//if err := restartEtcd(mgr); err != nil {
+	//	return err
+	//}
 
 	var addrList []string
 	for _, host := range mgr.EtcdNodes {
@@ -192,12 +197,45 @@ func generateEtcdService(mgr *manager.Manager, _ *kubekeyapiv1alpha1.HostCfg) er
 	return nil
 }
 
+func installEtcdBinaries(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) error {
+	if !mgr.EtcdContainer {
+		tmpDir := "/tmp/kubekey"
+		_, err := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"if [ -d %s ]; then rm -rf %s ;fi\" && mkdir -p %s", tmpDir, tmpDir, tmpDir), 1, false)
+		if err != nil {
+			return errors.Wrap(errors.WithStack(err), "Failed to create tmp dir")
+		}
+
+		currentDir, err1 := filepath.Abs(filepath.Dir(os.Args[0]))
+		if err1 != nil {
+			return errors.Wrap(err1, "Failed to get current dir")
+		}
+		etcdFile := fmt.Sprintf("etcd-%s-linux-%s", kubekeyapiv1alpha1.DefaultEtcdVersion, node.Arch)
+		filesDir := fmt.Sprintf("%s/%s/%s/%s", currentDir, kubekeyapiv1alpha1.DefaultPreDir, mgr.Cluster.Kubernetes.Version, node.Arch)
+		if err := mgr.Runner.ScpFile(fmt.Sprintf("%s/%s.tar.gz", filesDir, etcdFile), fmt.Sprintf("%s/%s.tar.gz", "/tmp/kubekey", etcdFile)); err != nil {
+			return errors.Wrap(errors.WithStack(err), fmt.Sprintf("Failed to sync etcd tar.gz"))
+		}
+
+		installCmd := fmt.Sprintf("tar -zxf %s/%s.tar.gz && cp -f %s/etcd* /usr/local/bin/ && chmod +x /usr/local/bin/etcd* && rm -rf %s", "/tmp/kubekey", etcdFile, etcdFile, etcdFile)
+		if _, err := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", installCmd), 2, false); err != nil {
+			return errors.Wrap(errors.WithStack(err), fmt.Sprintf("Failed to install etcd binaries."))
+		}
+	} else {
+		getEtcdCtlCmd := fmt.Sprintf("docker run --rm -v /usr/local/bin:/systembindir %s /bin/cp -f /usr/local/bin/etcdctl /systembindir/etcdctl", preinstall.GetImage(mgr, "etcd").ImageName())
+		_, err := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", getEtcdCtlCmd), 2, false)
+		if err != nil {
+			return errors.Wrap(errors.WithStack(err), "Failed to get etcdctl")
+		}
+	}
+	return nil
+}
+
 func SetupEtcdCluster(mgr *manager.Manager) error {
 	mgr.Logger.Infoln("Starting etcd cluster")
 
 	return mgr.RunTaskOnEtcdNodes(setupEtcdCluster, false)
 }
 
+// Configuring and starting etcd cluster.
 func setupEtcdCluster(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) error {
 	var localPeerAddresses []string
 	output, _ := mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"[ -f /etc/etcd.env ] && echo 'Configuration file already exists' || echo 'Configuration file will be created'\"", 0, true)
@@ -290,6 +328,7 @@ func BackupEtcd(mgr *manager.Manager) error {
 	return nil
 }
 
+// Create etcd backup scripts.
 func backupEtcd(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) error {
 	_, err := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"mkdir -p %s\"", mgr.Cluster.Kubernetes.EtcdBackupScriptDir), 0, false)
 	if err != nil {

pkg/cluster/etcd/tmpl/etcd.go

@@ -30,20 +30,34 @@ import (
 var (
 	EtcdServiceTempl = template.Must(template.New("EtcdService").Parse(
 		dedent.Dedent(`[Unit]
+{{- if .EtcdContainer }}
 Description=etcd docker wrapper
 Wants=docker.socket
 After=docker.service
+{{- else }}
+Description=etcd
+After=network.target
+{{- end }}
 
 [Service]
 User=root
+{{- if .EtcdContainer }}
 PermissionsStartOnly=true
 EnvironmentFile=-/etc/etcd.env
 ExecStart=/usr/local/bin/etcd
 ExecStartPre=-/usr/bin/docker rm -f {{ .Name }}
 ExecStop=/usr/bin/docker stop {{ .Name }}
-Restart=always
 RestartSec=15s
 TimeoutStartSec=30s
+{{- else }}
+Type=notify
+EnvironmentFile=/etc/etcd.env
+ExecStart=/usr/local/bin/etcd
+NotifyAccess=all
+RestartSec=10s
+LimitNOFILE=40000
+{{- end }}
+Restart=always
 
 [Install]
 WantedBy=multi-user.target
@@ -65,7 +79,7 @@ ETCD_NAME={{ .Name }}
 ETCD_PROXY=off
 ETCD_ENABLE_V2=true
 ETCD_INITIAL_CLUSTER={{ .peerAddresses }}
-ETCD_AUTO_COMPACTION_RETENTION=8
+ETCD_AUTO_COEtcdContainerMPACTION_RETENTION=8
 ETCD_SNAPSHOT_COUNT=10000
 {{- if .UnsupportedArch }}
 ETCD_UNSUPPORTED_ARCH={{ .Arch }}
@@ -114,9 +128,10 @@ func GenerateEtcdBinary(mgr *manager.Manager, index int) (string, error) {
 	})
 }
 
-func GenerateEtcdService(index int) (string, error) {
+func GenerateEtcdService(index int, etcdContainer bool) (string, error) {
 	return util.Render(EtcdServiceTempl, util.Data{
-		"Name": fmt.Sprintf("etcd%d", index+1),
+		"Name":          fmt.Sprintf("etcd%d", index+1),
+		"EtcdContainer": etcdContainer,
 	})
 }
 

pkg/cluster/kubernetes/master.go

@@ -124,7 +124,7 @@ func initKubernetesCluster(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCf
 		}
 
 		for i := 0; i < 3; i++ {
-			_, err2 := mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"/usr/local/bin/kubeadm init --config=/etc/kubernetes/kubeadm-config.yaml\"", 0, true)
+			_, err2 := mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"/usr/local/bin/kubeadm init --config=/etc/kubernetes/kubeadm-config.yaml --ignore-preflight-errors=FileExisting-crictl\"", 0, true)
 			if err2 != nil {
 				if i == 2 {
 					return errors.Wrap(errors.WithStack(err2), "Failed to init kubernetes cluster")

pkg/cluster/kubernetes/nodes.go

@@ -120,7 +120,7 @@ func SetKubelet(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) error {
 		return errors.Wrap(errors.WithStack(err), "Failed to enable kubelet service")
 	}
 
-	kubeletEnv, err3 := tmpl.GenerateKubeletEnv(node)
+	kubeletEnv, err3 := tmpl.GenerateKubeletEnv(mgr, node)
 	if err3 != nil {
 		return err3
 	}

pkg/cluster/kubernetes/tmpl/kubeadm.go

@@ -94,6 +94,13 @@ scheduler:
     port: "10251"
     feature-gates: CSINodeInfo=true,VolumeSnapshotDataSource=true,ExpandCSIVolumes=true,RotateKubeletClientCertificate=true
 
+{{- if .CriSock }}
+---
+apiVersion: kubeadm.k8s.io/v1beta2
+kind: InitConfiguration
+nodeRegistration:
+  criSocket: {{ .CriSock }}
+{{- end }}
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
@@ -140,6 +147,7 @@ kubeReserved:
 systemReserved:
   cpu: 200m
   memory: 250Mi
+{{- if not .CriSock }}
 evictionHard:
   memory.available: 5%
 evictionSoft:
@@ -148,6 +156,7 @@ evictionSoftGracePeriod:
   memory.available: 2m
 evictionMaxPodGracePeriod: 120
 evictionPressureTransitionPeriod: 30s
+{{- end }}
 featureGates:
   CSINodeInfo: true
   VolumeSnapshotDataSource: true
@@ -157,9 +166,10 @@ featureGates:
     `)))
 
 func GenerateKubeadmCfg(mgr *manager.Manager) (string, error) {
+	// generate etcd configuration
 	var externalEtcd kubekeyapiv1alpha1.ExternalEtcd
 	var endpointsList []string
-	var caFile, certFile, keyFile string
+	var caFile, certFile, keyFile, containerRuntimeEndpoint string
 
 	for _, host := range mgr.EtcdNodes {
 		endpoint := fmt.Sprintf("https://%s:%s", host.InternalAddress, kubekeyapiv1alpha1.DefaultEtcdPort)
@@ -175,11 +185,12 @@ func GenerateKubeadmCfg(mgr *manager.Manager) (string, error) {
 	externalEtcd.CertFile = certFile
 	externalEtcd.KeyFile = keyFile
 
+	// generate images repo configuration
 	var imageRepo string
 	if mgr.Cluster.Registry.PrivateRegistry != "" {
-		imageRepo = fmt.Sprintf("%s/%s", mgr.Cluster.Registry.PrivateRegistry, mgr.Cluster.Kubernetes.ImageRepo)
+		imageRepo = fmt.Sprintf("%s/%s", mgr.Cluster.Registry.PrivateRegistry, kubekeyapiv1alpha1.DefaultKubeImageNamespace)
 	} else {
-		imageRepo = mgr.Cluster.Kubernetes.ImageRepo
+		imageRepo = kubekeyapiv1alpha1.DefaultKubeImageNamespace
 	}
 
 	var corednsRepo string
@@ -188,6 +199,25 @@ func GenerateKubeadmCfg(mgr *manager.Manager) (string, error) {
 	} else {
 		corednsRepo = ""
 	}
+
+	// generate cri configuration
+	switch mgr.Cluster.Kubernetes.ContainerManager {
+	case "docker":
+		containerRuntimeEndpoint = ""
+	case "crio":
+		containerRuntimeEndpoint = kubekeyapiv1alpha1.DefaultCrioEndpoint
+	case "containerd":
+		containerRuntimeEndpoint = kubekeyapiv1alpha1.DefaultContainerdEndpoint
+	case "isula":
+		containerRuntimeEndpoint = kubekeyapiv1alpha1.DefaultIsulaEndpoint
+	default:
+		containerRuntimeEndpoint = ""
+	}
+
+	if mgr.Cluster.Kubernetes.ContainerRuntimeEndpoint != "" {
+		containerRuntimeEndpoint = mgr.Cluster.Kubernetes.ContainerRuntimeEndpoint
+	}
+
 	return util.Render(KubeadmCfgTempl, util.Data{
 		"ImageRepo":            imageRepo,
 		"CorednsRepo":          corednsRepo,
@@ -204,5 +234,6 @@ func GenerateKubeadmCfg(mgr *manager.Manager) (string, error) {
 		"NodeCidrMaskSize":     mgr.Cluster.Kubernetes.NodeCidrMaskSize,
 		"MaxPods":              mgr.Cluster.Kubernetes.MaxPods,
 		"ProxyMode":            mgr.Cluster.Kubernetes.ProxyMode,
+		"CriSock":              containerRuntimeEndpoint,
 	})
 }