feat: kubekey support kubeovn network plugin with kube-ovn v1.5.0

Signed-off-by: pixiake <[email protected]>
stoneshi-yunify · Nov 11, 2020 · 36a8ac9 · 36a8ac9
1 parent 13b3ce6
commit 36a8ac9
Show file tree

Hide file tree

Showing 9 changed files with 3,348 additions and 12 deletions.
diff --git a/apis/kubekey/v1alpha1/default.go b/apis/kubekey/v1alpha1/default.go
@@ -18,10 +18,11 @@ package v1alpha1
 
 import (
 	"fmt"
-	"github.com/kubesphere/kubekey/pkg/util"
-	log "github.com/sirupsen/logrus"
 	"os"
 	"strings"
+
+	"github.com/kubesphere/kubekey/pkg/util"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -42,6 +43,7 @@ const (
 	DefaultFlannelVersion      = "v0.12.0"
 	DefaultCniVersion          = "v0.8.6"
 	DefaultCiliumVersion       = "v1.8.3"
+	DefaultKubeovnVersion      = "v1.5.0"
 	DefaultHelmVersion         = "v3.2.1"
 	DefaultMaxPods             = 110
 	DefaultNodeCidrMaskSize    = 24
@@ -58,6 +60,12 @@ const (
 	DefaultEtcdBackupPeriod    = 30
 	DefaultKeepBackNumber      = 5
 	DefaultEtcdBackupScriptDir = "/usr/local/bin/kube-scripts"
+	DefaultJoinCIDR            = "100.64.0.0/16"
+	DefaultNetworkType         = "geneve"
+	DefaultVlanID              = "100"
+	DefaultOvnLabel            = "node-role.kubernetes.io/master"
+	DefaultDPDKVersion         = "19.11"
+	DefaultDNSAddress          = "114.114.114.114"
 )
 
 func (cfg *ClusterSpec) SetDefaultClusterSpec(incluster bool, logger *log.Logger) (*ClusterSpec, *HostGroups, error) {
@@ -182,6 +190,25 @@ func SetDefaultNetworkCfg(cfg *ClusterSpec) NetworkConfig {
 	if cfg.Network.Flannel.BackendMode == "" {
 		cfg.Network.Flannel.BackendMode = DefaultBackendMode
 	}
+	// kube-ovn default config
+	if cfg.Network.Kubeovn.JoinCIDR == "" {
+		cfg.Network.Kubeovn.JoinCIDR = DefaultJoinCIDR
+	}
+	if cfg.Network.Kubeovn.Label == "" {
+		cfg.Network.Kubeovn.Label = DefaultOvnLabel
+	}
+	if cfg.Network.Kubeovn.VlanID == "" {
+		cfg.Network.Kubeovn.VlanID = DefaultVlanID
+	}
+	if cfg.Network.Kubeovn.NetworkType == "" {
+		cfg.Network.Kubeovn.NetworkType = DefaultNetworkType
+	}
+	if cfg.Network.Kubeovn.PingerExternalAddress == "" {
+		cfg.Network.Kubeovn.PingerExternalAddress = DefaultDNSAddress
+	}
+	if cfg.Network.Kubeovn.DpdkVersion == "" {
+		cfg.Network.Kubeovn.DpdkVersion = DefaultDPDKVersion
+	}
 	defaultNetworkCfg := cfg.Network
 
 	return defaultNetworkCfg

diff --git a/apis/kubekey/v1alpha1/network_types.go b/apis/kubekey/v1alpha1/network_types.go
@@ -22,6 +22,7 @@ type NetworkConfig struct {
 	KubeServiceCIDR string     `yaml:"kubeServiceCIDR" json:"kubeServiceCIDR,omitempty"`
 	Calico          CalicoCfg  `yaml:"calico" json:"calico,omitempty"`
 	Flannel         FlannelCfg `yaml:"flannel" json:"flannel,omitempty"`
+	Kubeovn         KubeovnCfg `yaml:"kubeovn" json:"kubeovn,omitempty"`
 }
 
 type CalicoCfg struct {
@@ -33,3 +34,19 @@ type CalicoCfg struct {
 type FlannelCfg struct {
 	BackendMode string `yaml:"backendMode" json:"backendMode,omitempty"`
 }
+
+type KubeovnCfg struct {
+	JoinCIDR              string `yaml:"joinCIDR" json:"joinCIDR,omitempty"`
+	NetworkType           string `yaml:"networkType" json:"networkType,omitempty"`
+	Label                 string `yaml:"label" json:"label,omitempty"`
+	Iface                 string `yaml:"iface" json:"iface,omitempty"`
+	VlanInterfaceName     string `yaml:"vlanInterfaceName" json:"vlanInterfaceName,omitempty"`
+	VlanID                string `yaml:"vlanID" json:"vlanID,omitempty"`
+	DpdkMode              bool   `yaml:"dpdkMode" json:"dpdkMode,omitempty"`
+	EnableSSL             bool   `yaml:"enableSSL" json:"enableSSL,omitempty"`
+	EnableMirror          bool   `yaml:"enableMirror" json:"enableMirror,omitempty"`
+	HwOffload             bool   `yaml:"hwOffload" json:"hwOffload,omitempty"`
+	DpdkVersion           string `yaml:"dpdkVersion" json:"dpdkVersion,omitempty"`
+	PingerExternalAddress string `yaml:"pingerExternalAddress" json:"pingerExternalAddress,omitempty"`
+	PingerExternalDomain  string `yaml:"pingerExternalDomain" json:"pingerExternalDomain,omitempty"`
+}
diff --git a/pkg/cluster/preinstall/prepull_images.go b/pkg/cluster/preinstall/prepull_images.go
@@ -1,13 +1,14 @@
 package preinstall
 
 import (
+	"strings"
+
 	kubekeyapiv1alpha1 "github.com/kubesphere/kubekey/apis/kubekey/v1alpha1"
 	kubekeycontroller "github.com/kubesphere/kubekey/controllers/kubekey"
 	"github.com/kubesphere/kubekey/pkg/images"
 	"github.com/kubesphere/kubekey/pkg/util/manager"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	versionutil "k8s.io/apimachinery/pkg/util/version"
-	"strings"
 )
 
 func PrePullImages(mgr *manager.Manager) error {
@@ -50,6 +51,7 @@ func PullImages(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) error {
 		GetImage(mgr, "cilium"),
 		GetImage(mgr, "operator-generic"),
 		GetImage(mgr, "flannel"),
+		GetImage(mgr, "kubeovn"),
 	}
 	if err := i.PullImages(mgr, node); err != nil {
 		return err
@@ -89,6 +91,7 @@ func GetImage(mgr *manager.Manager, name string) images.Image {
 		"flannel":                 {RepoAddr: mgr.Cluster.Registry.PrivateRegistry, Namespace: "kubesphere", Repo: "flannel", Tag: kubekeyapiv1alpha1.DefaultFlannelVersion, Group: kubekeyapiv1alpha1.K8s, Enable: strings.EqualFold(mgr.Cluster.Network.Plugin, "flannel")},
 		"cilium":                  {RepoAddr: mgr.Cluster.Registry.PrivateRegistry, Namespace: "cilium", Repo: "cilium", Tag: kubekeyapiv1alpha1.DefaultCiliumVersion, Group: kubekeyapiv1alpha1.K8s, Enable: strings.EqualFold(mgr.Cluster.Network.Plugin, "cilium")},
 		"operator-generic":        {RepoAddr: mgr.Cluster.Registry.PrivateRegistry, Namespace: "cilium", Repo: "operator-generic", Tag: kubekeyapiv1alpha1.DefaultCiliumVersion, Group: kubekeyapiv1alpha1.K8s, Enable: strings.EqualFold(mgr.Cluster.Network.Plugin, "cilium")},
+		"kubeovn":                 {RepoAddr: mgr.Cluster.Registry.PrivateRegistry, Namespace: "kubeovn", Repo: "kube-ovn", Tag: kubekeyapiv1alpha1.DefaultKubeovnVersion, Group: kubekeyapiv1alpha1.K8s, Enable: strings.EqualFold(mgr.Cluster.Network.Plugin, "kubeovn")},
 		// storage
 		"provisioner-localpv":    {RepoAddr: mgr.Cluster.Registry.PrivateRegistry, Namespace: "kubesphere", Repo: "provisioner-localpv", Tag: "1.10.0", Group: kubekeyapiv1alpha1.Worker, Enable: false},
 		"openebs-tools":          {RepoAddr: mgr.Cluster.Registry.PrivateRegistry, Namespace: "kubesphere", Repo: "openebs-tools", Tag: "3.8", Group: kubekeyapiv1alpha1.Worker, Enable: false},

diff --git a/pkg/delete/delete.go b/pkg/delete/delete.go
@@ -19,18 +19,19 @@ package delete
 import (
 	"bufio"
 	"fmt"
-	kubekeyapiv1alpha1 "github.com/kubesphere/kubekey/apis/kubekey/v1alpha1"
-	"github.com/kubesphere/kubekey/pkg/config"
-	"github.com/kubesphere/kubekey/pkg/util/executor"
-	"github.com/kubesphere/kubekey/pkg/util/manager"
-	"github.com/pkg/errors"
-	log "github.com/sirupsen/logrus"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"unicode"
+
+	kubekeyapiv1alpha1 "github.com/kubesphere/kubekey/apis/kubekey/v1alpha1"
+	"github.com/kubesphere/kubekey/pkg/config"
+	"github.com/kubesphere/kubekey/pkg/util/executor"
+	"github.com/kubesphere/kubekey/pkg/util/manager"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
 )
 
 func ResetCluster(clusterCfgFile string, logger *log.Logger, verbose bool) error {
@@ -276,6 +277,17 @@ var clusterFiles = []string{
 	"/usr/bin/kubelet",
 }
 
+var kubeovnFiles = []string{
+	"/var/run/openvswitch",
+	"/var/run/ovn",
+	"/etc/openvswitch",
+	"/etc/ovn",
+	"/var/log/openvswitch",
+	"/var/log/ovn",
+	"/etc/cni/net.d/00-kube-ovn.conflist",
+	"/etc/cni/net.d/01-kube-ovn.conflist",
+}
+
 var cmdsList = []string{
 	"iptables -F",
 	"iptables -X",
@@ -286,6 +298,11 @@ var cmdsList = []string{
 }
 
 func resetKubeCluster(mgr *manager.Manager, _ *kubekeyapiv1alpha1.HostCfg) error {
+	// delete OVN/OVS DB and config files on every node
+	if mgr.Cluster.Network.Plugin == "kubeovn" {
+		deleteOvnFiles(mgr)
+	}
+
 	_, _ = mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"/usr/local/bin/kubeadm reset -f\"", 0, true)
 	_, _ = mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", strings.Join(cmdsList, " && ")), 0, true, "printCmd")
 	_ = deleteFiles(mgr)
@@ -301,6 +318,13 @@ func deleteFiles(mgr *manager.Manager) error {
 	return nil
 }
 
+func deleteOvnFiles(mgr *manager.Manager) {
+	_, _ = mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"/usr/share/openvswitch/scripts/ovs-ctl stop && ovs-dpctl del-dp ovs-system\"", 1, true)
+	for _, file := range kubeovnFiles {
+		_, _ = mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"rm -rf %s\"", file), 1, true)
+	}
+}
+
 func Confirm(reader *bufio.Reader) (string, error) {
 	for {
 		fmt.Printf("Are you sure to delete this cluster? [yes/no]: ")