fix yurthub static pod isntallation in yurtadm join command (#1458)

* fix yurthub static pod isntallation in yurtadm join command * revise changeLog with YurtStaticSet * improve logs of yurtadm * modify print name of yurtstaticset * update yurtstaticset for reconciling static pods * add yurthub-cloud-yurtstaticset cr for yurthub pod on cloud nodes
openyurtio · May 15, 2023 · ac6ce54 · ac6ce54
1 parent fdcedd7
commit ac6ce54
Show file tree

Hide file tree

Showing 17 changed files with 189 additions and 66 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -20,8 +20,8 @@ As you know, static pods are managed directly by the kubelet daemon on the node
 In general, if a user wants to upgrade a static pod(like YurtHub), the user should manually modify or replace the manifest
 of the static pod. this can be a very tedious and painful task when the number of static pods becomes very large.
 
-Users can define Pod templates and upgrade models through StaticPod CRD. The upgrade models support both OTA and AdvancedRollingUpdate kinds,
-thus easily meeting the upgrade needs of large-scale Static Pods. Also the Pod template in yurthub StaticPod CRD is used to
+Users can define Pod templates and upgrade models through YurtStaticSet CRD. The upgrade models support both OTA and AdvancedRollingUpdate kinds,
+thus easily meeting the upgrade needs of large-scale Static Pods. Also the Pod template in yurthub YurtAppSet CRD is used to
 install YurtHub component on the node when the node is joined. [#1261](https://github.com/openyurtio/openyurt/pull/1261), [#1168](https://github.com/openyurtio/openyurt/pull/1168), [#1172](https://github.com/openyurtio/openyurt/pull/1172)
 
 **NodePort Service supports nodepool isolation**

diff --git a/charts/yurt-manager/crds/apps.openyurt.io_yurtstaticsets.yaml b/charts/yurt-manager/crds/apps.openyurt.io_yurtstaticsets.yaml
@@ -26,15 +26,15 @@ spec:
       type: date
     - description: The total number of static pods
       jsonPath: .status.totalNumber
-      name: TotalNumber
+      name: Total
       type: integer
     - description: The number of ready static pods
       jsonPath: .status.readyNumber
-      name: ReadyNumber
+      name: Ready
       type: integer
     - description: The number of static pods that have been upgraded
       jsonPath: .status.upgradedNumber
-      name: UpgradedNumber
+      name: Upgraded
       type: integer
     name: v1alpha1
     schema:

diff --git a/charts/yurthub/templates/yurthub-cfg.yaml b/charts/yurthub/templates/yurthub-cfg.yaml
@@ -71,14 +71,15 @@ subjects:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: bootstrap-static-pod
+  name: yurt-hub-yurt-static-set-role
 rules:
   - apiGroups:
       - ""
     resources:
       - configmaps
     resourceNames:
-      - static-pod-cm-{{ .Release.Namespace }}-yurt-hub
+      - yurt-static-set-yurt-hub
+      - yurt-static-set-yurt-hub-cloud
     verbs:
       - get
 ---
@@ -89,7 +90,7 @@ metadata:
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: bootstrap-static-pod
+  name: yurt-hub-yurt-static-set-role
 subjects:
   - apiGroup: rbac.authorization.k8s.io
     kind: Group

diff --git a/charts/yurthub/templates/yurthub-cloud-yurtstaticset.yaml b/charts/yurthub/templates/yurthub-cloud-yurtstaticset.yaml
@@ -0,0 +1,73 @@
+apiVersion: apps.openyurt.io/v1alpha1
+kind: YurtStaticSet
+metadata:
+  name: yurt-hub-cloud
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "yurthub.labels" . | nindent 4 }}
+spec:
+  staticPodManifest: yurthub
+  template:
+    metadata:
+      labels:
+        k8s-app: yurt-hub-cloud
+      name: yurt-hub-cloud
+      namespace: {{ .Release.Namespace }}
+    spec:
+      volumes:
+        - name: hub-dir
+          hostPath:
+            path: /var/lib/yurthub
+            type: DirectoryOrCreate
+        - name: kubernetes
+          hostPath:
+            path: /etc/kubernetes
+            type: Directory
+      containers:
+        - name: yurt-hub
+          image: {{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: hub-dir
+              mountPath: /var/lib/yurthub
+            - name: kubernetes
+              mountPath: /etc/kubernetes
+          command:
+            - yurthub
+            - --v=2
+            - --bind-address={{ .Values.yurthubBindingAddr }}
+            - --server-addr={{ .Values.kubernetesServerAddr }}
+            - --node-name=$(NODE_NAME)
+            - --bootstrap-file={{ .Values.bootstrapFile }}
+            - --working-mode=cloud
+            - --namespace={{ .Release.Namespace }}
+            {{- if .Values.organizations }}
+            - --hub-cert-organizations={{ .Values.organizations }}
+            {{- end }}
+          livenessProbe:
+            httpGet:
+              host: {{ .Values.yurthubBindingAddr }}
+              path: /v1/healthz
+              port: 10267
+            initialDelaySeconds: 300
+            periodSeconds: 5
+            failureThreshold: 3
+          resources:
+            requests:
+              cpu: 150m
+              memory: 150Mi
+            limits:
+              memory: 300Mi
+          securityContext:
+            capabilities:
+              add: [ "NET_ADMIN", "NET_RAW" ]
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+      hostNetwork: true
+      priorityClassName: system-node-critical
+      priority: 2000001000
+
+
diff --git a/.../yurthub/templates/yurthub-staticpod.yaml → ...thub/templates/yurthub-yurtstaticset.yaml b/.../yurthub/templates/yurthub-staticpod.yaml → ...thub/templates/yurthub-yurtstaticset.yaml
@@ -1,5 +1,5 @@
 apiVersion: apps.openyurt.io/v1alpha1
-kind: StaticPod
+kind: YurtStaticSet
 metadata:
   name: yurt-hub
   namespace: {{ .Release.Namespace }}
@@ -35,18 +35,18 @@ spec:
           command:
             - yurthub
             - --v=2
-            - --bind-address={{`{{.yurthubBindingAddr}}`}}
-            - --server-addr={{`{{.kubernetesServerAddr}}`}}
+            - --bind-address={{ .Values.yurthubBindingAddr }}
+            - --server-addr={{ .Values.kubernetesServerAddr }}
             - --node-name=$(NODE_NAME)
             - --bootstrap-file={{ .Values.bootstrapFile }}
-            - --working-mode={{`{{.workingMode}}`}}
+            - --working-mode=edge
             - --namespace={{ .Release.Namespace }}
-              {{`{{if .organizations }}`}}
-            - --hub-cert-organizations={{`{{.organizations}}`}}
-              {{`{{end}}`}}
+            {{- if .Values.organizations }}
+            - --hub-cert-organizations={{ .Values.organizations }}
+            {{- end }}
           livenessProbe:
             httpGet:
-              host: {{`{{.yurthubBindingAddr}}`}}
+              host: {{ .Values.yurthubBindingAddr }}
               path: /v1/healthz
               port: 10267
             initialDelaySeconds: 300

diff --git a/charts/yurthub/values.yaml b/charts/yurthub/values.yaml
@@ -6,7 +6,10 @@ nameOverride: ""
 
 cacheAgents: ""
 
+yurthubBindingAddr: 127.0.0.1
+kubernetesServerAddr: https://127.0.0.1:6443
 bootstrapFile: "/var/lib/yurthub/bootstrap-hub.conf"
+organizations: ""
 
 image:
   registry: openyurt

diff --git a/cmd/yurthub/app/config/config.go b/cmd/yurthub/app/config/config.go
@@ -210,7 +210,7 @@ func parseRemoteServers(serverAddr string) ([]*url.URL, error) {
 	for _, server := range servers {
 		u, err := url.Parse(server)
 		if err != nil {
-			klog.Errorf("failed to parse server address %s, %v", servers, err)
+			klog.Errorf("failed to parse server address %q, %v", server, err)
 			return us, err
 		}
 		if u.Scheme == "" {

diff --git a/pkg/apis/apps/v1alpha1/yurtstaticset_types.go b/pkg/apis/apps/v1alpha1/yurtstaticset_types.go
@@ -83,9 +83,9 @@ type YurtStaticSetStatus struct {
 // +kubebuilder:subresource:status
 // +kubebuilder:resource:shortName=yss
 // +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC."
-//+kubebuilder:printcolumn:name="TotalNumber",type="integer",JSONPath=".status.totalNumber",description="The total number of static pods"
-//+kubebuilder:printcolumn:name="ReadyNumber",type="integer",JSONPath=".status.readyNumber",description="The number of ready static pods"
-//+kubebuilder:printcolumn:name="UpgradedNumber",type="integer",JSONPath=".status.upgradedNumber",description="The number of static pods that have been upgraded"
+//+kubebuilder:printcolumn:name="Total",type="integer",JSONPath=".status.totalNumber",description="The total number of static pods"
+//+kubebuilder:printcolumn:name="Ready",type="integer",JSONPath=".status.readyNumber",description="The number of ready static pods"
+//+kubebuilder:printcolumn:name="Upgraded",type="integer",JSONPath=".status.upgradedNumber",description="The number of static pods that have been upgraded"
 
 // YurtStaticSet is the Schema for the yurtstaticsets API
 type YurtStaticSet struct {

diff --git a/pkg/controller/yurtstaticset/upgradeinfo/upgrade_info.go b/pkg/controller/yurtstaticset/upgradeinfo/upgrade_info.go
@@ -79,7 +79,7 @@ func New(c client.Client, instance *appsv1alpha1.YurtStaticSet, workerPodName, h
 		}
 
 		// The name format of mirror static pod is `StaticPodName-NodeName`
-		if util.Hyphen(instance.Name, nodeName) == pod.Name && isStaticPod(&pod) {
+		if util.Hyphen(instance.Name, nodeName) == pod.Name && util.IsStaticPod(&pod) {
 			// initialize static pod info
 			if err := initStaticPodInfo(c, nodeName, hash, &podList.Items[i], infos); err != nil {
 				return nil, err
@@ -149,16 +149,6 @@ func initWorkerPodInfo(nodeName, hash string, pod *corev1.Pod, infos map[string]
 	return nil
 }
 
-// isStaticPod judges whether a pod is static by its OwnerReference
-func isStaticPod(pod *corev1.Pod) bool {
-	for _, ownerRef := range pod.GetOwnerReferences() {
-		if ownerRef.Kind == "Node" {
-			return true
-		}
-	}
-	return false
-}
-
 // ReadyUpgradeWaitingNodes gets those nodes that satisfied
 // 1. node is ready
 // 2. node needs to be upgraded

diff --git a/pkg/controller/yurtstaticset/util/util.go b/pkg/controller/yurtstaticset/util/util.go
@@ -198,3 +198,13 @@ func GetPodConditionFromList(conditions []corev1.PodCondition, conditionType cor
 	}
 	return -1, nil
 }
+
+// IsStaticPod judges whether a pod is static by its OwnerReference
+func IsStaticPod(pod *corev1.Pod) bool {
+	for _, ownerRef := range pod.GetOwnerReferences() {
+		if ownerRef.Kind == "Node" {
+			return true
+		}
+	}
+	return false
+}
diff --git a/pkg/controller/yurtstaticset/yurtstaticset_controller.go b/pkg/controller/yurtstaticset/yurtstaticset_controller.go
@@ -215,6 +215,33 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error {
 		return err
 	}
 
+	// 4. Watch for changes of static pods
+	reconcileYurtStatisSetForStaticPod := func(obj client.Object) []reconcile.Request {
+		var reqs []reconcile.Request
+		pod, ok := obj.(*corev1.Pod)
+		if !ok {
+			return reqs
+		}
+
+		if !util.IsStaticPod(pod) {
+			return reqs
+		}
+
+		yurtStaticSetName := strings.TrimSuffix(pod.Name, fmt.Sprintf("-%s", pod.Spec.NodeName))
+		reqs = append(reqs, reconcile.Request{NamespacedName: types.NamespacedName{
+			Namespace: pod.Namespace,
+			Name:      yurtStaticSetName,
+		}})
+
+		return reqs
+	}
+	if err := c.Watch(&source.Kind{Type: &corev1.Pod{}}, handler.EnqueueRequestsFromMapFunc(
+		func(obj client.Object) []reconcile.Request {
+			return reconcileYurtStatisSetForStaticPod(obj)
+		})); err != nil {
+		return err
+	}
+
 	return nil
 }
 

diff --git a/pkg/yurtadm/cmd/join/join.go b/pkg/yurtadm/cmd/join/join.go
@@ -30,6 +30,7 @@ import (
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	"k8s.io/klog/v2"
 
+	"github.com/openyurtio/openyurt/pkg/controller/yurtstaticset/util"
 	kubeconfigutil "github.com/openyurtio/openyurt/pkg/util/kubeconfig"
 	"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/joindata"
 	yurtphases "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/phases"
@@ -209,6 +210,7 @@ type joinData struct {
 	pauseImage               string
 	yurthubImage             string
 	yurthubTemplate          string
+	yurthubManifest          string
 	kubernetesVersion        string
 	caCertHashes             []string
 	nodeLabels               map[string]string
@@ -321,13 +323,20 @@ func newJoinData(args []string, opt *joinOptions) (*joinData, error) {
 	klog.Infof("node join data info: %#+v", *data)
 
 	// get the yurthub template from the staticpod cr
-	yurthubTemplate, err := yurtadmutil.GetYurthubTemplateFromStaticPod(client, opt.namespace)
+	yurthubYurtStaticSetName := yurtconstants.YurthubYurtStaticSetName
+	if data.NodeRegistration().WorkingMode == "cloud" {
+		yurthubYurtStaticSetName = yurtconstants.YurthubCloudYurtStaticSetName
+	}
+
+	yurthubManifest, yurthubTemplate, err := yurtadmutil.GetYurthubTemplateFromStaticPod(client, opt.namespace, util.WithConfigMapPrefix(yurthubYurtStaticSetName))
 	if err != nil {
 		klog.Errorf("hard-code yurthub manifest will be used, because failed to get yurthub template from kube-apiserver, %v", err)
+		yurthubManifest = yurtconstants.YurthubStaticPodManifest
 		yurthubTemplate = yurtconstants.YurthubTemplate
+
 	}
 	data.yurthubTemplate = yurthubTemplate
-	klog.Infof("yurthub template: %s", yurthubTemplate)
+	data.yurthubManifest = yurthubManifest
 
 	return data, nil
 }
@@ -362,6 +371,10 @@ func (j *joinData) YurtHubTemplate() string {
 	return j.yurthubTemplate
 }
 
+func (j *joinData) YurtHubManifest() string {
+	return j.yurthubManifest
+}
+
 // KubernetesVersion returns the kubernetes version.
 func (j *joinData) KubernetesVersion() string {
 	return j.kubernetesVersion

diff --git a/pkg/yurtadm/cmd/join/joindata/data.go b/pkg/yurtadm/cmd/join/joindata/data.go
@@ -36,6 +36,7 @@ type YurtJoinData interface {
 	YurtHubImage() string
 	YurtHubServer() string
 	YurtHubTemplate() string
+	YurtHubManifest() string
 	KubernetesVersion() string
 	TLSBootstrapCfg() *clientcmdapi.Config
 	BootstrapClient() *clientset.Clientset

diff --git a/pkg/yurtadm/cmd/join/phases/prepare.go b/pkg/yurtadm/cmd/join/phases/prepare.go
@@ -67,7 +67,7 @@ func RunPrepare(data joindata.YurtJoinData) error {
 	if err := yurthub.SetHubBootstrapConfig(data.ServerAddr(), data.JoinToken(), data.CaCertHashes()); err != nil {
 		return err
 	}
-	if err := yurthub.AddYurthubStaticYaml(data, filepath.Join(constants.KubeletConfigureDir, constants.ManifestsSubDirName)); err != nil {
+	if err := yurthub.AddYurthubStaticYaml(data, constants.StaticPodPath); err != nil {
 		return err
 	}
 	if err := yurtadmutil.SetDiscoveryConfig(data); err != nil {

diff --git a/pkg/yurtadm/constants/constants.go b/pkg/yurtadm/constants/constants.go
@@ -17,28 +17,29 @@ limitations under the License.
 package constants
 
 const (
-	Hostname                 = "/etc/hostname"
-	SysctlK8sConfig          = "/etc/sysctl.d/k8s.conf"
-	StaticPodPath            = "/etc/kubernetes/manifests"
-	KubeletConfigureDir      = "/etc/kubernetes"
-	KubeletWorkdir           = "/var/lib/kubelet"
-	YurtHubWorkdir           = "/var/lib/yurthub"
-	YurtHubBootstrapConfig   = "/var/lib/yurthub/bootstrap-hub.conf"
-	OpenyurtDir              = "/var/lib/openyurt"
-	YurttunnelAgentWorkdir   = "/var/lib/yurttunnel-agent"
-	YurttunnelServerWorkdir  = "/var/lib/yurttunnel-server"
-	KubeCniDir               = "/opt/cni/bin"
-	KubeCniVersion           = "v0.8.0"
-	KubeletServiceFilepath   = "/etc/systemd/system/kubelet.service"
-	KubeletServiceConfPath   = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"
-	KubeletSvcPath           = "/usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf"
-	YurthubStaticPodFileName = "yurthub.yaml"
-	PauseImagePath           = "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2"
-	DefaultCertificatesDir   = "/etc/kubernetes/pki"
-	DefaultDockerCRISocket   = "/var/run/dockershim.sock"
-	YurthubYamlName          = "yurt-hub.yaml"
-	YurthubStaticPodManifest = "yurt-hub"
-	YurthubNamespace         = "kube-system"
+	Hostname                      = "/etc/hostname"
+	SysctlK8sConfig               = "/etc/sysctl.d/k8s.conf"
+	StaticPodPath                 = "/etc/kubernetes/manifests"
+	KubeletConfigureDir           = "/etc/kubernetes"
+	KubeletWorkdir                = "/var/lib/kubelet"
+	YurtHubWorkdir                = "/var/lib/yurthub"
+	YurtHubBootstrapConfig        = "/var/lib/yurthub/bootstrap-hub.conf"
+	OpenyurtDir                   = "/var/lib/openyurt"
+	YurttunnelAgentWorkdir        = "/var/lib/yurttunnel-agent"
+	YurttunnelServerWorkdir       = "/var/lib/yurttunnel-server"
+	KubeCniDir                    = "/opt/cni/bin"
+	KubeCniVersion                = "v0.8.0"
+	KubeletServiceFilepath        = "/etc/systemd/system/kubelet.service"
+	KubeletServiceConfPath        = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"
+	KubeletSvcPath                = "/usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf"
+	PauseImagePath                = "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2"
+	DefaultCertificatesDir        = "/etc/kubernetes/pki"
+	DefaultDockerCRISocket        = "/var/run/dockershim.sock"
+	YurthubYamlName               = "yurthub.yaml"
+	YurthubStaticPodManifest      = "yurthub"
+	YurthubNamespace              = "kube-system"
+	YurthubYurtStaticSetName      = "yurt-hub"
+	YurthubCloudYurtStaticSetName = "yurt-hub-cloud"
 	// ManifestsSubDirName defines directory name to store manifests
 	ManifestsSubDirName = "manifests"
 	// KubeletKubeConfigFileName defines the file name for the kubeconfig that the control-plane kubelet will use for talking