feat: add hub leader config controller (#2299)

* feat: add hub leader config controller

Co-authored-by: Simon Tien <[email protected]>

charts/yurt-manager/templates/yurt-manager-auto-generated.yaml

@@ -56,6 +56,12 @@ metadata:
 ---
 apiVersion: v1
 kind: ServiceAccount
+metadata:
+  name: yurt-manager-hubleaderconfig-controller
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: v1
+kind: ServiceAccount
 metadata:
   name: yurt-manager-load-balancer-set-controller
   namespace: {{ .Release.Namespace }}
@@ -476,12 +482,34 @@ rules:
 - apiGroups:
   - apps.openyurt.io
   resources:
-  - nodepool
-  - nodepool/status
+  - nodepools
+  - nodepools/status
+  verbs:
+  - get
+  - patch
+  - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: yurt-manager-hubleaderconfig-controller
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
   verbs:
+  - create
   - get
   - patch
   - update
+- apiGroups:
+  - apps.openyurt.io
+  resources:
+  - nodepools
+  - nodepools/status
+  verbs:
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -1042,6 +1070,19 @@ subjects:
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
+metadata:
+  name: yurt-manager-hubleaderconfig-controller-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: yurt-manager-hubleaderconfig-controller
+subjects:
+- kind: ServiceAccount
+  name: yurt-manager-hubleaderconfig-controller
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
 metadata:
   name: yurt-manager-load-balancer-set-controller-binding
 roleRef:

cmd/yurt-manager/app/options/hubleaderconfigcontroller.go

@@ -0,0 +1,71 @@
+/*
+Copyright 2025 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the License);
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an AS IS BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"github.com/spf13/pflag"
+
+	"github.com/openyurtio/openyurt/pkg/yurtmanager/controller/hubleaderconfig/config"
+)
+
+type HubLeaderConfigControllerOptions struct {
+	*config.HubLeaderConfigControllerConfiguration
+}
+
+func NewHubLeaderConfigControllerOptions(hubleaderNamespace string) *HubLeaderConfigControllerOptions {
+	return &HubLeaderConfigControllerOptions{
+		&config.HubLeaderConfigControllerConfiguration{
+			ConcurrentHubLeaderConfigWorkers: 3,
+			HubLeaderNamespace:               hubleaderNamespace,
+		},
+	}
+}
+
+// AddFlags adds flags related to hubleader for yurt-manager to the specified FlagSet.
+func (h *HubLeaderConfigControllerOptions) AddFlags(fs *pflag.FlagSet) {
+	if h == nil {
+		return
+	}
+
+	fs.Int32Var(
+		&h.ConcurrentHubLeaderConfigWorkers,
+		"concurrent-hubleaderconfig-workers",
+		h.ConcurrentHubLeaderConfigWorkers,
+		"The number of nodepool objects that are allowed to reconcile concurrently.",
+	)
+}
+
+// ApplyTo fills up hubleader config with options.
+func (h *HubLeaderConfigControllerOptions) ApplyTo(cfg *config.HubLeaderConfigControllerConfiguration) error {
+	if h == nil {
+		return nil
+	}
+
+	cfg.ConcurrentHubLeaderConfigWorkers = h.ConcurrentHubLeaderConfigWorkers
+	cfg.HubLeaderNamespace = h.HubLeaderNamespace
+
+	return nil
+}
+
+// Validate checks validation of HubLeaderControllerOptions.
+func (h *HubLeaderConfigControllerOptions) Validate() []error {
+	if h == nil {
+		return nil
+	}
+	errs := []error{}
+	return errs
+}

cmd/yurt-manager/app/options/options.go

@@ -46,13 +46,14 @@ type YurtManagerOptions struct {
 	GatewayInternalSvcController  *GatewayInternalSvcControllerOptions
 	GatewayPublicSvcController    *GatewayPublicSvcControllerOptions
 	HubLeaderController           *HubLeaderControllerOptions
+	HubLeaderConfigController     *HubLeaderConfigControllerOptions
 }
 
 // NewYurtManagerOptions creates a new YurtManagerOptions with a default config.
 func NewYurtManagerOptions() (*YurtManagerOptions, error) {
-
+	genericOptions := NewGenericOptions()
 	s := YurtManagerOptions{
-		Generic:                       NewGenericOptions(),
+		Generic:                       genericOptions,
 		DelegateLeaseController:       NewDelegateLeaseControllerOptions(),
 		PodBindingController:          NewPodBindingControllerOptions(),
 		DaemonPodUpdaterController:    NewDaemonPodUpdaterControllerOptions(),
@@ -73,6 +74,7 @@ func NewYurtManagerOptions() (*YurtManagerOptions, error) {
 		GatewayInternalSvcController:  NewGatewayInternalSvcControllerOptions(),
 		GatewayPublicSvcController:    NewGatewayPublicSvcControllerOptions(),
 		HubLeaderController:           NewHubLeaderControllerOptions(),
+		HubLeaderConfigController:     NewHubLeaderConfigControllerOptions(genericOptions.WorkingNamespace),
 	}
 
 	return &s, nil
@@ -101,6 +103,7 @@ func (y *YurtManagerOptions) Flags(allControllers, disabledByDefaultControllers
 	y.GatewayInternalSvcController.AddFlags(fss.FlagSet("gatewayinternalsvc controller"))
 	y.GatewayPublicSvcController.AddFlags(fss.FlagSet("gatewaypublicsvc controller"))
 	y.HubLeaderController.AddFlags(fss.FlagSet("hubleader controller"))
+	y.HubLeaderConfigController.AddFlags(fss.FlagSet("hubleaderconfig controller"))
 	return fss
 }
 
@@ -128,6 +131,7 @@ func (y *YurtManagerOptions) Validate(allControllers []string, controllerAliases
 	errs = append(errs, y.GatewayInternalSvcController.Validate()...)
 	errs = append(errs, y.GatewayPublicSvcController.Validate()...)
 	errs = append(errs, y.HubLeaderController.Validate()...)
+	errs = append(errs, y.HubLeaderConfigController.Validate()...)
 	return utilerrors.NewAggregate(errs)
 }
 
@@ -196,6 +200,9 @@ func (y *YurtManagerOptions) ApplyTo(c *config.Config, controllerAliases map[str
 	if err := y.HubLeaderController.ApplyTo(&c.ComponentConfig.HubLeaderController); err != nil {
 		return err
 	}
+	if err := y.HubLeaderConfigController.ApplyTo(&c.ComponentConfig.HubLeaderConfigController); err != nil {
+		return err
+	}
 	return nil
 }
 

cmd/yurt-manager/names/controller_names.go

@@ -37,6 +37,7 @@ const (
 	NodeBucketController                   = "node-bucket-controller"
 	LoadBalancerSetController              = "load-balancer-set-controller"
 	HubLeaderController                    = "hubleader-controller"
+	HubLeaderConfigController              = "hubleaderconfig-controller"
 )
 
 func YurtManagerControllerAliases() map[string]string {
@@ -61,5 +62,7 @@ func YurtManagerControllerAliases() map[string]string {
 		"nodelifecycle":                 NodeLifeCycleController,
 		"nodebucket":                    NodeBucketController,
 		"loadbalancerset":               LoadBalancerSetController,
+		"hubleader":                     HubLeaderController,
+		"hubleaderconfig":               HubLeaderConfigController,
 	}
 }

pkg/projectinfo/projectinfo.go

@@ -128,6 +128,16 @@ func GetNodePoolLabel() string {
 	return nodePoolLabelKey
 }
 
+// GetHubleaderConfigMapName returns the name of the leader ConfigMap for the nodepool
+func GetHubleaderConfigMapName(nodepoolName string) string {
+	return fmt.Sprintf("leader-hub-%s", nodepoolName)
+}
+
+// GetHubLeaderConfigMapLabel returns the label of the leader ConfigMap for the nodepool
+func GetHubLeaderConfigMapLabel() string {
+	return fmt.Sprintf("%s/configmap-name", labelPrefix)
+}
+
 // Info contains version information.
 type Info struct {
 	GitVersion       string   `json:"gitVersion"`

pkg/yurtmanager/controller/apis/config/types.go

@@ -24,6 +24,7 @@ import (
 	csrapproverconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/csrapprover/config"
 	daemonpodupdaterconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/daemonpodupdater/config"
 	hubleaderconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/hubleader/config"
+	hubleadercfgconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/hubleaderconfig/config"
 	loadbalancersetconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/config"
 	nodebucketconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/nodebucket/config"
 	nodepoolconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/nodepool/config"
@@ -106,6 +107,9 @@ type YurtManagerConfiguration struct {
 
 	// HubLeaderController holds configuration for HubLeaderController related features.
 	HubLeaderController hubleaderconfig.HubLeaderControllerConfiguration
+
+	// HubLeaderConfigController holds configuration for HubLeaderController related features.
+	HubLeaderConfigController hubleadercfgconfig.HubLeaderConfigControllerConfiguration
 }
 
 type GenericConfiguration struct {

pkg/yurtmanager/controller/base/controller.go

@@ -32,6 +32,8 @@ import (
 	"github.com/openyurtio/openyurt/cmd/yurt-manager/names"
 	"github.com/openyurtio/openyurt/pkg/yurtmanager/controller/csrapprover"
 	"github.com/openyurtio/openyurt/pkg/yurtmanager/controller/daemonpodupdater"
+	"github.com/openyurtio/openyurt/pkg/yurtmanager/controller/hubleader"
+	"github.com/openyurtio/openyurt/pkg/yurtmanager/controller/hubleaderconfig"
 	"github.com/openyurtio/openyurt/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset"
 	"github.com/openyurtio/openyurt/pkg/yurtmanager/controller/nodebucket"
 	"github.com/openyurtio/openyurt/pkg/yurtmanager/controller/nodelifecycle"
@@ -97,6 +99,8 @@ func NewControllerInitializers() map[string]InitFunc {
 	register(names.NodeLifeCycleController, nodelifecycle.Add)
 	register(names.NodeBucketController, nodebucket.Add)
 	register(names.LoadBalancerSetController, loadbalancerset.Add)
+	register(names.HubLeaderController, hubleader.Add)
+	register(names.HubLeaderConfigController, hubleaderconfig.Add)
 
 	return controllers
 }
@@ -134,7 +138,11 @@ func NewControllerInitializers() map[string]InitFunc {
 
 func SetupWithManager(ctx context.Context, c *config.CompletedConfig, m manager.Manager) error {
 	for controllerName, fn := range NewControllerInitializers() {
-		if !app.IsControllerEnabled(controllerName, ControllersDisabledByDefault, c.ComponentConfig.Generic.Controllers) {
+		if !app.IsControllerEnabled(
+			controllerName,
+			ControllersDisabledByDefault,
+			c.ComponentConfig.Generic.Controllers,
+		) {
 			klog.Warningf("Controller %v is disabled", controllerName)
 			continue
 		}
@@ -150,8 +158,16 @@ func SetupWithManager(ctx context.Context, c *config.CompletedConfig, m manager.
 		}
 	}
 
-	if app.IsControllerEnabled(names.NodeLifeCycleController, ControllersDisabledByDefault, c.ComponentConfig.Generic.Controllers) ||
-		app.IsControllerEnabled(names.PodBindingController, ControllersDisabledByDefault, c.ComponentConfig.Generic.Controllers) {
+	if app.IsControllerEnabled(
+		names.NodeLifeCycleController,
+		ControllersDisabledByDefault,
+		c.ComponentConfig.Generic.Controllers,
+	) ||
+		app.IsControllerEnabled(
+			names.PodBindingController,
+			ControllersDisabledByDefault,
+			c.ComponentConfig.Generic.Controllers,
+		) {
 		// Register spec.NodeName field indexers
 		if err := m.GetFieldIndexer().IndexField(context.TODO(), &v1.Pod{}, "spec.nodeName", func(rawObj client.Object) []string {
 			pod, ok := rawObj.(*v1.Pod)

pkg/yurtmanager/controller/hubleader/hubleader_controller.go

@@ -41,6 +41,7 @@ import (
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
 	"github.com/openyurtio/openyurt/pkg/yurtmanager/controller/hubleader/config"
 	nodeutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/node"
+	nodepoolutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/nodepool"
 )
 
 var (
@@ -131,8 +132,8 @@ type ReconcileHubLeader struct {
 	Configuration config.HubLeaderControllerConfiguration
 }
 
-// +kubebuilder:rbac:groups=apps.openyurt.io,resources=nodepool,verbs=get;update;patch
-// +kubebuilder:rbac:groups=apps.openyurt.io,resources=nodepool/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=apps.openyurt.io,resources=nodepools,verbs=get;update;patch
+// +kubebuilder:rbac:groups=apps.openyurt.io,resources=nodepools/status,verbs=get;update;patch
 
 // Reconcile reads that state of the cluster for a HubLeader object and makes changes based on the state read
 // and what is in the HubLeader.Spec
@@ -235,7 +236,7 @@ func (r *ReconcileHubLeader) reconcileHubLeader(ctx context.Context, nodepool *a
 
 	updatedNodePool.Status.LeaderEndpoints = updatedLeaders
 
-	if !hasLeadersChanged(nodepool.Status.LeaderEndpoints, updatedNodePool.Status.LeaderEndpoints) {
+	if !nodepoolutil.HasSliceContentChanged(nodepool.Status.LeaderEndpoints, updatedNodePool.Status.LeaderEndpoints) {
 		return nil
 	}
 
@@ -248,27 +249,6 @@ func (r *ReconcileHubLeader) reconcileHubLeader(ctx context.Context, nodepool *a
 	return nil
 }
 
-// hasLeadersChanged checks if the leader endpoints have changed
-func hasLeadersChanged(old, new []appsv1beta2.Leader) bool {
-	if len(old) != len(new) {
-		return true
-	}
-
-	oldSet := make(map[appsv1beta2.Leader]struct{}, len(old))
-
-	for i := range old {
-		oldSet[old[i]] = struct{}{}
-	}
-
-	for i := range new {
-		if _, ok := oldSet[new[i]]; !ok {
-			return true
-		}
-	}
-
-	return false
-}
-
 // electNLeaders elects N leaders from the candidates based on the strategy
 func electNLeaders(
 	strategy string,

pkg/yurtmanager/controller/hubleaderconfig/config/types.go

@@ -0,0 +1,24 @@
+/*
+Copyright 2025 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the License);
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an AS IS BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+// HubLeaderConfigControllerConfiguration contains elements describing HubLeaderConfigController.
+type HubLeaderConfigControllerConfiguration struct {
+	ConcurrentHubLeaderConfigWorkers int32
+
+	HubLeaderNamespace string
+}