From c3962bf6c8eb4e6e7532ec573daa7d35a2bcac62 Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Thu, 31 Oct 2024 15:59:41 +0800 Subject: [PATCH 01/34] api redefine --- apis/apps/v1/componentdefinition_types.go | 65 ++++++++++++++++++++++- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/apis/apps/v1/componentdefinition_types.go b/apis/apps/v1/componentdefinition_types.go index 5f56af5ed39..ea0a7c6bb21 100644 --- a/apis/apps/v1/componentdefinition_types.go +++ b/apis/apps/v1/componentdefinition_types.go @@ -1372,8 +1372,13 @@ type ComponentAvailableProbeAssertion struct { // ReplicaRole represents a role that can be assumed by a component instance. type ReplicaRole struct { - // Defines the role's identifier. It is used to set the "apps.kubeblocks.io/role" label value - // on the corresponding object. + // Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label + // on the corresponding object to identify its role. + // + // For example, common role names include: + // - "leader": The primary/master instance that handles write operations + // - "follower": Secondary/replica instances that replicate data from the leader + // - "learner": Read-only instances that don't participate in elections // // This field is immutable once set. // @@ -1382,10 +1387,62 @@ type ReplicaRole struct { // +kubebuilder:validation:Pattern=`^.*[^\s]+.*$` Name string `json:"name"` + // Required indicates if at least one replica with this role must exist for the component to be considered + // operationally running. For example, a leader role may be required for the component to function. + // + // This field is immutable once set. + // + // +kubebuilder:default=false + // +optional + Required bool `json:"required"` + + // UpdatePriority determines the order in which pods with different roles are updated. + // Pods are sorted by this priority (higher numbers = higher priority) and updated accordingly. + // Roles with the highest priority will be updated last. + // + // For example: + // - Leader role may have priority 2 (updated last) + // - Follower role may have priority 1 (updated before leader) + // - Learner role may have priority 0 (updated first) + // + // This field is immutable once set. + // + // +kubebuilder:default=0 + // +optional + UpdatePriority int `json:"updatePriorit"` + + // ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. + // This affects update strategies that need to maintain quorum for availability. + // + // For example, in a 5-pod component where: + // - 2 learner pods (participatesInQuorum=false) + // - 2 follower pods (participatesInQuorum=true) + // - 1 leader pod (participatesInQuorum=true) + // The quorum size would be 3 (based on the 3 participating pods), allowing parallel updates + // of 2 learners and 1 follower while maintaining quorum. + // + // This field is immutable once set. + // + // +kubebuilder:default=false + // +optional + ParticipatesInQuorum bool `json:"participatesInQuorum bool"` + + // SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before + // updating or scaling in pods with this role. This is typically used for leader roles to + // ensure minimal disruption during updates. + // + // This field is immutable once set. + // + // +kubebuilder:default=false + // +optional + SwitchoverBeforeUpdate bool `json:"switchoverBeforeUpdate"` + // Indicates whether a replica assigned this role is capable of providing services. // // This field is immutable once set. // + // Deprecated: use other fields + // // +kubebuilder:default=false // +optional Serviceable bool `json:"serviceable,omitempty"` @@ -1395,6 +1452,8 @@ type ReplicaRole struct { // // This field is immutable once set. // + // Deprecated: use other fields + // // +kubebuilder:default=false // +optional Writable bool `json:"writable,omitempty"` @@ -1405,6 +1464,8 @@ type ReplicaRole struct { // // This field is immutable once set. // + // Deprecated: use other fields + // // +kubebuilder:default=false // +optional Votable bool `json:"votable,omitempty"` From 9981d3f81ed9d2ed31a4a10686331ad15dbf50a0 Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Thu, 31 Oct 2024 17:36:07 +0800 Subject: [PATCH 02/34] wip --- apis/workloads/v1/instanceset_types.go | 66 +++++++++++++++---- controllers/apps/component_controller_test.go | 27 ++++---- .../apps/transformer_component_status.go | 23 ++++++- .../apps/transformer_component_workload.go | 8 +-- controllers/k8score/event_controller_test.go | 18 ++--- pkg/controller/instanceset/suite_test.go | 36 +++++----- pkg/controller/instanceset/utils.go | 42 ++++++------ pkg/controller/instanceset/utils_test.go | 65 ++++++++++++++++++ 8 files changed, 210 insertions(+), 75 deletions(-) diff --git a/apis/workloads/v1/instanceset_types.go b/apis/workloads/v1/instanceset_types.go index b67f4a2fcd0..a6a46195a73 100644 --- a/apis/workloads/v1/instanceset_types.go +++ b/apis/workloads/v1/instanceset_types.go @@ -468,32 +468,72 @@ const ( PreferInPlacePodUpdatePolicyType PodUpdatePolicyType = "PreferInPlace" ) +// ReplicaRole represents a role that can be assigned to a component instance, defining its behavior and responsibilities. type ReplicaRole struct { - - // Defines the role name of the replica. + // Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label + // on the corresponding object to identify its role. + // + // For example, common role names include: + // - "leader": The primary/master instance that handles write operations + // - "follower": Secondary/replica instances that replicate data from the leader + // - "learner": Read-only instances that don't participate in elections + // + // This field is immutable once set. // // +kubebuilder:validation:Required - // +kubebuilder:default=leader + // +kubebuilder:validation:MaxLength=32 + // +kubebuilder:validation:Pattern=`^.*[^\s]+.*$` Name string `json:"name"` - // Specifies the service capabilities of this member. + // Required indicates if at least one replica with this role must exist for the component to be considered + // operationally running. For example, a leader role may be required for the component to function. // - // +kubebuilder:validation:Required - // +kubebuilder:default=ReadWrite - // +kubebuilder:validation:Enum={None, Readonly, ReadWrite} - AccessMode AccessMode `json:"accessMode"` + // This field is immutable once set. + // + // +kubebuilder:default=false + // +optional + Required bool `json:"required"` - // Indicates if this member has voting rights. + // UpdatePriority determines the order in which pods with different roles are updated. + // Pods are sorted by this priority (higher numbers = higher priority) and updated accordingly. + // Roles with the highest priority will be updated last. + // + // For example: + // - Leader role may have priority 2 (updated last) + // - Follower role may have priority 1 (updated before leader) + // - Learner role may have priority 0 (updated first) + // + // This field is immutable once set. // - // +kubebuilder:default=true + // +kubebuilder:default=0 + // +optional + UpdatePriority int `json:"updatePriorit"` + + // ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. + // This affects update strategies that need to maintain quorum for availability. + // + // For example, in a 5-pod component where: + // - 2 learner pods (participatesInQuorum=false) + // - 2 follower pods (participatesInQuorum=true) + // - 1 leader pod (participatesInQuorum=true) + // The quorum size would be 3 (based on the 3 participating pods), allowing parallel updates + // of 2 learners and 1 follower while maintaining quorum. + // + // This field is immutable once set. + // + // +kubebuilder:default=false // +optional - CanVote bool `json:"canVote"` + ParticipatesInQuorum bool `json:"participatesInQuorum bool"` - // Determines if this member is the leader. + // SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before + // updating or scaling in pods with this role. This is typically used for leader roles to + // ensure minimal disruption during updates. + // + // This field is immutable once set. // // +kubebuilder:default=false // +optional - IsLeader bool `json:"isLeader"` + SwitchoverBeforeUpdate bool `json:"switchoverBeforeUpdate"` } // AccessMode defines SVC access mode enums. diff --git a/controllers/apps/component_controller_test.go b/controllers/apps/component_controller_test.go index eae59b40a4a..ab7146ee206 100644 --- a/controllers/apps/component_controller_test.go +++ b/controllers/apps/component_controller_test.go @@ -1325,22 +1325,25 @@ var _ = Describe("Component Controller", func() { By("check default component roles") targetRoles := []workloads.ReplicaRole{ { - Name: "leader", - AccessMode: workloads.ReadWriteMode, - CanVote: true, - IsLeader: true, + Name: "leader", + Required: true, + SwitchoverBeforeUpdate: true, + ParticipatesInQuorum: true, + UpdatePriority: 5, }, { - Name: "follower", - AccessMode: workloads.ReadonlyMode, - CanVote: true, - IsLeader: false, + Name: "follower", + Required: false, + SwitchoverBeforeUpdate: false, + ParticipatesInQuorum: true, + UpdatePriority: 4, }, { - Name: "learner", - AccessMode: workloads.NoneMode, - CanVote: false, - IsLeader: false, + Name: "learner", + Required: false, + SwitchoverBeforeUpdate: false, + ParticipatesInQuorum: false, + UpdatePriority: 2, }, } itsKey := types.NamespacedName{ diff --git a/controllers/apps/transformer_component_status.go b/controllers/apps/transformer_component_status.go index 51b248200a1..6e75010a010 100644 --- a/controllers/apps/transformer_component_status.go +++ b/controllers/apps/transformer_component_status.go @@ -199,7 +199,28 @@ func (t *componentStatusTransformer) isWorkloadUpdated() bool { return generation == strconv.FormatInt(t.comp.Generation, 10) } -// isRunning checks if the component underlying workload is running. +// isComponentAvailable tells whether the component is basically available, ether working well or in a fragile state: +// 1. at least one pod is available +// 2. with latest revision +// 3. and with leader role label set +func (t *componentStatusTransformer) isComponentAvailable() bool { + if !t.isWorkloadUpdated() { + return false + } + if t.runningITS.Status.CurrentRevision != t.runningITS.Status.UpdateRevision { + return false + } + if t.runningITS.Status.AvailableReplicas <= 0 { + return false + } + if len(t.synthesizeComp.Roles) == 0 { + return true + } + + return instanceset.IsAllRequiredRolesExist(t.runningITS) +} + +// isRunning checks if the componentbo underlying workload is running. func (t *componentStatusTransformer) isInstanceSetRunning() bool { if t.runningITS == nil { return false diff --git a/controllers/apps/transformer_component_workload.go b/controllers/apps/transformer_component_workload.go index da69abe4d30..0d301c1eb6d 100644 --- a/controllers/apps/transformer_component_workload.go +++ b/controllers/apps/transformer_component_workload.go @@ -602,7 +602,7 @@ func (r *componentWorkloadOps) leaveMember4ScaleIn() error { if err != nil { return err } - isLeader := func(pod *corev1.Pod) bool { + needSwitchover := func(pod *corev1.Pod) bool { if pod == nil || len(pod.Labels) == 0 { return false } @@ -612,7 +612,7 @@ func (r *componentWorkloadOps) leaveMember4ScaleIn() error { } for _, replicaRole := range r.runningITS.Spec.Roles { - if roleName == replicaRole.Name && replicaRole.IsLeader { + if roleName == replicaRole.Name && replicaRole.SwitchoverBeforeUpdate { return true } } @@ -621,7 +621,7 @@ func (r *componentWorkloadOps) leaveMember4ScaleIn() error { tryToSwitchover := func(lfa lifecycle.Lifecycle, pod *corev1.Pod) error { // if pod is not leader/primary, no need to switchover - if !isLeader(pod) { + if !needSwitchover(pod) { return nil } // if HA functionality is not enabled, no need to switchover @@ -645,7 +645,7 @@ func (r *componentWorkloadOps) leaveMember4ScaleIn() error { podsToMemberLeave = append(podsToMemberLeave, pod) } for _, pod := range podsToMemberLeave { - if !(isLeader(pod) || // if the pod is leader, it needs to call switchover + if !(needSwitchover(pod) || // if the pod is leader, it needs to call switchover (r.synthesizeComp.LifecycleActions != nil && r.synthesizeComp.LifecycleActions.MemberLeave != nil)) { // if the memberLeave action is defined, it needs to call it continue } diff --git a/controllers/k8score/event_controller_test.go b/controllers/k8score/event_controller_test.go index 70ceb23bf17..0edcb25907a 100644 --- a/controllers/k8score/event_controller_test.go +++ b/controllers/k8score/event_controller_test.go @@ -143,16 +143,18 @@ var _ = Describe("Event Controller", func() { Expect(testapps.GetAndChangeObj(&testCtx, client.ObjectKeyFromObject(its), func(tmpITS *workloads.InstanceSet) { tmpITS.Spec.Roles = []workloads.ReplicaRole{ { - Name: "leader", - IsLeader: true, - AccessMode: workloads.ReadWriteMode, - CanVote: true, + Name: "leader", + Required: true, + SwitchoverBeforeUpdate: true, + ParticipatesInQuorum: true, + UpdatePriority: 5, }, { - Name: "follower", - IsLeader: false, - AccessMode: workloads.ReadonlyMode, - CanVote: true, + Name: "follower", + Required: false, + SwitchoverBeforeUpdate: false, + ParticipatesInQuorum: true, + UpdatePriority: 4, }, } })()).Should(Succeed()) diff --git a/pkg/controller/instanceset/suite_test.go b/pkg/controller/instanceset/suite_test.go index 60cbfabf695..1e0b9dcf865 100644 --- a/pkg/controller/instanceset/suite_test.go +++ b/pkg/controller/instanceset/suite_test.go @@ -75,28 +75,32 @@ var ( } roles = []workloads.ReplicaRole{ { - Name: "leader", - IsLeader: true, - CanVote: true, - AccessMode: workloads.ReadWriteMode, + Name: "leader", + Required: true, + SwitchoverBeforeUpdate: true, + ParticipatesInQuorum: true, + UpdatePriority: 5, }, { - Name: "follower", - IsLeader: false, - CanVote: true, - AccessMode: workloads.ReadonlyMode, + Name: "follower", + Required: false, + SwitchoverBeforeUpdate: false, + ParticipatesInQuorum: true, + UpdatePriority: 4, }, { - Name: "logger", - IsLeader: false, - CanVote: true, - AccessMode: workloads.NoneMode, + Name: "logger", + Required: false, + SwitchoverBeforeUpdate: false, + ParticipatesInQuorum: false, + UpdatePriority: 3, }, { - Name: "learner", - IsLeader: false, - CanVote: false, - AccessMode: workloads.ReadonlyMode, + Name: "learner", + Required: false, + SwitchoverBeforeUpdate: false, + ParticipatesInQuorum: false, + UpdatePriority: 2, }, } pod = builder.NewPodBuilder("", ""). diff --git a/pkg/controller/instanceset/utils.go b/pkg/controller/instanceset/utils.go index f60dfd80559..e5d1a1f39b2 100644 --- a/pkg/controller/instanceset/utils.go +++ b/pkg/controller/instanceset/utils.go @@ -51,21 +51,7 @@ func ComposeRolePriorityMap(roles []workloads.ReplicaRole) map[string]int { rolePriorityMap[""] = emptyPriority for _, role := range roles { roleName := strings.ToLower(role.Name) - switch { - case role.IsLeader: - rolePriorityMap[roleName] = leaderPriority - case role.CanVote: - switch role.AccessMode { - case workloads.NoneMode: - rolePriorityMap[roleName] = followerNonePriority - case workloads.ReadonlyMode: - rolePriorityMap[roleName] = followerReadonlyPriority - case workloads.ReadWriteMode: - rolePriorityMap[roleName] = followerReadWritePriority - } - default: - rolePriorityMap[roleName] = learnerPriority - } + rolePriorityMap[roleName] = role.UpdatePriority } return rolePriorityMap @@ -141,14 +127,28 @@ func IsInstanceSetReady(its *workloads.InstanceSet) bool { if its.Status.ReadyWithoutPrimary { return true } - hasLeader := false - for _, status := range membersStatus { - if status.ReplicaRole != nil && status.ReplicaRole.IsLeader { - hasLeader = true - break + + return IsAllRequiredRolesExist(its) +} + +func IsAllRequiredRolesExist(its *workloads.InstanceSet) bool { + requiredRoleExist := make(map[string]bool) + for _, role := range its.Spec.Roles { + if role.Required { + requiredRoleExist[role.Name] = false } } - return hasLeader + for _, status := range its.Status.MembersStatus { + if status.ReplicaRole != nil && status.ReplicaRole.Required { + requiredRoleExist[status.ReplicaRole.Name] = true + } + } + for _, exist := range requiredRoleExist { + if exist == false { + return false + } + } + return true } // AddAnnotationScope will add AnnotationScope defined by 'scope' to all keys in map 'annotations'. diff --git a/pkg/controller/instanceset/utils_test.go b/pkg/controller/instanceset/utils_test.go index 239d2c014c7..3d5f571a404 100644 --- a/pkg/controller/instanceset/utils_test.go +++ b/pkg/controller/instanceset/utils_test.go @@ -232,6 +232,71 @@ var _ = Describe("utils test", func() { }) }) + Context("IsAllRequiredRolesExist", func() { + It("works well", func() { + its = builder.NewInstanceSetBuilder(namespace, name). + SetRoles(roles). + GetObject() + By("no leader") + its.Status.MembersStatus = []workloads.MemberStatus{ + { + PodName: name + "-0", + ReplicaRole: &roles[1], + }, + { + PodName: name + "-1", + ReplicaRole: &roles[1], + }, + { + PodName: name + "-2", + ReplicaRole: &roles[2], + }, + } + Expect(IsAllRequiredRolesExist(its)).Should(BeFalse()) + + By("has leader") + its.Status.MembersStatus[0].ReplicaRole = &roles[1] + Expect(IsAllRequiredRolesExist(its)).Should(BeTrue()) + + By("set two required roles") + roles = []workloads.ReplicaRole{ + { + Name: "r1", + Required: true, + }, + { + Name: "r2", + Required: true, + }, + { + Name: "r3", + Required: false, + }, + } + its.Spec.Roles = roles + By("only one required role presents") + its.Status.MembersStatus = []workloads.MemberStatus{ + { + PodName: name + "-0", + ReplicaRole: &roles[0], + }, + { + PodName: name + "-1", + ReplicaRole: &roles[0], + }, + { + PodName: name + "-2", + ReplicaRole: &roles[2], + }, + } + Expect(IsAllRequiredRolesExist(its)).Should(BeFalse()) + + By("all required roles present") + its.Status.MembersStatus[1].ReplicaRole = &roles[1] + Expect(IsAllRequiredRolesExist(its)).Should(BeTrue()) + }) + }) + Context("CalculateConcurrencyReplicas function", func() { It("should work well", func() { By("concurrency = 50%, replicas = 10") From 3ddc10eb327800f3e31eaa20706ebe269dc2ee5a Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Mon, 4 Nov 2024 18:00:03 +0800 Subject: [PATCH 03/34] migrations (mostly tests) --- apis/apps/v1/componentdefinition_types.go | 4 +- apis/workloads/v1/instanceset_types.go | 2 +- .../builder/builder_instance_set_test.go | 9 ++-- .../instanceset/pod_role_event_handler.go | 3 +- .../pod_role_event_handler_test.go | 12 +++-- .../apps/cluster_instance_set_test_util.go | 52 ++++++++++++------- 6 files changed, 51 insertions(+), 31 deletions(-) diff --git a/apis/apps/v1/componentdefinition_types.go b/apis/apps/v1/componentdefinition_types.go index ea0a7c6bb21..8032e400f4e 100644 --- a/apis/apps/v1/componentdefinition_types.go +++ b/apis/apps/v1/componentdefinition_types.go @@ -1409,7 +1409,7 @@ type ReplicaRole struct { // // +kubebuilder:default=0 // +optional - UpdatePriority int `json:"updatePriorit"` + UpdatePriority int `json:"updatePriority"` // ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. // This affects update strategies that need to maintain quorum for availability. @@ -1425,7 +1425,7 @@ type ReplicaRole struct { // // +kubebuilder:default=false // +optional - ParticipatesInQuorum bool `json:"participatesInQuorum bool"` + ParticipatesInQuorum bool `json:"participatesInQuorum"` // SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before // updating or scaling in pods with this role. This is typically used for leader roles to diff --git a/apis/workloads/v1/instanceset_types.go b/apis/workloads/v1/instanceset_types.go index a6a46195a73..36ae769863a 100644 --- a/apis/workloads/v1/instanceset_types.go +++ b/apis/workloads/v1/instanceset_types.go @@ -507,7 +507,7 @@ type ReplicaRole struct { // // +kubebuilder:default=0 // +optional - UpdatePriority int `json:"updatePriorit"` + UpdatePriority int `json:"updatePriority"` // ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. // This affects update strategies that need to maintain quorum for availability. diff --git a/pkg/controller/builder/builder_instance_set_test.go b/pkg/controller/builder/builder_instance_set_test.go index fd4a3dcabcc..0f56a72f622 100644 --- a/pkg/controller/builder/builder_instance_set_test.go +++ b/pkg/controller/builder/builder_instance_set_test.go @@ -50,10 +50,11 @@ var _ = Describe("instance_set builder", func() { parallelPodManagementConcurrency := &intstr.IntOrString{Type: intstr.String, StrVal: "100%"} selectors := map[string]string{selectorKey4: selectorValue4} role := workloads.ReplicaRole{ - Name: "foo", - AccessMode: workloads.ReadWriteMode, - IsLeader: true, - CanVote: true, + Name: "foo", + Required: true, + SwitchoverBeforeUpdate: true, + ParticipatesInQuorum: true, + UpdatePriority: 1, } reconfiguration := workloads.MembershipReconfiguration{ SwitchoverAction: &workloads.Action{ diff --git a/pkg/controller/instanceset/pod_role_event_handler.go b/pkg/controller/instanceset/pod_role_event_handler.go index 26eca264bb0..e53142af9f5 100644 --- a/pkg/controller/instanceset/pod_role_event_handler.go +++ b/pkg/controller/instanceset/pod_role_event_handler.go @@ -250,7 +250,8 @@ func updatePodRoleLabel(cli client.Client, reqCtx intctrlutil.RequestCtx, switch ok { case true: pod.Labels[RoleLabelKey] = role.Name - pod.Labels[AccessModeLabelKey] = string(role.AccessMode) + // FIXME: find out why this label is needed + // pod.Labels[AccessModeLabelKey] = string(role.AccessMode) case false: delete(pod.Labels, RoleLabelKey) delete(pod.Labels, AccessModeLabelKey) diff --git a/pkg/controller/instanceset/pod_role_event_handler_test.go b/pkg/controller/instanceset/pod_role_event_handler_test.go index eb94949c9b8..ed53de3b30d 100644 --- a/pkg/controller/instanceset/pod_role_event_handler_test.go +++ b/pkg/controller/instanceset/pod_role_event_handler_test.go @@ -55,10 +55,11 @@ var _ = Describe("pod role label event handler test", func() { FieldPath: readinessProbeEventFieldPath, } role := workloads.ReplicaRole{ - Name: "leader", - AccessMode: workloads.ReadWriteMode, - IsLeader: true, - CanVote: true, + Name: "leader", + Required: true, + SwitchoverBeforeUpdate: true, + ParticipatesInQuorum: true, + UpdatePriority: 5, } By("build an expected message") @@ -96,7 +97,8 @@ var _ = Describe("pod role label event handler test", func() { Expect(pd).ShouldNot(BeNil()) Expect(pd.Labels).ShouldNot(BeNil()) Expect(pd.Labels[RoleLabelKey]).Should(Equal(role.Name)) - Expect(pd.Labels[AccessModeLabelKey]).Should(BeEquivalentTo(role.AccessMode)) + // FIXME: find out why this label is needed + // Expect(pd.Labels[AccessModeLabelKey]).Should(BeEquivalentTo(role.AccessMode)) return nil }).Times(1) k8sMock.EXPECT(). diff --git a/pkg/testutil/apps/cluster_instance_set_test_util.go b/pkg/testutil/apps/cluster_instance_set_test_util.go index 58320029bfc..a03dc7a7f6a 100644 --- a/pkg/testutil/apps/cluster_instance_set_test_util.go +++ b/pkg/testutil/apps/cluster_instance_set_test_util.go @@ -75,8 +75,20 @@ func MockInstanceSetComponent( return NewInstanceSetFactory(testCtx.DefaultNamespace, itsName, clusterName, itsCompName).SetReplicas(replicas). AddContainer(corev1.Container{Name: DefaultMySQLContainerName, Image: ApeCloudMySQLImage}). SetRoles([]workloads.ReplicaRole{ - {Name: "leader", AccessMode: workloads.ReadWriteMode, CanVote: true, IsLeader: true}, - {Name: "follower", AccessMode: workloads.ReadonlyMode, CanVote: true, IsLeader: false}, + { + Name: "leader", + Required: true, + SwitchoverBeforeUpdate: true, + ParticipatesInQuorum: true, + UpdatePriority: 5, + }, + { + Name: "follower", + Required: false, + SwitchoverBeforeUpdate: false, + ParticipatesInQuorum: true, + UpdatePriority: 4, + }, }).Create(testCtx).GetObject() } @@ -97,7 +109,7 @@ func MockInstanceSetPods( return nil } for i := range its.Spec.Roles { - if its.Spec.Roles[i].IsLeader { + if its.Spec.Roles[i].Required { return &its.Spec.Roles[i] } } @@ -108,7 +120,7 @@ func MockInstanceSetPods( return nil } for i := range its.Spec.Roles { - if !its.Spec.Roles[i].IsLeader { + if !its.Spec.Roles[i].Required { return &its.Spec.Roles[i] } } @@ -117,17 +129,18 @@ func MockInstanceSetPods( podList := make([]*corev1.Pod, getReplicas()) podNames := generatePodNames(cluster, compName) for i, pName := range podNames { - var podRole, accessMode string + var podRole string if its != nil && len(its.Spec.Roles) > 0 { if i == 0 { podRole = leaderRole.Name - accessMode = string(leaderRole.AccessMode) + // accessMode = string(leaderRole.AccessMode) } else { podRole = noneLeaderRole.Name - accessMode = string(noneLeaderRole.AccessMode) + // accessMode = string(noneLeaderRole.AccessMode) } } - pod := MockInstanceSetPod(testCtx, its, cluster.Name, compName, pName, podRole, accessMode) + // FIXME: is access mode label needed? + pod := MockInstanceSetPod(testCtx, its, cluster.Name, compName, pName, podRole, "foo") annotations := pod.Annotations if annotations == nil { annotations = make(map[string]string) @@ -273,6 +286,9 @@ func podIsReady(pod *corev1.Pod) bool { } func MockInstanceSetStatus(testCtx testutil.TestContext, cluster *appsv1.Cluster, fullCompName string) { + itsName := constant.GenerateClusterComponentName(cluster.Name, fullCompName) + its := &workloads.InstanceSet{} + gomega.Expect(testCtx.Cli.Get(testCtx.Ctx, client.ObjectKey{Name: itsName, Namespace: cluster.Namespace}, its)).Should(gomega.Succeed()) currentPodNames := generatePodNames(cluster, fullCompName) updateRevisions := map[string]string{} for _, podName := range currentPodNames { @@ -295,20 +311,20 @@ func MockInstanceSetStatus(testCtx testutil.TestContext, cluster *appsv1.Cluster if _, ok := pod.Labels[constant.RoleLabelKey]; !ok { continue } - memberStatus := workloads.MemberStatus{ - PodName: pod.Name, - ReplicaRole: &workloads.ReplicaRole{ - Name: pod.Labels[constant.RoleLabelKey], - AccessMode: workloads.AccessMode(pod.Labels[constant.AccessModeLabelKey]), - CanVote: true, - }, + role := &workloads.ReplicaRole{} + for _, r := range its.Spec.Roles { + if r.Name == pod.Labels[constant.RoleLabelKey] { + role = r.DeepCopy() + break + } } - if memberStatus.ReplicaRole.AccessMode == workloads.ReadWriteMode { - memberStatus.ReplicaRole.IsLeader = true + gomega.Expect(role).ShouldNot(gomega.BeNil()) + memberStatus := workloads.MemberStatus{ + PodName: pod.Name, + ReplicaRole: role, } newMembersStatus = append(newMembersStatus, memberStatus) } - itsName := constant.GenerateClusterComponentName(cluster.Name, fullCompName) compSpec := cluster.Spec.GetComponentByName(fullCompName) gomega.Eventually(GetAndChangeObjStatus(&testCtx, client.ObjectKey{Name: itsName, Namespace: cluster.Namespace}, func(its *workloads.InstanceSet) { its.Status.CurrentRevisions = currRevisions From caf994fbd716a36345a0b61b28778d88a591926e Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Wed, 6 Nov 2024 15:23:14 +0800 Subject: [PATCH 04/34] cmpd controller migration --- apis/apps/v1/componentdefinition_types.go | 33 ---------------- .../componentdefinition_controller_test.go | 38 +++++++++---------- pkg/controller/component/its_convertor.go | 25 +++--------- .../apps/componentdefinition_factory.go | 6 +-- pkg/testutil/apps/constant.go | 27 +++++++------ 5 files changed, 41 insertions(+), 88 deletions(-) diff --git a/apis/apps/v1/componentdefinition_types.go b/apis/apps/v1/componentdefinition_types.go index 8032e400f4e..f1a92fc485c 100644 --- a/apis/apps/v1/componentdefinition_types.go +++ b/apis/apps/v1/componentdefinition_types.go @@ -1436,39 +1436,6 @@ type ReplicaRole struct { // +kubebuilder:default=false // +optional SwitchoverBeforeUpdate bool `json:"switchoverBeforeUpdate"` - - // Indicates whether a replica assigned this role is capable of providing services. - // - // This field is immutable once set. - // - // Deprecated: use other fields - // - // +kubebuilder:default=false - // +optional - Serviceable bool `json:"serviceable,omitempty"` - - // Determines if a replica in this role has the authority to perform write operations. - // A writable replica can modify data, handle update operations. - // - // This field is immutable once set. - // - // Deprecated: use other fields - // - // +kubebuilder:default=false - // +optional - Writable bool `json:"writable,omitempty"` - - // Specifies whether a replica with this role has voting rights. - // In distributed systems, this typically means the replica can participate in consensus decisions, - // configuration changes, or other processes that require a quorum. - // - // This field is immutable once set. - // - // Deprecated: use other fields - // - // +kubebuilder:default=false - // +optional - Votable bool `json:"votable,omitempty"` } // UpdateStrategy defines the update strategy for cluster components. This strategy determines how updates are applied diff --git a/controllers/apps/componentdefinition_controller_test.go b/controllers/apps/componentdefinition_controller_test.go index 204a50169e0..16752dbbfc2 100644 --- a/controllers/apps/componentdefinition_controller_test.go +++ b/controllers/apps/componentdefinition_controller_test.go @@ -175,8 +175,8 @@ var _ = Describe("ComponentDefinition Controller", func() { SetRuntime(nil). AddService("default", "", 3306, "leader"). AddService("readonly", "readonly", 3306, "follower"). - AddRole("leader", true, true). - AddRole("follower", true, false). + AddRole("leader"). + AddRole("follower"). Create(&testCtx).GetObject() checkObjectStatus(componentDefObj, kbappsv1.AvailablePhase) @@ -188,8 +188,8 @@ var _ = Describe("ComponentDefinition Controller", func() { SetRuntime(nil). AddService("default", "", 3306, "leader"). AddService("default", "readonly", 3306, "follower"). - AddRole("leader", true, true). - AddRole("follower", true, false). + AddRole("leader"). + AddRole("follower"). Create(&testCtx).GetObject() checkObjectStatus(componentDefObj, kbappsv1.UnavailablePhase) @@ -201,8 +201,8 @@ var _ = Describe("ComponentDefinition Controller", func() { SetRuntime(nil). AddService("default", "default", 3306, "leader"). AddService("readonly", "default", 3306, "follower"). - AddRole("leader", true, true). - AddRole("follower", true, false). + AddRole("leader"). + AddRole("follower"). Create(&testCtx).GetObject() checkObjectStatus(componentDefObj, kbappsv1.UnavailablePhase) @@ -214,8 +214,8 @@ var _ = Describe("ComponentDefinition Controller", func() { SetRuntime(nil). AddService("default", "", 3306, "leader"). AddService("readonly", "", 3306, "follower"). - AddRole("leader", true, true). - AddRole("follower", true, false). + AddRole("leader"). + AddRole("follower"). Create(&testCtx).GetObject() checkObjectStatus(componentDefObj, kbappsv1.UnavailablePhase) @@ -226,8 +226,8 @@ var _ = Describe("ComponentDefinition Controller", func() { componentDefObj := testapps.NewComponentDefinitionFactory(componentDefName). SetRuntime(nil). AddServiceExt("default", "", corev1.ServiceSpec{}, "leader"). - AddRole("leader", true, true). - AddRole("follower", true, false). + AddRole("leader"). + AddRole("follower"). Create(&testCtx).GetObject() checkObjectStatus(componentDefObj, kbappsv1.UnavailablePhase) @@ -240,8 +240,8 @@ var _ = Describe("ComponentDefinition Controller", func() { AddService("default", "", 3306, "leader"). AddService("readonly", "readonly", 3306, "follower"). AddService("undefined", "undefined", 3306, "undefined"). - AddRole("leader", true, true). - AddRole("follower", true, false). + AddRole("leader"). + AddRole("follower"). Create(&testCtx).GetObject() checkObjectStatus(componentDefObj, kbappsv1.UnavailablePhase) @@ -441,9 +441,9 @@ var _ = Describe("ComponentDefinition Controller", func() { By("create a ComponentDefinition obj") componentDefObj := testapps.NewComponentDefinitionFactory(componentDefName). SetRuntime(nil). - AddRole("leader", true, true). - AddRole("follower", true, false). - AddRole("learner", false, false). + AddRole("leader"). + AddRole("follower"). + AddRole("learner"). Create(&testCtx).GetObject() checkObjectStatus(componentDefObj, kbappsv1.AvailablePhase) @@ -453,10 +453,10 @@ var _ = Describe("ComponentDefinition Controller", func() { By("create a ComponentDefinition obj") componentDefObj := testapps.NewComponentDefinitionFactory(componentDefName). SetRuntime(nil). - AddRole("leader", true, true). - AddRole("follower", true, false). - AddRole("learner", false, false). - AddRole("learner", true, false). + AddRole("leader"). + AddRole("follower"). + AddRole("learner"). + AddRole("learner"). Create(&testCtx).GetObject() checkObjectStatus(componentDefObj, kbappsv1.UnavailablePhase) diff --git a/pkg/controller/component/its_convertor.go b/pkg/controller/component/its_convertor.go index af8af9b5d16..ee3e125e129 100644 --- a/pkg/controller/component/its_convertor.go +++ b/pkg/controller/component/its_convertor.go @@ -348,29 +348,14 @@ func ConvertSynthesizeCompRoleToInstanceSetRole(synthesizedComp *SynthesizedComp return nil } - accessMode := func(role kbappsv1.ReplicaRole) workloads.AccessMode { - switch { - case role.Serviceable && role.Writable: - return workloads.ReadWriteMode - case role.Serviceable: - return workloads.ReadonlyMode - default: - return workloads.NoneMode - } - } itsReplicaRoles := make([]workloads.ReplicaRole, 0) for _, role := range synthesizedComp.Roles { itsReplicaRole := workloads.ReplicaRole{ - Name: role.Name, - AccessMode: accessMode(role), - CanVote: role.Votable, - // HACK: Since the InstanceSet relies on IsLeader field to determine whether a workload is available, we are using - // such a workaround to combine these two fields to provide the information. - // However, the condition will be broken if a service with multiple different roles that can be writable - // at the same time, such as Zookeeper. - // TODO: We need to discuss further whether we should rely on the concept of "Leader" in the case - // where the KB controller does not provide HA functionality. - IsLeader: role.Serviceable && role.Writable, + Name: role.Name, + Required: role.Required, + UpdatePriority: role.UpdatePriority, + ParticipatesInQuorum: role.ParticipatesInQuorum, + SwitchoverBeforeUpdate: role.SwitchoverBeforeUpdate, } itsReplicaRoles = append(itsReplicaRoles, itsReplicaRole) } diff --git a/pkg/testutil/apps/componentdefinition_factory.go b/pkg/testutil/apps/componentdefinition_factory.go index 0d8e586512f..98d9df85348 100644 --- a/pkg/testutil/apps/componentdefinition_factory.go +++ b/pkg/testutil/apps/componentdefinition_factory.go @@ -289,11 +289,9 @@ func (f *MockComponentDefinitionFactory) SetAvailable(available *kbappsv1.Compon return f } -func (f *MockComponentDefinitionFactory) AddRole(name string, serviceable, writable bool) *MockComponentDefinitionFactory { +func (f *MockComponentDefinitionFactory) AddRole(name string) *MockComponentDefinitionFactory { role := kbappsv1.ReplicaRole{ - Name: name, - Serviceable: serviceable, - Writable: writable, + Name: name, } if f.Get().Spec.Roles == nil { f.Get().Spec.Roles = make([]kbappsv1.ReplicaRole, 0) diff --git a/pkg/testutil/apps/constant.go b/pkg/testutil/apps/constant.go index c18fa35603d..99efa7dc1f9 100644 --- a/pkg/testutil/apps/constant.go +++ b/pkg/testutil/apps/constant.go @@ -209,22 +209,25 @@ var ( UpdateStrategy: &[]appsv1.UpdateStrategy{appsv1.BestEffortParallelStrategy}[0], Roles: []appsv1.ReplicaRole{ { - Name: "leader", - Serviceable: true, - Writable: true, - Votable: true, + Name: "leader", + Required: true, + SwitchoverBeforeUpdate: true, + ParticipatesInQuorum: true, + UpdatePriority: 5, }, { - Name: "follower", - Serviceable: true, - Writable: false, - Votable: true, + Name: "follower", + Required: false, + SwitchoverBeforeUpdate: false, + ParticipatesInQuorum: true, + UpdatePriority: 4, }, { - Name: "learner", - Serviceable: false, - Writable: false, - Votable: false, + Name: "learner", + Required: false, + SwitchoverBeforeUpdate: false, + ParticipatesInQuorum: false, + UpdatePriority: 2, }, }, Exporter: &appsv1.Exporter{ From cf64001fcefd95ac0d0e781a58ea26ea851d6376 Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Wed, 6 Nov 2024 16:14:00 +0800 Subject: [PATCH 05/34] migrate ops --- .../v1alpha1/opsrequest_validation.go | 5 +- .../apps/configuration/policy_util_test.go | 18 ++++--- .../builder/builder_component_definition.go | 25 ++++----- .../component/lifecycle/lfa_member.go | 2 +- pkg/controller/plan/restore_test.go | 17 ++++--- pkg/operations/switchover.go | 8 +-- pkg/operations/switchover_util.go | 51 +++++++++++-------- 7 files changed, 71 insertions(+), 55 deletions(-) diff --git a/apis/operations/v1alpha1/opsrequest_validation.go b/apis/operations/v1alpha1/opsrequest_validation.go index ee3eaada1bb..d27177d862b 100644 --- a/apis/operations/v1alpha1/opsrequest_validation.go +++ b/apis/operations/v1alpha1/opsrequest_validation.go @@ -795,9 +795,10 @@ func validateSwitchoverResourceList(ctx context.Context, cli client.Client, clus return targetRole, errors.New("component has no roles definition, does not support switchover") } for _, role := range roles { - if role.Serviceable && role.Writable { + // FIXME: the assumption that only one role supports switchover may change in the future + if role.SwitchoverBeforeUpdate { if targetRole != "" { - return targetRole, errors.New("componentDefinition has more than role is serviceable and writable, does not support switchover") + return targetRole, errors.New("componentDefinition has more than role that needs switchover before, does not support switchover") } targetRole = role.Name } diff --git a/controllers/apps/configuration/policy_util_test.go b/controllers/apps/configuration/policy_util_test.go index d243009c070..924009f2453 100644 --- a/controllers/apps/configuration/policy_util_test.go +++ b/controllers/apps/configuration/policy_util_test.go @@ -164,16 +164,18 @@ func newMockReconfigureParams(testName string, cli client.Client, paramOps ...Pa MinReadySeconds: 5, Roles: []appsv1.ReplicaRole{ { - Name: "leader", - Serviceable: true, - Writable: true, - Votable: true, + Name: "leader", + Required: true, + SwitchoverBeforeUpdate: true, + ParticipatesInQuorum: true, + UpdatePriority: 5, }, { - Name: "follower", - Serviceable: true, - Writable: false, - Votable: true, + Name: "follower", + Required: false, + SwitchoverBeforeUpdate: false, + ParticipatesInQuorum: true, + UpdatePriority: 4, }, }, }, diff --git a/pkg/controller/builder/builder_component_definition.go b/pkg/controller/builder/builder_component_definition.go index 92497251145..b49b050ff67 100644 --- a/pkg/controller/builder/builder_component_definition.go +++ b/pkg/controller/builder/builder_component_definition.go @@ -210,18 +210,19 @@ func (builder *ComponentDefinitionBuilder) SetUpdateStrategy(strategy *appsv1.Up return builder } -func (builder *ComponentDefinitionBuilder) AddRole(name string, serviceable, writable bool) *ComponentDefinitionBuilder { - role := appsv1.ReplicaRole{ - Name: name, - Serviceable: serviceable, - Writable: writable, - } - if builder.get().Spec.Roles == nil { - builder.get().Spec.Roles = make([]appsv1.ReplicaRole, 0) - } - builder.get().Spec.Roles = append(builder.get().Spec.Roles, role) - return builder -} +// FIXME: currently not used +// func (builder *ComponentDefinitionBuilder) AddRole(name string, serviceable, writable bool) *ComponentDefinitionBuilder { +// role := appsv1.ReplicaRole{ +// Name: name, +// Serviceable: serviceable, +// Writable: writable, +// } +// if builder.get().Spec.Roles == nil { +// builder.get().Spec.Roles = make([]appsv1.ReplicaRole, 0) +// } +// builder.get().Spec.Roles = append(builder.get().Spec.Roles, role) +// return builder +// } func (builder *ComponentDefinitionBuilder) SetLifecycleAction(name string, val interface{}) *ComponentDefinitionBuilder { obj := &builder.get().Spec.LifecycleActions diff --git a/pkg/controller/component/lifecycle/lfa_member.go b/pkg/controller/component/lifecycle/lfa_member.go index f3072069085..dafdc9a15d8 100644 --- a/pkg/controller/component/lifecycle/lfa_member.go +++ b/pkg/controller/component/lifecycle/lfa_member.go @@ -174,7 +174,7 @@ func hackParameters4Switchover(ctx context.Context, cli client.Reader, namespace func leaderRole(roles []appsv1.ReplicaRole) (string, error) { targetRole := "" for _, role := range roles { - if role.Serviceable && role.Writable { + if role.Required && role.ParticipatesInQuorum { if targetRole != "" { return "", fmt.Errorf("more than one role defined as leader: %s,%s", targetRole, role.Name) } diff --git a/pkg/controller/plan/restore_test.go b/pkg/controller/plan/restore_test.go index 2d473f0e7b2..f620ee5dec6 100644 --- a/pkg/controller/plan/restore_test.go +++ b/pkg/controller/plan/restore_test.go @@ -157,16 +157,17 @@ var _ = Describe("Restore", func() { Replicas: 1, Roles: []appsv1.ReplicaRole{ { - Name: "leader", - Serviceable: true, - Writable: true, - Votable: true, + Name: "leader", + // FIXME + // Serviceable: true, + // Writable: true, + // Votable: true, }, { - Name: "follower", - Serviceable: true, - Writable: false, - Votable: true, + Name: "follower", + // Serviceable: true, + // Writable: false, + // Votable: true, }, }, } diff --git a/pkg/operations/switchover.go b/pkg/operations/switchover.go index 1e1527bbd4b..b1e3082b977 100644 --- a/pkg/operations/switchover.go +++ b/pkg/operations/switchover.go @@ -47,8 +47,8 @@ var _ OpsHandler = switchoverOpsHandler{} // SwitchoverMessage is the OpsRequest.Status.Condition.Message for switchover. type SwitchoverMessage struct { opsv1alpha1.Switchover - OldPrimary string - Cluster string + OldPod string + Cluster string } func init() { @@ -72,13 +72,13 @@ func (r switchoverOpsHandler) ActionStartedCondition(reqCtx intctrlutil.RequestC if err != nil { return nil, err } - pod, err := getServiceableNWritablePod(reqCtx.Ctx, cli, *synthesizedComp) + pod, err := getPodToPerformSwitchover(reqCtx.Ctx, cli, synthesizedComp) if err != nil { return nil, err } switchoverMessageMap[switchover.ComponentName] = SwitchoverMessage{ Switchover: switchover, - OldPrimary: pod.Name, + OldPod: pod.Name, Cluster: opsRes.Cluster.Name, } } diff --git a/pkg/operations/switchover_util.go b/pkg/operations/switchover_util.go index 08c66d37847..4306b08b0c5 100644 --- a/pkg/operations/switchover_util.go +++ b/pkg/operations/switchover_util.go @@ -30,6 +30,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + appsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" opsv1alpha1 "github.com/apecloud/kubeblocks/apis/operations/v1alpha1" "github.com/apecloud/kubeblocks/pkg/constant" "github.com/apecloud/kubeblocks/pkg/controller/component" @@ -49,7 +50,7 @@ func needDoSwitchover(ctx context.Context, synthesizedComp *component.SynthesizedComponent, switchover *opsv1alpha1.Switchover) (bool, error) { // get the Pod object whose current role label is already serviceable and writable - pod, err := getServiceableNWritablePod(ctx, cli, *synthesizedComp) + pod, err := getPodToPerformSwitchover(ctx, cli, synthesizedComp) if err != nil { return false, err } @@ -87,7 +88,7 @@ func checkPodRoleLabelConsistency(ctx context.Context, if switchover == nil || switchoverCondition == nil { return false, nil } - pod, err := getServiceableNWritablePod(ctx, cli, synthesizedComp) + pod, err := getPodToPerformSwitchover(ctx, cli, &synthesizedComp) if err != nil { return false, err } @@ -105,7 +106,7 @@ func checkPodRoleLabelConsistency(ctx context.Context, } switch switchoverMessage.Switchover.InstanceName { case KBSwitchoverCandidateInstanceForAnyPod: - if pod.Name != switchoverMessage.OldPrimary { + if pod.Name != switchoverMessage.OldPod { return true, nil } default: @@ -117,25 +118,16 @@ func checkPodRoleLabelConsistency(ctx context.Context, return false, nil } -// getServiceableNWritablePod returns the serviceable and writable pod of the component. -func getServiceableNWritablePod(ctx context.Context, cli client.Reader, synthesizeComp component.SynthesizedComponent) (*corev1.Pod, error) { - if synthesizeComp.Roles == nil { - return nil, errors.New("component does not support switchover") - } - - targetRole := "" - for _, role := range synthesizeComp.Roles { - if role.Serviceable && role.Writable { - if targetRole != "" { - return nil, errors.New("component has more than role is serviceable and writable, does not support switchover") - } - targetRole = role.Name - } - } - if targetRole == "" { - return nil, errors.New("component has no role is serviceable and writable, does not support switchover") +func getPodToPerformSwitchover(ctx context.Context, cli client.Reader, synthesizedComp *component.SynthesizedComponent) (*corev1.Pod, error) { + role, err := getTargetRoleName(synthesizedComp.Roles) + if err != nil { + return nil, err } + pod, err := getPodByRole(ctx, cli, synthesizedComp, role) + return pod, err +} +func getPodByRole(ctx context.Context, cli client.Reader, synthesizeComp *component.SynthesizedComponent, targetRole string) (*corev1.Pod, error) { pods, err := component.ListOwnedPodsWithRole(ctx, cli, synthesizeComp.Namespace, synthesizeComp.ClusterName, synthesizeComp.Name, targetRole) if err != nil { return nil, err @@ -145,3 +137,22 @@ func getServiceableNWritablePod(ctx context.Context, cli client.Reader, synthesi } return pods[0], nil } + +// getTargetRole returns the role on which the switchover is performed +// FIXME: the assumption that only one role supports switchover may change in the future +func getTargetRoleName(roles []appsv1.ReplicaRole) (string, error) { + targetRole := "" + if len(roles) == 0 { + return targetRole, errors.New("component has no roles definition, does not support switchover") + } + for _, role := range roles { + // FIXME: the assumption that only one role supports switchover may change in the future + if role.SwitchoverBeforeUpdate { + if targetRole != "" { + return targetRole, errors.New("componentDefinition has more than role that needs switchover before, does not support switchover") + } + targetRole = role.Name + } + } + return targetRole, nil +} From f94842c3e9cf28107f92adf996a5440dcdccb196 Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Wed, 6 Nov 2024 16:38:50 +0800 Subject: [PATCH 06/34] fix unit test --- apis/workloads/v1/instanceset_types.go | 1 + ...ps.kubeblocks.io_componentdefinitions.yaml | 53 ++++-- .../workloads.kubeblocks.io_instancesets.yaml | 167 ++++++++++++++---- ...ps.kubeblocks.io_componentdefinitions.yaml | 53 ++++-- .../workloads.kubeblocks.io_instancesets.yaml | 167 ++++++++++++++---- pkg/controller/instanceset/update_plan.go | 14 +- .../instanceset/update_plan_test.go | 5 +- pkg/controller/instanceset/utils.go | 14 +- pkg/controller/instanceset/utils_test.go | 12 +- 9 files changed, 370 insertions(+), 116 deletions(-) diff --git a/apis/workloads/v1/instanceset_types.go b/apis/workloads/v1/instanceset_types.go index 36ae769863a..8a797edb89f 100644 --- a/apis/workloads/v1/instanceset_types.go +++ b/apis/workloads/v1/instanceset_types.go @@ -497,6 +497,7 @@ type ReplicaRole struct { // UpdatePriority determines the order in which pods with different roles are updated. // Pods are sorted by this priority (higher numbers = higher priority) and updated accordingly. // Roles with the highest priority will be updated last. + // The default priority is 0. // // For example: // - Leader role may have priority 2 (updated last) diff --git a/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml b/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml index d43b2f699de..a82ea9844ff 100644 --- a/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml +++ b/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml @@ -8477,41 +8477,72 @@ spec: properties: name: description: |- - Defines the role's identifier. It is used to set the "apps.kubeblocks.io/role" label value - on the corresponding object. + Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label + on the corresponding object to identify its role. + + + For example, common role names include: + - "leader": The primary/master instance that handles write operations + - "follower": Secondary/replica instances that replicate data from the leader + - "learner": Read-only instances that don't participate in elections This field is immutable once set. maxLength: 32 pattern: ^.*[^\s]+.*$ type: string - serviceable: + participatesInQuorum: default: false description: |- - Indicates whether a replica assigned this role is capable of providing services. + ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. + This affects update strategies that need to maintain quorum for availability. + + + For example, in a 5-pod component where: + - 2 learner pods (participatesInQuorum=false) + - 2 follower pods (participatesInQuorum=true) + - 1 leader pod (participatesInQuorum=true) + The quorum size would be 3 (based on the 3 participating pods), allowing parallel updates + of 2 learners and 1 follower while maintaining quorum. This field is immutable once set. type: boolean - votable: + required: default: false description: |- - Specifies whether a replica with this role has voting rights. - In distributed systems, this typically means the replica can participate in consensus decisions, - configuration changes, or other processes that require a quorum. + Required indicates if at least one replica with this role must exist for the component to be considered + operationally running. For example, a leader role may be required for the component to function. This field is immutable once set. type: boolean - writable: + switchoverBeforeUpdate: default: false description: |- - Determines if a replica in this role has the authority to perform write operations. - A writable replica can modify data, handle update operations. + SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before + updating or scaling in pods with this role. This is typically used for leader roles to + ensure minimal disruption during updates. This field is immutable once set. type: boolean + updatePriority: + default: 0 + description: |- + UpdatePriority determines the order in which pods with different roles are updated. + Pods are sorted by this priority (higher numbers = higher priority) and updated accordingly. + Roles with the highest priority will be updated last. + + + For example: + - Leader role may have priority 2 (updated last) + - Follower role may have priority 1 (updated before leader) + - Learner role may have priority 0 (updated first) + + + This field is immutable once set. + type: integer required: - name type: object diff --git a/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml b/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml index 3da51ffb9f8..a97b11d6267 100644 --- a/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml +++ b/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml @@ -4256,29 +4256,79 @@ spec: roles: description: A list of roles defined in the system. items: + description: ReplicaRole represents a role that can be assigned + to a component instance, defining its behavior and responsibilities. properties: - accessMode: - default: ReadWrite - description: Specifies the service capabilities of this member. - enum: - - None - - Readonly - - ReadWrite + name: + description: |- + Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label + on the corresponding object to identify its role. + + + For example, common role names include: + - "leader": The primary/master instance that handles write operations + - "follower": Secondary/replica instances that replicate data from the leader + - "learner": Read-only instances that don't participate in elections + + + This field is immutable once set. + maxLength: 32 + pattern: ^.*[^\s]+.*$ type: string - canVote: - default: true - description: Indicates if this member has voting rights. + participatesInQuorum bool: + default: false + description: |- + ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. + This affects update strategies that need to maintain quorum for availability. + + + For example, in a 5-pod component where: + - 2 learner pods (participatesInQuorum=false) + - 2 follower pods (participatesInQuorum=true) + - 1 leader pod (participatesInQuorum=true) + The quorum size would be 3 (based on the 3 participating pods), allowing parallel updates + of 2 learners and 1 follower while maintaining quorum. + + + This field is immutable once set. type: boolean - isLeader: + required: default: false - description: Determines if this member is the leader. + description: |- + Required indicates if at least one replica with this role must exist for the component to be considered + operationally running. For example, a leader role may be required for the component to function. + + + This field is immutable once set. type: boolean - name: - default: leader - description: Defines the role name of the replica. - type: string + switchoverBeforeUpdate: + default: false + description: |- + SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before + updating or scaling in pods with this role. This is typically used for leader roles to + ensure minimal disruption during updates. + + + This field is immutable once set. + type: boolean + updatePriority: + default: 0 + description: |- + UpdatePriority determines the order in which pods with different roles are updated. + Pods are sorted by this priority (higher numbers = higher priority) and updated accordingly. + Roles with the highest priority will be updated last. + The default priority is 0. + + + For example: + - Leader role may have priority 2 (updated last) + - Follower role may have priority 1 (updated before leader) + - Learner role may have priority 0 (updated first) + + + This field is immutable once set. + type: integer required: - - accessMode - name type: object type: array @@ -12430,29 +12480,76 @@ spec: role: description: Defines the role of the replica in the cluster. properties: - accessMode: - default: ReadWrite - description: Specifies the service capabilities of this - member. - enum: - - None - - Readonly - - ReadWrite + name: + description: |- + Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label + on the corresponding object to identify its role. + + + For example, common role names include: + - "leader": The primary/master instance that handles write operations + - "follower": Secondary/replica instances that replicate data from the leader + - "learner": Read-only instances that don't participate in elections + + + This field is immutable once set. + maxLength: 32 + pattern: ^.*[^\s]+.*$ type: string - canVote: - default: true - description: Indicates if this member has voting rights. + participatesInQuorum bool: + default: false + description: |- + ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. + This affects update strategies that need to maintain quorum for availability. + + + For example, in a 5-pod component where: + - 2 learner pods (participatesInQuorum=false) + - 2 follower pods (participatesInQuorum=true) + - 1 leader pod (participatesInQuorum=true) + The quorum size would be 3 (based on the 3 participating pods), allowing parallel updates + of 2 learners and 1 follower while maintaining quorum. + + + This field is immutable once set. type: boolean - isLeader: + required: default: false - description: Determines if this member is the leader. + description: |- + Required indicates if at least one replica with this role must exist for the component to be considered + operationally running. For example, a leader role may be required for the component to function. + + + This field is immutable once set. type: boolean - name: - default: leader - description: Defines the role name of the replica. - type: string + switchoverBeforeUpdate: + default: false + description: |- + SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before + updating or scaling in pods with this role. This is typically used for leader roles to + ensure minimal disruption during updates. + + + This field is immutable once set. + type: boolean + updatePriority: + default: 0 + description: |- + UpdatePriority determines the order in which pods with different roles are updated. + Pods are sorted by this priority (higher numbers = higher priority) and updated accordingly. + Roles with the highest priority will be updated last. + The default priority is 0. + + + For example: + - Leader role may have priority 2 (updated last) + - Follower role may have priority 1 (updated before leader) + - Learner role may have priority 0 (updated first) + + + This field is immutable once set. + type: integer required: - - accessMode - name type: object required: diff --git a/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml b/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml index d43b2f699de..a82ea9844ff 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml @@ -8477,41 +8477,72 @@ spec: properties: name: description: |- - Defines the role's identifier. It is used to set the "apps.kubeblocks.io/role" label value - on the corresponding object. + Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label + on the corresponding object to identify its role. + + + For example, common role names include: + - "leader": The primary/master instance that handles write operations + - "follower": Secondary/replica instances that replicate data from the leader + - "learner": Read-only instances that don't participate in elections This field is immutable once set. maxLength: 32 pattern: ^.*[^\s]+.*$ type: string - serviceable: + participatesInQuorum: default: false description: |- - Indicates whether a replica assigned this role is capable of providing services. + ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. + This affects update strategies that need to maintain quorum for availability. + + + For example, in a 5-pod component where: + - 2 learner pods (participatesInQuorum=false) + - 2 follower pods (participatesInQuorum=true) + - 1 leader pod (participatesInQuorum=true) + The quorum size would be 3 (based on the 3 participating pods), allowing parallel updates + of 2 learners and 1 follower while maintaining quorum. This field is immutable once set. type: boolean - votable: + required: default: false description: |- - Specifies whether a replica with this role has voting rights. - In distributed systems, this typically means the replica can participate in consensus decisions, - configuration changes, or other processes that require a quorum. + Required indicates if at least one replica with this role must exist for the component to be considered + operationally running. For example, a leader role may be required for the component to function. This field is immutable once set. type: boolean - writable: + switchoverBeforeUpdate: default: false description: |- - Determines if a replica in this role has the authority to perform write operations. - A writable replica can modify data, handle update operations. + SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before + updating or scaling in pods with this role. This is typically used for leader roles to + ensure minimal disruption during updates. This field is immutable once set. type: boolean + updatePriority: + default: 0 + description: |- + UpdatePriority determines the order in which pods with different roles are updated. + Pods are sorted by this priority (higher numbers = higher priority) and updated accordingly. + Roles with the highest priority will be updated last. + + + For example: + - Leader role may have priority 2 (updated last) + - Follower role may have priority 1 (updated before leader) + - Learner role may have priority 0 (updated first) + + + This field is immutable once set. + type: integer required: - name type: object diff --git a/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml b/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml index 3da51ffb9f8..a97b11d6267 100644 --- a/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml +++ b/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml @@ -4256,29 +4256,79 @@ spec: roles: description: A list of roles defined in the system. items: + description: ReplicaRole represents a role that can be assigned + to a component instance, defining its behavior and responsibilities. properties: - accessMode: - default: ReadWrite - description: Specifies the service capabilities of this member. - enum: - - None - - Readonly - - ReadWrite + name: + description: |- + Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label + on the corresponding object to identify its role. + + + For example, common role names include: + - "leader": The primary/master instance that handles write operations + - "follower": Secondary/replica instances that replicate data from the leader + - "learner": Read-only instances that don't participate in elections + + + This field is immutable once set. + maxLength: 32 + pattern: ^.*[^\s]+.*$ type: string - canVote: - default: true - description: Indicates if this member has voting rights. + participatesInQuorum bool: + default: false + description: |- + ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. + This affects update strategies that need to maintain quorum for availability. + + + For example, in a 5-pod component where: + - 2 learner pods (participatesInQuorum=false) + - 2 follower pods (participatesInQuorum=true) + - 1 leader pod (participatesInQuorum=true) + The quorum size would be 3 (based on the 3 participating pods), allowing parallel updates + of 2 learners and 1 follower while maintaining quorum. + + + This field is immutable once set. type: boolean - isLeader: + required: default: false - description: Determines if this member is the leader. + description: |- + Required indicates if at least one replica with this role must exist for the component to be considered + operationally running. For example, a leader role may be required for the component to function. + + + This field is immutable once set. type: boolean - name: - default: leader - description: Defines the role name of the replica. - type: string + switchoverBeforeUpdate: + default: false + description: |- + SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before + updating or scaling in pods with this role. This is typically used for leader roles to + ensure minimal disruption during updates. + + + This field is immutable once set. + type: boolean + updatePriority: + default: 0 + description: |- + UpdatePriority determines the order in which pods with different roles are updated. + Pods are sorted by this priority (higher numbers = higher priority) and updated accordingly. + Roles with the highest priority will be updated last. + The default priority is 0. + + + For example: + - Leader role may have priority 2 (updated last) + - Follower role may have priority 1 (updated before leader) + - Learner role may have priority 0 (updated first) + + + This field is immutable once set. + type: integer required: - - accessMode - name type: object type: array @@ -12430,29 +12480,76 @@ spec: role: description: Defines the role of the replica in the cluster. properties: - accessMode: - default: ReadWrite - description: Specifies the service capabilities of this - member. - enum: - - None - - Readonly - - ReadWrite + name: + description: |- + Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label + on the corresponding object to identify its role. + + + For example, common role names include: + - "leader": The primary/master instance that handles write operations + - "follower": Secondary/replica instances that replicate data from the leader + - "learner": Read-only instances that don't participate in elections + + + This field is immutable once set. + maxLength: 32 + pattern: ^.*[^\s]+.*$ type: string - canVote: - default: true - description: Indicates if this member has voting rights. + participatesInQuorum bool: + default: false + description: |- + ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. + This affects update strategies that need to maintain quorum for availability. + + + For example, in a 5-pod component where: + - 2 learner pods (participatesInQuorum=false) + - 2 follower pods (participatesInQuorum=true) + - 1 leader pod (participatesInQuorum=true) + The quorum size would be 3 (based on the 3 participating pods), allowing parallel updates + of 2 learners and 1 follower while maintaining quorum. + + + This field is immutable once set. type: boolean - isLeader: + required: default: false - description: Determines if this member is the leader. + description: |- + Required indicates if at least one replica with this role must exist for the component to be considered + operationally running. For example, a leader role may be required for the component to function. + + + This field is immutable once set. type: boolean - name: - default: leader - description: Defines the role name of the replica. - type: string + switchoverBeforeUpdate: + default: false + description: |- + SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before + updating or scaling in pods with this role. This is typically used for leader roles to + ensure minimal disruption during updates. + + + This field is immutable once set. + type: boolean + updatePriority: + default: 0 + description: |- + UpdatePriority determines the order in which pods with different roles are updated. + Pods are sorted by this priority (higher numbers = higher priority) and updated accordingly. + Roles with the highest priority will be updated last. + The default priority is 0. + + + For example: + - Leader role may have priority 2 (updated last) + - Follower role may have priority 1 (updated before leader) + - Learner role may have priority 0 (updated first) + + + This field is immutable once set. + type: integer required: - - accessMode - name type: object required: diff --git a/pkg/controller/instanceset/update_plan.go b/pkg/controller/instanceset/update_plan.go index 7c05e6aa60a..a9201da76ed 100644 --- a/pkg/controller/instanceset/update_plan.go +++ b/pkg/controller/instanceset/update_plan.go @@ -21,6 +21,7 @@ package instanceset import ( "errors" + "math" corev1 "k8s.io/api/core/v1" @@ -144,12 +145,23 @@ func (p *realUpdatePlan) buildBestEffortParallelUpdatePlan(rolePriorityMap map[s currentVertex, _ := model.FindRootVertex(p.dag) preVertex := currentVertex + quorumPriority := math.MaxInt32 + leaderPriority := 0 + for _, role := range p.its.Spec.Roles { + if rolePriorityMap[role.Name] > leaderPriority { + leaderPriority = rolePriorityMap[role.Name] + } + if role.ParticipatesInQuorum && quorumPriority > rolePriorityMap[role.Name] { + quorumPriority = rolePriorityMap[role.Name] + } + } + // append unknown, empty and learner index := 0 podList := p.pods for i, pod := range podList { roleName := getRoleName(&pod) - if rolePriorityMap[roleName] <= learnerPriority { + if rolePriorityMap[roleName] < quorumPriority { vertex := &model.ObjectVertex{Obj: &podList[i]} p.dag.AddConnect(preVertex, vertex) currentVertex = vertex diff --git a/pkg/controller/instanceset/update_plan_test.go b/pkg/controller/instanceset/update_plan_test.go index a789130d9d5..3ebd636219d 100644 --- a/pkg/controller/instanceset/update_plan_test.go +++ b/pkg/controller/instanceset/update_plan_test.go @@ -161,13 +161,12 @@ var _ = Describe("update plan test.", func() { checkPlan(expectedPlan, true) }) - It("should work well in a best effort parallel", func() { + FIt("should work well in a best effort parallel", func() { By("build a best effort parallel plan") strategy := workloads.BestEffortParallelUpdateStrategy its.Spec.MemberUpdateStrategy = &strategy expectedPlan := [][]*corev1.Pod{ - {pod2, pod3, pod4, pod6}, - {pod1}, + {pod2, pod3, pod4, pod6, pod1}, {pod0}, {pod5}, } diff --git a/pkg/controller/instanceset/utils.go b/pkg/controller/instanceset/utils.go index e5d1a1f39b2..3b346b643b5 100644 --- a/pkg/controller/instanceset/utils.go +++ b/pkg/controller/instanceset/utils.go @@ -35,20 +35,12 @@ import ( "github.com/apecloud/kubeblocks/pkg/constant" ) -const ( - leaderPriority = 1 << 5 - followerReadWritePriority = 1 << 4 - followerReadonlyPriority = 1 << 3 - followerNonePriority = 1 << 2 - learnerPriority = 1 << 1 - emptyPriority = 1 << 0 - // unknownPriority = 0 -) +const defaultPriority = 0 // ComposeRolePriorityMap generates a priority map based on roles. func ComposeRolePriorityMap(roles []workloads.ReplicaRole) map[string]int { rolePriorityMap := make(map[string]int) - rolePriorityMap[""] = emptyPriority + rolePriorityMap[""] = defaultPriority for _, role := range roles { roleName := strings.ToLower(role.Name) rolePriorityMap[roleName] = role.UpdatePriority @@ -144,7 +136,7 @@ func IsAllRequiredRolesExist(its *workloads.InstanceSet) bool { } } for _, exist := range requiredRoleExist { - if exist == false { + if !exist { return false } } diff --git a/pkg/controller/instanceset/utils_test.go b/pkg/controller/instanceset/utils_test.go index 3d5f571a404..bc8ac69f8ea 100644 --- a/pkg/controller/instanceset/utils_test.go +++ b/pkg/controller/instanceset/utils_test.go @@ -62,16 +62,10 @@ var _ = Describe("utils test", func() { Context("ComposeRolePriorityMap function", func() { It("should work well", func() { - priorityList := []int{ - leaderPriority, - followerReadonlyPriority, - followerNonePriority, - learnerPriority, - } Expect(priorityMap).ShouldNot(BeZero()) Expect(priorityMap).Should(HaveLen(len(roles) + 1)) - for i, role := range roles { - Expect(priorityMap[role.Name]).Should(Equal(priorityList[i])) + for _, role := range roles { + Expect(priorityMap[role.Name]).Should(Equal(role.UpdatePriority)) } }) }) @@ -255,7 +249,7 @@ var _ = Describe("utils test", func() { Expect(IsAllRequiredRolesExist(its)).Should(BeFalse()) By("has leader") - its.Status.MembersStatus[0].ReplicaRole = &roles[1] + its.Status.MembersStatus[0].ReplicaRole = &roles[0] Expect(IsAllRequiredRolesExist(its)).Should(BeTrue()) By("set two required roles") From 16567e097c955d662e813dd7e2c8954e043cea8f Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Wed, 6 Nov 2024 16:47:23 +0800 Subject: [PATCH 07/34] use type alias --- apis/apps/v1/componentdefinition_types.go | 70 ++----------------- apis/apps/v1/zz_generated.deepcopy.go | 18 +---- ...ps.kubeblocks.io_componentdefinitions.yaml | 7 +- ...ps.kubeblocks.io_componentdefinitions.yaml | 7 +- 4 files changed, 14 insertions(+), 88 deletions(-) diff --git a/apis/apps/v1/componentdefinition_types.go b/apis/apps/v1/componentdefinition_types.go index f1a92fc485c..55736fa4abd 100644 --- a/apis/apps/v1/componentdefinition_types.go +++ b/apis/apps/v1/componentdefinition_types.go @@ -26,6 +26,8 @@ import ( corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + workloads "github.com/apecloud/kubeblocks/apis/workloads/v1" ) // +genclient @@ -1371,72 +1373,8 @@ type ComponentAvailableProbeAssertion struct { } // ReplicaRole represents a role that can be assumed by a component instance. -type ReplicaRole struct { - // Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label - // on the corresponding object to identify its role. - // - // For example, common role names include: - // - "leader": The primary/master instance that handles write operations - // - "follower": Secondary/replica instances that replicate data from the leader - // - "learner": Read-only instances that don't participate in elections - // - // This field is immutable once set. - // - // +kubebuilder:validation:Required - // +kubebuilder:validation:MaxLength=32 - // +kubebuilder:validation:Pattern=`^.*[^\s]+.*$` - Name string `json:"name"` - - // Required indicates if at least one replica with this role must exist for the component to be considered - // operationally running. For example, a leader role may be required for the component to function. - // - // This field is immutable once set. - // - // +kubebuilder:default=false - // +optional - Required bool `json:"required"` - - // UpdatePriority determines the order in which pods with different roles are updated. - // Pods are sorted by this priority (higher numbers = higher priority) and updated accordingly. - // Roles with the highest priority will be updated last. - // - // For example: - // - Leader role may have priority 2 (updated last) - // - Follower role may have priority 1 (updated before leader) - // - Learner role may have priority 0 (updated first) - // - // This field is immutable once set. - // - // +kubebuilder:default=0 - // +optional - UpdatePriority int `json:"updatePriority"` - - // ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. - // This affects update strategies that need to maintain quorum for availability. - // - // For example, in a 5-pod component where: - // - 2 learner pods (participatesInQuorum=false) - // - 2 follower pods (participatesInQuorum=true) - // - 1 leader pod (participatesInQuorum=true) - // The quorum size would be 3 (based on the 3 participating pods), allowing parallel updates - // of 2 learners and 1 follower while maintaining quorum. - // - // This field is immutable once set. - // - // +kubebuilder:default=false - // +optional - ParticipatesInQuorum bool `json:"participatesInQuorum"` - - // SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before - // updating or scaling in pods with this role. This is typically used for leader roles to - // ensure minimal disruption during updates. - // - // This field is immutable once set. - // - // +kubebuilder:default=false - // +optional - SwitchoverBeforeUpdate bool `json:"switchoverBeforeUpdate"` -} +// +kubebuilder:object:generate=false +type ReplicaRole = workloads.ReplicaRole // UpdateStrategy defines the update strategy for cluster components. This strategy determines how updates are applied // across the cluster. diff --git a/apis/apps/v1/zz_generated.deepcopy.go b/apis/apps/v1/zz_generated.deepcopy.go index 8c0d243f383..df777861d5c 100644 --- a/apis/apps/v1/zz_generated.deepcopy.go +++ b/apis/apps/v1/zz_generated.deepcopy.go @@ -24,6 +24,7 @@ along with this program. If not, see . package v1 import ( + workloadsv1 "github.com/apecloud/kubeblocks/apis/workloads/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -1188,7 +1189,7 @@ func (in *ComponentDefinitionSpec) DeepCopyInto(out *ComponentDefinitionSpec) { } if in.Roles != nil { in, out := &in.Roles, &out.Roles - *out = make([]ReplicaRole, len(*in)) + *out = make([]workloadsv1.ReplicaRole, len(*in)) copy(*out, *in) } if in.UpdateStrategy != nil { @@ -2380,21 +2381,6 @@ func (in *ProvisionSecretRef) DeepCopy() *ProvisionSecretRef { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ReplicaRole) DeepCopyInto(out *ReplicaRole) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicaRole. -func (in *ReplicaRole) DeepCopy() *ReplicaRole { - if in == nil { - return nil - } - out := new(ReplicaRole) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ReplicasLimit) DeepCopyInto(out *ReplicasLimit) { *out = *in diff --git a/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml b/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml index a82ea9844ff..02419b70c62 100644 --- a/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml +++ b/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml @@ -8472,8 +8472,8 @@ spec: This field is immutable. items: - description: ReplicaRole represents a role that can be assumed by - a component instance. + description: ReplicaRole represents a role that can be assigned + to a component instance, defining its behavior and responsibilities. properties: name: description: |- @@ -8491,7 +8491,7 @@ spec: maxLength: 32 pattern: ^.*[^\s]+.*$ type: string - participatesInQuorum: + participatesInQuorum bool: default: false description: |- ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. @@ -8533,6 +8533,7 @@ spec: UpdatePriority determines the order in which pods with different roles are updated. Pods are sorted by this priority (higher numbers = higher priority) and updated accordingly. Roles with the highest priority will be updated last. + The default priority is 0. For example: diff --git a/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml b/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml index a82ea9844ff..02419b70c62 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml @@ -8472,8 +8472,8 @@ spec: This field is immutable. items: - description: ReplicaRole represents a role that can be assumed by - a component instance. + description: ReplicaRole represents a role that can be assigned + to a component instance, defining its behavior and responsibilities. properties: name: description: |- @@ -8491,7 +8491,7 @@ spec: maxLength: 32 pattern: ^.*[^\s]+.*$ type: string - participatesInQuorum: + participatesInQuorum bool: default: false description: |- ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. @@ -8533,6 +8533,7 @@ spec: UpdatePriority determines the order in which pods with different roles are updated. Pods are sorted by this priority (higher numbers = higher priority) and updated accordingly. Roles with the highest priority will be updated last. + The default priority is 0. For example: From 2a2966d0a3517913fb0260735bc39c3043182325 Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Wed, 6 Nov 2024 17:48:18 +0800 Subject: [PATCH 08/34] typo --- controllers/apps/transformer_component_status.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/apps/transformer_component_status.go b/controllers/apps/transformer_component_status.go index 6e75010a010..1d982d1d6f3 100644 --- a/controllers/apps/transformer_component_status.go +++ b/controllers/apps/transformer_component_status.go @@ -220,7 +220,7 @@ func (t *componentStatusTransformer) isComponentAvailable() bool { return instanceset.IsAllRequiredRolesExist(t.runningITS) } -// isRunning checks if the componentbo underlying workload is running. +// isRunning checks if the component's underlying workload is running. func (t *componentStatusTransformer) isInstanceSetRunning() bool { if t.runningITS == nil { return false From 4285d88c4611d4f9d6fc7a32236cb1d8db52b7ba Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Wed, 6 Nov 2024 23:47:56 +0800 Subject: [PATCH 09/34] wip --- docs/developer_docs/api-reference/cluster.md | 141 +++++++----------- .../apps/cluster_instance_set_test_util.go | 2 +- 2 files changed, 53 insertions(+), 90 deletions(-) diff --git a/docs/developer_docs/api-reference/cluster.md b/docs/developer_docs/api-reference/cluster.md index b14e2d0f76a..5bcb31a6d9b 100644 --- a/docs/developer_docs/api-reference/cluster.md +++ b/docs/developer_docs/api-reference/cluster.md @@ -1346,7 +1346,7 @@ ComponentAvailable roles
- + []ReplicaRole @@ -5004,7 +5004,7 @@ ComponentAvailable roles
- + []ReplicaRole @@ -8501,79 +8501,6 @@ string -

ReplicaRole -

-

-(Appears on:ComponentDefinitionSpec) -

-
-

ReplicaRole represents a role that can be assumed by a component instance.

-
- - - - - - - - - - - - - - - - - - - - - - - - - -
FieldDescription
-name
- -string - -		 -

Defines the role’s identifier. It is used to set the “apps.kubeblocks.io/role” label value -on the corresponding object.

-

This field is immutable once set.

-
-serviceable
- -bool - -		 -(Optional) -

Indicates whether a replica assigned this role is capable of providing services.

-

This field is immutable once set.

-
-writable
- -bool - -		 -(Optional) -

Determines if a replica in this role has the authority to perform write operations. -A writable replica can modify data, handle update operations.

-

This field is immutable once set.

-
-votable
- -bool - -		 -(Optional) -

Specifies whether a replica with this role has voting rights. -In distributed systems, this typically means the replica can participate in consensus decisions, -configuration changes, or other processes that require a quorum.

-

This field is immutable once set.

-

ReplicasLimit

@@ -28363,9 +28290,6 @@ InstanceSetStatus

AccessMode (string alias)

-

-(Appears on:ReplicaRole) -

AccessMode defines SVC access mode enums.

@@ -29652,9 +29576,10 @@ int32

ReplicaRole

-(Appears on:InstanceSetSpec, MemberStatus) +(Appears on:ComponentDefinitionSpec, InstanceSetSpec, MemberStatus)

+

ReplicaRole represents a role that can be assigned to a component instance, defining its behavior and responsibilities.

@@ -29672,44 +29597,82 @@ string + + + + diff --git a/pkg/testutil/apps/cluster_instance_set_test_util.go b/pkg/testutil/apps/cluster_instance_set_test_util.go index a03dc7a7f6a..61924d5ec47 100644 --- a/pkg/testutil/apps/cluster_instance_set_test_util.go +++ b/pkg/testutil/apps/cluster_instance_set_test_util.go @@ -178,7 +178,7 @@ func MockInstanceSetPod( AddAppComponentLabel(consensusCompName). AddAppManagedByLabel(). AddRoleLabel(podRole). - AddAccessModeLabel(accessMode). + // AddAccessModeLabel(accessMode). AddControllerRevisionHashLabel(stsUpdateRevision). AddLabelsInMap(ml). AddVolume(corev1.Volume{ From 589adca8e00d92c6cd6382c7a8b840837c569964 Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Thu, 7 Nov 2024 00:02:24 +0800 Subject: [PATCH 10/34] remove unused function --- .../apps/transformer_component_status.go | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/controllers/apps/transformer_component_status.go b/controllers/apps/transformer_component_status.go index 1d982d1d6f3..3f0a085edf7 100644 --- a/controllers/apps/transformer_component_status.go +++ b/controllers/apps/transformer_component_status.go @@ -199,27 +199,6 @@ func (t *componentStatusTransformer) isWorkloadUpdated() bool { return generation == strconv.FormatInt(t.comp.Generation, 10) } -// isComponentAvailable tells whether the component is basically available, ether working well or in a fragile state: -// 1. at least one pod is available -// 2. with latest revision -// 3. and with leader role label set -func (t *componentStatusTransformer) isComponentAvailable() bool { - if !t.isWorkloadUpdated() { - return false - } - if t.runningITS.Status.CurrentRevision != t.runningITS.Status.UpdateRevision { - return false - } - if t.runningITS.Status.AvailableReplicas <= 0 { - return false - } - if len(t.synthesizeComp.Roles) == 0 { - return true - } - - return instanceset.IsAllRequiredRolesExist(t.runningITS) -} - // isRunning checks if the component's underlying workload is running. func (t *componentStatusTransformer) isInstanceSetRunning() bool { if t.runningITS == nil { From 027c6d9d59b4d1355a13ff67f442da6c60ea819b Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Thu, 7 Nov 2024 11:31:04 +0800 Subject: [PATCH 11/34] fix --- pkg/controller/instanceset/update_plan_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/instanceset/update_plan_test.go b/pkg/controller/instanceset/update_plan_test.go index 3ebd636219d..9cbf0e58cc4 100644 --- a/pkg/controller/instanceset/update_plan_test.go +++ b/pkg/controller/instanceset/update_plan_test.go @@ -161,7 +161,7 @@ var _ = Describe("update plan test.", func() { checkPlan(expectedPlan, true) }) - FIt("should work well in a best effort parallel", func() { + It("should work well in a best effort parallel", func() { By("build a best effort parallel plan") strategy := workloads.BestEffortParallelUpdateStrategy its.Spec.MemberUpdateStrategy = &strategy From 40a77d5098a0faf1cfbaa9bcf206e0bf3bd64a56 Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Fri, 8 Nov 2024 15:13:12 +0800 Subject: [PATCH 12/34] remove required field, move replicarole to base package --- apis/apps/v1/componentdefinition_types.go | 5 +- apis/apps/v1/zz_generated.deepcopy.go | 4 +- apis/base/v1/groupversion_info.go | 39 ++++++ apis/base/v1/types.go | 80 +++++++++++ apis/base/v1/zz_generated.deepcopy.go | 41 ++++++ apis/workloads/v1/instanceset_types.go | 72 +--------- apis/workloads/v1/zz_generated.deepcopy.go | 20 +-- ...ps.kubeblocks.io_componentdefinitions.yaml | 9 -- .../workloads.kubeblocks.io_instancesets.yaml | 18 --- controllers/apps/component_controller_test.go | 3 - .../apps/configuration/policy_util_test.go | 2 - controllers/k8score/event_controller_test.go | 2 - ...ps.kubeblocks.io_componentdefinitions.yaml | 9 -- .../workloads.kubeblocks.io_instancesets.yaml | 18 --- docs/developer_docs/api-reference/cluster.md | 124 +----------------- .../builder/builder_instance_set_test.go | 1 - pkg/controller/component/its_convertor.go | 9 +- .../component/lifecycle/lfa_member.go | 18 +-- .../pod_role_event_handler_test.go | 1 - pkg/controller/instanceset/suite_test.go | 4 - pkg/controller/instanceset/utils.go | 43 +----- pkg/controller/instanceset/utils_test.go | 65 --------- pkg/operations/switchover_util.go | 2 +- .../apps/cluster_instance_set_test_util.go | 25 ++-- pkg/testutil/apps/constant.go | 3 - 25 files changed, 202 insertions(+), 415 deletions(-) create mode 100644 apis/base/v1/groupversion_info.go create mode 100644 apis/base/v1/types.go create mode 100644 apis/base/v1/zz_generated.deepcopy.go diff --git a/apis/apps/v1/componentdefinition_types.go b/apis/apps/v1/componentdefinition_types.go index 55736fa4abd..0ef5d6b9f68 100644 --- a/apis/apps/v1/componentdefinition_types.go +++ b/apis/apps/v1/componentdefinition_types.go @@ -27,7 +27,7 @@ import ( rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - workloads "github.com/apecloud/kubeblocks/apis/workloads/v1" + basev1 "github.com/apecloud/kubeblocks/apis/base/v1" ) // +genclient @@ -1372,9 +1372,8 @@ type ComponentAvailableProbeAssertion struct { Strict *bool `json:"strict,omitempty"` } -// ReplicaRole represents a role that can be assumed by a component instance. // +kubebuilder:object:generate=false -type ReplicaRole = workloads.ReplicaRole +type ReplicaRole = basev1.ReplicaRole // UpdateStrategy defines the update strategy for cluster components. This strategy determines how updates are applied // across the cluster. diff --git a/apis/apps/v1/zz_generated.deepcopy.go b/apis/apps/v1/zz_generated.deepcopy.go index df777861d5c..c92ff6dca9b 100644 --- a/apis/apps/v1/zz_generated.deepcopy.go +++ b/apis/apps/v1/zz_generated.deepcopy.go @@ -24,7 +24,7 @@ along with this program. If not, see . package v1 import ( - workloadsv1 "github.com/apecloud/kubeblocks/apis/workloads/v1" + basev1 "github.com/apecloud/kubeblocks/apis/base/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -1189,7 +1189,7 @@ func (in *ComponentDefinitionSpec) DeepCopyInto(out *ComponentDefinitionSpec) { } if in.Roles != nil { in, out := &in.Roles, &out.Roles - *out = make([]workloadsv1.ReplicaRole, len(*in)) + *out = make([]basev1.ReplicaRole, len(*in)) copy(*out, *in) } if in.UpdateStrategy != nil { diff --git a/apis/base/v1/groupversion_info.go b/apis/base/v1/groupversion_info.go new file mode 100644 index 00000000000..2fd928f9bd9 --- /dev/null +++ b/apis/base/v1/groupversion_info.go @@ -0,0 +1,39 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +// Package v1 contains API Schema definitions for the base v1 API group +// +kubebuilder:object:generate=true +// +groupName=base.kubeblocks.io +package v1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "base.kubeblocks.io", Version: "v1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/apis/base/v1/types.go b/apis/base/v1/types.go new file mode 100644 index 00000000000..318f810f06e --- /dev/null +++ b/apis/base/v1/types.go @@ -0,0 +1,80 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package v1 + +// ReplicaRole represents a role that can be assigned to a component instance, defining its behavior and responsibilities. +type ReplicaRole struct { + // Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label + // on the corresponding object to identify its role. + // + // For example, common role names include: + // - "leader": The primary/master instance that handles write operations + // - "follower": Secondary/replica instances that replicate data from the leader + // - "learner": Read-only instances that don't participate in elections + // + // This field is immutable once set. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MaxLength=32 + // +kubebuilder:validation:Pattern=`^.*[^\s]+.*$` + Name string `json:"name"` + + // UpdatePriority determines the order in which pods with different roles are updated. + // Pods are sorted by this priority (higher numbers = higher priority) and updated accordingly. + // Roles with the highest priority will be updated last. + // The default priority is 0. + // + // For example: + // - Leader role may have priority 2 (updated last) + // - Follower role may have priority 1 (updated before leader) + // - Learner role may have priority 0 (updated first) + // + // This field is immutable once set. + // + // +kubebuilder:default=0 + // +optional + UpdatePriority int `json:"updatePriority"` + + // ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. + // This affects update strategies that need to maintain quorum for availability. + // + // For example, in a 5-pod component where: + // - 2 learner pods (participatesInQuorum=false) + // - 2 follower pods (participatesInQuorum=true) + // - 1 leader pod (participatesInQuorum=true) + // The quorum size would be 3 (based on the 3 participating pods), allowing parallel updates + // of 2 learners and 1 follower while maintaining quorum. + // + // This field is immutable once set. + // + // +kubebuilder:default=false + // +optional + ParticipatesInQuorum bool `json:"participatesInQuorum bool"` + + // SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before + // updating or scaling in pods with this role. This is typically used for leader roles to + // ensure minimal disruption during updates. + // + // This field is immutable once set. + // + // +kubebuilder:default=false + // +optional + SwitchoverBeforeUpdate bool `json:"switchoverBeforeUpdate"` +} diff --git a/apis/base/v1/zz_generated.deepcopy.go b/apis/base/v1/zz_generated.deepcopy.go new file mode 100644 index 00000000000..dfd135819e5 --- /dev/null +++ b/apis/base/v1/zz_generated.deepcopy.go @@ -0,0 +1,41 @@ +//go:build !ignore_autogenerated + +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1 + +import () + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReplicaRole) DeepCopyInto(out *ReplicaRole) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicaRole. +func (in *ReplicaRole) DeepCopy() *ReplicaRole { + if in == nil { + return nil + } + out := new(ReplicaRole) + in.DeepCopyInto(out) + return out +} diff --git a/apis/workloads/v1/instanceset_types.go b/apis/workloads/v1/instanceset_types.go index 8a797edb89f..c890d7f54cd 100644 --- a/apis/workloads/v1/instanceset_types.go +++ b/apis/workloads/v1/instanceset_types.go @@ -24,6 +24,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + + basev1 "github.com/apecloud/kubeblocks/apis/base/v1" ) // +genclient @@ -468,74 +470,8 @@ const ( PreferInPlacePodUpdatePolicyType PodUpdatePolicyType = "PreferInPlace" ) -// ReplicaRole represents a role that can be assigned to a component instance, defining its behavior and responsibilities. -type ReplicaRole struct { - // Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label - // on the corresponding object to identify its role. - // - // For example, common role names include: - // - "leader": The primary/master instance that handles write operations - // - "follower": Secondary/replica instances that replicate data from the leader - // - "learner": Read-only instances that don't participate in elections - // - // This field is immutable once set. - // - // +kubebuilder:validation:Required - // +kubebuilder:validation:MaxLength=32 - // +kubebuilder:validation:Pattern=`^.*[^\s]+.*$` - Name string `json:"name"` - - // Required indicates if at least one replica with this role must exist for the component to be considered - // operationally running. For example, a leader role may be required for the component to function. - // - // This field is immutable once set. - // - // +kubebuilder:default=false - // +optional - Required bool `json:"required"` - - // UpdatePriority determines the order in which pods with different roles are updated. - // Pods are sorted by this priority (higher numbers = higher priority) and updated accordingly. - // Roles with the highest priority will be updated last. - // The default priority is 0. - // - // For example: - // - Leader role may have priority 2 (updated last) - // - Follower role may have priority 1 (updated before leader) - // - Learner role may have priority 0 (updated first) - // - // This field is immutable once set. - // - // +kubebuilder:default=0 - // +optional - UpdatePriority int `json:"updatePriority"` - - // ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. - // This affects update strategies that need to maintain quorum for availability. - // - // For example, in a 5-pod component where: - // - 2 learner pods (participatesInQuorum=false) - // - 2 follower pods (participatesInQuorum=true) - // - 1 leader pod (participatesInQuorum=true) - // The quorum size would be 3 (based on the 3 participating pods), allowing parallel updates - // of 2 learners and 1 follower while maintaining quorum. - // - // This field is immutable once set. - // - // +kubebuilder:default=false - // +optional - ParticipatesInQuorum bool `json:"participatesInQuorum bool"` - - // SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before - // updating or scaling in pods with this role. This is typically used for leader roles to - // ensure minimal disruption during updates. - // - // This field is immutable once set. - // - // +kubebuilder:default=false - // +optional - SwitchoverBeforeUpdate bool `json:"switchoverBeforeUpdate"` -} +// +kubebuilder:object:generate=false +type ReplicaRole = basev1.ReplicaRole // AccessMode defines SVC access mode enums. // +enum diff --git a/apis/workloads/v1/zz_generated.deepcopy.go b/apis/workloads/v1/zz_generated.deepcopy.go index f36aeda179e..12c1e7d3f9d 100644 --- a/apis/workloads/v1/zz_generated.deepcopy.go +++ b/apis/workloads/v1/zz_generated.deepcopy.go @@ -24,6 +24,7 @@ along with this program. If not, see . package v1 import ( + basev1 "github.com/apecloud/kubeblocks/apis/base/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" @@ -193,7 +194,7 @@ func (in *InstanceSetSpec) DeepCopyInto(out *InstanceSetSpec) { in.UpdateStrategy.DeepCopyInto(&out.UpdateStrategy) if in.Roles != nil { in, out := &in.Roles, &out.Roles - *out = make([]ReplicaRole, len(*in)) + *out = make([]basev1.ReplicaRole, len(*in)) copy(*out, *in) } if in.RoleProbe != nil { @@ -374,7 +375,7 @@ func (in *MemberStatus) DeepCopyInto(out *MemberStatus) { *out = *in if in.ReplicaRole != nil { in, out := &in.ReplicaRole, &out.ReplicaRole - *out = new(ReplicaRole) + *out = new(basev1.ReplicaRole) **out = **in } } @@ -469,21 +470,6 @@ func (in *Range) DeepCopy() *Range { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ReplicaRole) DeepCopyInto(out *ReplicaRole) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicaRole. -func (in *ReplicaRole) DeepCopy() *ReplicaRole { - if in == nil { - return nil - } - out := new(ReplicaRole) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RoleProbe) DeepCopyInto(out *RoleProbe) { *out = *in diff --git a/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml b/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml index 02419b70c62..6461e8c1a3e 100644 --- a/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml +++ b/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml @@ -8506,15 +8506,6 @@ spec: of 2 learners and 1 follower while maintaining quorum. - This field is immutable once set. - type: boolean - required: - default: false - description: |- - Required indicates if at least one replica with this role must exist for the component to be considered - operationally running. For example, a leader role may be required for the component to function. - - This field is immutable once set. type: boolean switchoverBeforeUpdate: diff --git a/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml b/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml index a97b11d6267..1dee1084225 100644 --- a/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml +++ b/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml @@ -4290,15 +4290,6 @@ spec: of 2 learners and 1 follower while maintaining quorum. - This field is immutable once set. - type: boolean - required: - default: false - description: |- - Required indicates if at least one replica with this role must exist for the component to be considered - operationally running. For example, a leader role may be required for the component to function. - - This field is immutable once set. type: boolean switchoverBeforeUpdate: @@ -12511,15 +12502,6 @@ spec: of 2 learners and 1 follower while maintaining quorum. - This field is immutable once set. - type: boolean - required: - default: false - description: |- - Required indicates if at least one replica with this role must exist for the component to be considered - operationally running. For example, a leader role may be required for the component to function. - - This field is immutable once set. type: boolean switchoverBeforeUpdate: diff --git a/controllers/apps/component_controller_test.go b/controllers/apps/component_controller_test.go index ab7146ee206..6912601e6a2 100644 --- a/controllers/apps/component_controller_test.go +++ b/controllers/apps/component_controller_test.go @@ -1326,21 +1326,18 @@ var _ = Describe("Component Controller", func() { targetRoles := []workloads.ReplicaRole{ { Name: "leader", - Required: true, SwitchoverBeforeUpdate: true, ParticipatesInQuorum: true, UpdatePriority: 5, }, { Name: "follower", - Required: false, SwitchoverBeforeUpdate: false, ParticipatesInQuorum: true, UpdatePriority: 4, }, { Name: "learner", - Required: false, SwitchoverBeforeUpdate: false, ParticipatesInQuorum: false, UpdatePriority: 2, diff --git a/controllers/apps/configuration/policy_util_test.go b/controllers/apps/configuration/policy_util_test.go index 924009f2453..5c8536cd6bd 100644 --- a/controllers/apps/configuration/policy_util_test.go +++ b/controllers/apps/configuration/policy_util_test.go @@ -165,14 +165,12 @@ func newMockReconfigureParams(testName string, cli client.Client, paramOps ...Pa Roles: []appsv1.ReplicaRole{ { Name: "leader", - Required: true, SwitchoverBeforeUpdate: true, ParticipatesInQuorum: true, UpdatePriority: 5, }, { Name: "follower", - Required: false, SwitchoverBeforeUpdate: false, ParticipatesInQuorum: true, UpdatePriority: 4, diff --git a/controllers/k8score/event_controller_test.go b/controllers/k8score/event_controller_test.go index 0edcb25907a..e7269abfc00 100644 --- a/controllers/k8score/event_controller_test.go +++ b/controllers/k8score/event_controller_test.go @@ -144,14 +144,12 @@ var _ = Describe("Event Controller", func() { tmpITS.Spec.Roles = []workloads.ReplicaRole{ { Name: "leader", - Required: true, SwitchoverBeforeUpdate: true, ParticipatesInQuorum: true, UpdatePriority: 5, }, { Name: "follower", - Required: false, SwitchoverBeforeUpdate: false, ParticipatesInQuorum: true, UpdatePriority: 4, diff --git a/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml b/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml index 02419b70c62..6461e8c1a3e 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml @@ -8506,15 +8506,6 @@ spec: of 2 learners and 1 follower while maintaining quorum. - This field is immutable once set. - type: boolean - required: - default: false - description: |- - Required indicates if at least one replica with this role must exist for the component to be considered - operationally running. For example, a leader role may be required for the component to function. - - This field is immutable once set. type: boolean switchoverBeforeUpdate: diff --git a/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml b/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml index a97b11d6267..1dee1084225 100644 --- a/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml +++ b/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml @@ -4290,15 +4290,6 @@ spec: of 2 learners and 1 follower while maintaining quorum. - This field is immutable once set. - type: boolean - required: - default: false - description: |- - Required indicates if at least one replica with this role must exist for the component to be considered - operationally running. For example, a leader role may be required for the component to function. - - This field is immutable once set. type: boolean switchoverBeforeUpdate: @@ -12511,15 +12502,6 @@ spec: of 2 learners and 1 follower while maintaining quorum. - This field is immutable once set. - type: boolean - required: - default: false - description: |- - Required indicates if at least one replica with this role must exist for the component to be considered - operationally running. For example, a leader role may be required for the component to function. - - This field is immutable once set. type: boolean switchoverBeforeUpdate: diff --git a/docs/developer_docs/api-reference/cluster.md b/docs/developer_docs/api-reference/cluster.md index 5bcb31a6d9b..602b6d4b90a 100644 --- a/docs/developer_docs/api-reference/cluster.md +++ b/docs/developer_docs/api-reference/cluster.md @@ -1346,9 +1346,7 @@ ComponentAvailable
-

Defines the role name of the replica.

+

Name defines the role’s unique identifier. This value is used to set the “apps.kubeblocks.io/role” label +on the corresponding object to identify its role.

+

For example, common role names include: +- “leader”: The primary/master instance that handles write operations +- “follower”: Secondary/replica instances that replicate data from the leader +- “learner”: Read-only instances that don’t participate in elections

+

This field is immutable once set.
-accessMode
+required
- -AccessMode - +bool 		-

Specifies the service capabilities of this member.

+(Optional) +

Required indicates if at least one replica with this role must exist for the component to be considered +operationally running. For example, a leader role may be required for the component to function.

+

This field is immutable once set.
-canVote
+updatePriority
+ +int + +		 +(Optional) +

UpdatePriority determines the order in which pods with different roles are updated. +Pods are sorted by this priority (higher numbers = higher priority) and updated accordingly. +Roles with the highest priority will be updated last. +The default priority is 0.

+

For example: +- Leader role may have priority 2 (updated last) +- Follower role may have priority 1 (updated before leader) +- Learner role may have priority 0 (updated first)

+

This field is immutable once set.

+
+participatesInQuorum bool
bool 		(Optional) -

Indicates if this member has voting rights.

+

ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. +This affects update strategies that need to maintain quorum for availability.

+

For example, in a 5-pod component where: +- 2 learner pods (participatesInQuorum=false) +- 2 follower pods (participatesInQuorum=true) +- 1 leader pod (participatesInQuorum=true) +The quorum size would be 3 (based on the 3 participating pods), allowing parallel updates +of 2 learners and 1 follower while maintaining quorum.

+

This field is immutable once set.
-isLeader
+switchoverBeforeUpdate
bool 		(Optional) -

Determines if this member is the leader.

+

SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before +updating or scaling in pods with this role. This is typically used for leader roles to +ensure minimal disruption during updates.

+

This field is immutable once set.
roles
- -[]ReplicaRole - +[]github.com/apecloud/kubeblocks/apis/base/v1.ReplicaRole 		@@ -5004,9 +5002,7 @@ ComponentAvailable roles
- -[]ReplicaRole - +[]github.com/apecloud/kubeblocks/apis/base/v1.ReplicaRole 		@@ -28187,9 +28183,7 @@ UpdateStrategy.Type will be set to appsv1.OnDeleteStatefulSetStrategyType if Mem roles
- -[]ReplicaRole - +[]github.com/apecloud/kubeblocks/apis/base/v1.ReplicaRole 		@@ -28715,9 +28709,7 @@ UpdateStrategy.Type will be set to appsv1.OnDeleteStatefulSetStrategyType if Mem roles
- -[]ReplicaRole - +[]github.com/apecloud/kubeblocks/apis/base/v1.ReplicaRole 		@@ -29337,9 +29329,7 @@ string role
- -ReplicaRole - +github.com/apecloud/kubeblocks/apis/base/v1.ReplicaRole 		@@ -29573,110 +29563,6 @@ int32
-

ReplicaRole -

-

-(Appears on:ComponentDefinitionSpec, InstanceSetSpec, MemberStatus) -

-
-

ReplicaRole represents a role that can be assigned to a component instance, defining its behavior and responsibilities.

-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FieldDescription
-name
- -string - -		 -

Name defines the role’s unique identifier. This value is used to set the “apps.kubeblocks.io/role” label -on the corresponding object to identify its role.

-

For example, common role names include: -- “leader”: The primary/master instance that handles write operations -- “follower”: Secondary/replica instances that replicate data from the leader -- “learner”: Read-only instances that don’t participate in elections

-

This field is immutable once set.

-
-required
- -bool - -		 -(Optional) -

Required indicates if at least one replica with this role must exist for the component to be considered -operationally running. For example, a leader role may be required for the component to function.

-

This field is immutable once set.

-
-updatePriority
- -int - -		 -(Optional) -

UpdatePriority determines the order in which pods with different roles are updated. -Pods are sorted by this priority (higher numbers = higher priority) and updated accordingly. -Roles with the highest priority will be updated last. -The default priority is 0.

-

For example: -- Leader role may have priority 2 (updated last) -- Follower role may have priority 1 (updated before leader) -- Learner role may have priority 0 (updated first)

-

This field is immutable once set.

-
-participatesInQuorum bool
- -bool - -		 -(Optional) -

ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. -This affects update strategies that need to maintain quorum for availability.

-

For example, in a 5-pod component where: -- 2 learner pods (participatesInQuorum=false) -- 2 follower pods (participatesInQuorum=true) -- 1 leader pod (participatesInQuorum=true) -The quorum size would be 3 (based on the 3 participating pods), allowing parallel updates -of 2 learners and 1 follower while maintaining quorum.

-

This field is immutable once set.

-
-switchoverBeforeUpdate
- -bool - -		 -(Optional) -

SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before -updating or scaling in pods with this role. This is typically used for leader roles to -ensure minimal disruption during updates.

-

This field is immutable once set.

-

RoleProbe

diff --git a/pkg/controller/builder/builder_instance_set_test.go b/pkg/controller/builder/builder_instance_set_test.go index 0f56a72f622..29389e1f71e 100644 --- a/pkg/controller/builder/builder_instance_set_test.go +++ b/pkg/controller/builder/builder_instance_set_test.go @@ -51,7 +51,6 @@ var _ = Describe("instance_set builder", func() { selectors := map[string]string{selectorKey4: selectorValue4} role := workloads.ReplicaRole{ Name: "foo", - Required: true, SwitchoverBeforeUpdate: true, ParticipatesInQuorum: true, UpdatePriority: 1, diff --git a/pkg/controller/component/its_convertor.go b/pkg/controller/component/its_convertor.go index ee3e125e129..7857842f568 100644 --- a/pkg/controller/component/its_convertor.go +++ b/pkg/controller/component/its_convertor.go @@ -350,14 +350,7 @@ func ConvertSynthesizeCompRoleToInstanceSetRole(synthesizedComp *SynthesizedComp itsReplicaRoles := make([]workloads.ReplicaRole, 0) for _, role := range synthesizedComp.Roles { - itsReplicaRole := workloads.ReplicaRole{ - Name: role.Name, - Required: role.Required, - UpdatePriority: role.UpdatePriority, - ParticipatesInQuorum: role.ParticipatesInQuorum, - SwitchoverBeforeUpdate: role.SwitchoverBeforeUpdate, - } - itsReplicaRoles = append(itsReplicaRoles, itsReplicaRole) + itsReplicaRoles = append(itsReplicaRoles, *role.DeepCopy()) } return itsReplicaRoles } diff --git a/pkg/controller/component/lifecycle/lfa_member.go b/pkg/controller/component/lifecycle/lfa_member.go index dafdc9a15d8..2d282bafce2 100644 --- a/pkg/controller/component/lifecycle/lfa_member.go +++ b/pkg/controller/component/lifecycle/lfa_member.go @@ -172,17 +172,13 @@ func hackParameters4Switchover(ctx context.Context, cli client.Reader, namespace } func leaderRole(roles []appsv1.ReplicaRole) (string, error) { - targetRole := "" - for _, role := range roles { - if role.Required && role.ParticipatesInQuorum { - if targetRole != "" { - return "", fmt.Errorf("more than one role defined as leader: %s,%s", targetRole, role.Name) - } - targetRole = role.Name + highestPriority := 0 + var role *appsv1.ReplicaRole + for _, r := range roles { + if r.UpdatePriority > highestPriority { + highestPriority = r.UpdatePriority + role = &r } } - if targetRole == "" { - return "", fmt.Errorf("%s", "has no appropriate role defined as leader") - } - return targetRole, nil + return role.Name, nil } diff --git a/pkg/controller/instanceset/pod_role_event_handler_test.go b/pkg/controller/instanceset/pod_role_event_handler_test.go index ed53de3b30d..49776c61d91 100644 --- a/pkg/controller/instanceset/pod_role_event_handler_test.go +++ b/pkg/controller/instanceset/pod_role_event_handler_test.go @@ -56,7 +56,6 @@ var _ = Describe("pod role label event handler test", func() { } role := workloads.ReplicaRole{ Name: "leader", - Required: true, SwitchoverBeforeUpdate: true, ParticipatesInQuorum: true, UpdatePriority: 5, diff --git a/pkg/controller/instanceset/suite_test.go b/pkg/controller/instanceset/suite_test.go index 1e0b9dcf865..fd167879767 100644 --- a/pkg/controller/instanceset/suite_test.go +++ b/pkg/controller/instanceset/suite_test.go @@ -76,28 +76,24 @@ var ( roles = []workloads.ReplicaRole{ { Name: "leader", - Required: true, SwitchoverBeforeUpdate: true, ParticipatesInQuorum: true, UpdatePriority: 5, }, { Name: "follower", - Required: false, SwitchoverBeforeUpdate: false, ParticipatesInQuorum: true, UpdatePriority: 4, }, { Name: "logger", - Required: false, SwitchoverBeforeUpdate: false, ParticipatesInQuorum: false, UpdatePriority: 3, }, { Name: "learner", - Required: false, SwitchoverBeforeUpdate: false, ParticipatesInQuorum: false, UpdatePriority: 2, diff --git a/pkg/controller/instanceset/utils.go b/pkg/controller/instanceset/utils.go index 3b346b643b5..dc178141a23 100644 --- a/pkg/controller/instanceset/utils.go +++ b/pkg/controller/instanceset/utils.go @@ -99,48 +99,9 @@ func IsInstancesReady(its *workloads.InstanceSet) bool { return true } -// IsInstanceSetReady gives InstanceSet level 'ready' state: -// 1. all instances are available -// 2. and all members have role set (if they are role-ful) +// IsInstanceSetReady gives InstanceSet level 'ready' state when all instances are available func IsInstanceSetReady(its *workloads.InstanceSet) bool { - instancesReady := IsInstancesReady(its) - if !instancesReady { - return false - } - - // check whether role probe has done - if len(its.Spec.Roles) == 0 && its.Spec.RoleProbe == nil { - return true - } - membersStatus := its.Status.MembersStatus - if len(membersStatus) != int(*its.Spec.Replicas) { - return false - } - if its.Status.ReadyWithoutPrimary { - return true - } - - return IsAllRequiredRolesExist(its) -} - -func IsAllRequiredRolesExist(its *workloads.InstanceSet) bool { - requiredRoleExist := make(map[string]bool) - for _, role := range its.Spec.Roles { - if role.Required { - requiredRoleExist[role.Name] = false - } - } - for _, status := range its.Status.MembersStatus { - if status.ReplicaRole != nil && status.ReplicaRole.Required { - requiredRoleExist[status.ReplicaRole.Name] = true - } - } - for _, exist := range requiredRoleExist { - if !exist { - return false - } - } - return true + return IsInstancesReady(its) } // AddAnnotationScope will add AnnotationScope defined by 'scope' to all keys in map 'annotations'. diff --git a/pkg/controller/instanceset/utils_test.go b/pkg/controller/instanceset/utils_test.go index bc8ac69f8ea..b01800330aa 100644 --- a/pkg/controller/instanceset/utils_test.go +++ b/pkg/controller/instanceset/utils_test.go @@ -226,71 +226,6 @@ var _ = Describe("utils test", func() { }) }) - Context("IsAllRequiredRolesExist", func() { - It("works well", func() { - its = builder.NewInstanceSetBuilder(namespace, name). - SetRoles(roles). - GetObject() - By("no leader") - its.Status.MembersStatus = []workloads.MemberStatus{ - { - PodName: name + "-0", - ReplicaRole: &roles[1], - }, - { - PodName: name + "-1", - ReplicaRole: &roles[1], - }, - { - PodName: name + "-2", - ReplicaRole: &roles[2], - }, - } - Expect(IsAllRequiredRolesExist(its)).Should(BeFalse()) - - By("has leader") - its.Status.MembersStatus[0].ReplicaRole = &roles[0] - Expect(IsAllRequiredRolesExist(its)).Should(BeTrue()) - - By("set two required roles") - roles = []workloads.ReplicaRole{ - { - Name: "r1", - Required: true, - }, - { - Name: "r2", - Required: true, - }, - { - Name: "r3", - Required: false, - }, - } - its.Spec.Roles = roles - By("only one required role presents") - its.Status.MembersStatus = []workloads.MemberStatus{ - { - PodName: name + "-0", - ReplicaRole: &roles[0], - }, - { - PodName: name + "-1", - ReplicaRole: &roles[0], - }, - { - PodName: name + "-2", - ReplicaRole: &roles[2], - }, - } - Expect(IsAllRequiredRolesExist(its)).Should(BeFalse()) - - By("all required roles present") - its.Status.MembersStatus[1].ReplicaRole = &roles[1] - Expect(IsAllRequiredRolesExist(its)).Should(BeTrue()) - }) - }) - Context("CalculateConcurrencyReplicas function", func() { It("should work well", func() { By("concurrency = 50%, replicas = 10") diff --git a/pkg/operations/switchover_util.go b/pkg/operations/switchover_util.go index 4306b08b0c5..fe7b27dff60 100644 --- a/pkg/operations/switchover_util.go +++ b/pkg/operations/switchover_util.go @@ -49,7 +49,6 @@ func needDoSwitchover(ctx context.Context, cli client.Client, synthesizedComp *component.SynthesizedComponent, switchover *opsv1alpha1.Switchover) (bool, error) { - // get the Pod object whose current role label is already serviceable and writable pod, err := getPodToPerformSwitchover(ctx, cli, synthesizedComp) if err != nil { return false, err @@ -118,6 +117,7 @@ func checkPodRoleLabelConsistency(ctx context.Context, return false, nil } +// get the Pod object whose current role needs to be transferred to another pod func getPodToPerformSwitchover(ctx context.Context, cli client.Reader, synthesizedComp *component.SynthesizedComponent) (*corev1.Pod, error) { role, err := getTargetRoleName(synthesizedComp.Roles) if err != nil { diff --git a/pkg/testutil/apps/cluster_instance_set_test_util.go b/pkg/testutil/apps/cluster_instance_set_test_util.go index 61924d5ec47..93153ca47e6 100644 --- a/pkg/testutil/apps/cluster_instance_set_test_util.go +++ b/pkg/testutil/apps/cluster_instance_set_test_util.go @@ -23,6 +23,7 @@ import ( "context" "encoding/json" "fmt" + "math" "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -77,14 +78,12 @@ func MockInstanceSetComponent( SetRoles([]workloads.ReplicaRole{ { Name: "leader", - Required: true, SwitchoverBeforeUpdate: true, ParticipatesInQuorum: true, UpdatePriority: 5, }, { Name: "follower", - Required: false, SwitchoverBeforeUpdate: false, ParticipatesInQuorum: true, UpdatePriority: 4, @@ -108,23 +107,29 @@ func MockInstanceSetPods( if its == nil { return nil } - for i := range its.Spec.Roles { - if its.Spec.Roles[i].Required { - return &its.Spec.Roles[i] + highestPriority := 0 + var role *workloads.ReplicaRole + for i, r := range its.Spec.Roles { + if its.Spec.Roles[i].UpdatePriority > highestPriority { + highestPriority = r.UpdatePriority + role = &r } } - return nil + return role }() noneLeaderRole := func() *workloads.ReplicaRole { if its == nil { return nil } - for i := range its.Spec.Roles { - if !its.Spec.Roles[i].Required { - return &its.Spec.Roles[i] + lowestPriority := math.MaxInt + var role *workloads.ReplicaRole + for i, r := range its.Spec.Roles { + if its.Spec.Roles[i].UpdatePriority < lowestPriority { + lowestPriority = r.UpdatePriority + role = &r } } - return nil + return role }() podList := make([]*corev1.Pod, getReplicas()) podNames := generatePodNames(cluster, compName) diff --git a/pkg/testutil/apps/constant.go b/pkg/testutil/apps/constant.go index 99efa7dc1f9..86816fac3ea 100644 --- a/pkg/testutil/apps/constant.go +++ b/pkg/testutil/apps/constant.go @@ -210,21 +210,18 @@ var ( Roles: []appsv1.ReplicaRole{ { Name: "leader", - Required: true, SwitchoverBeforeUpdate: true, ParticipatesInQuorum: true, UpdatePriority: 5, }, { Name: "follower", - Required: false, SwitchoverBeforeUpdate: false, ParticipatesInQuorum: true, UpdatePriority: 4, }, { Name: "learner", - Required: false, SwitchoverBeforeUpdate: false, ParticipatesInQuorum: false, UpdatePriority: 2, From bca059b5dbde29f10e7c3e03d09450abbdd23534 Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Fri, 8 Nov 2024 16:08:43 +0800 Subject: [PATCH 13/34] delete AccessModeLabelKey --- apis/apps/v1/componentdefinition_types.go | 1 + apis/workloads/v1/instanceset_types.go | 1 + controllers/apps/component_utils_test.go | 1 - pkg/constant/labels.go | 1 - pkg/controller/component/lifecycle/lfa_member.go | 1 + pkg/controller/instanceset/pod_role_event_handler.go | 3 --- .../instanceset/pod_role_event_handler_test.go | 2 -- pkg/controller/instanceset/types.go | 3 +-- pkg/controller/plan/restore.go | 12 ++++++++++-- pkg/testutil/apps/base_factory.go | 4 ---- pkg/testutil/apps/cluster_instance_set_test_util.go | 5 +++-- 11 files changed, 17 insertions(+), 17 deletions(-) diff --git a/apis/apps/v1/componentdefinition_types.go b/apis/apps/v1/componentdefinition_types.go index 0ef5d6b9f68..9c185b71057 100644 --- a/apis/apps/v1/componentdefinition_types.go +++ b/apis/apps/v1/componentdefinition_types.go @@ -1372,6 +1372,7 @@ type ComponentAvailableProbeAssertion struct { Strict *bool `json:"strict,omitempty"` } +// ReplicaRole represents a role that can be assigned to a component instance, defining its behavior and responsibilities. // +kubebuilder:object:generate=false type ReplicaRole = basev1.ReplicaRole diff --git a/apis/workloads/v1/instanceset_types.go b/apis/workloads/v1/instanceset_types.go index c890d7f54cd..320ef9ee044 100644 --- a/apis/workloads/v1/instanceset_types.go +++ b/apis/workloads/v1/instanceset_types.go @@ -470,6 +470,7 @@ const ( PreferInPlacePodUpdatePolicyType PodUpdatePolicyType = "PreferInPlace" ) +// ReplicaRole represents a role that can be assigned to a component instance, defining its behavior and responsibilities. // +kubebuilder:object:generate=false type ReplicaRole = basev1.ReplicaRole diff --git a/controllers/apps/component_utils_test.go b/controllers/apps/component_utils_test.go index cc90b46ad54..fc44711c7bd 100644 --- a/controllers/apps/component_utils_test.go +++ b/controllers/apps/component_utils_test.go @@ -74,7 +74,6 @@ var _ = Describe("Component Utils", func() { AddAppComponentLabel(compName). AddAppManagedByLabel(). AddRoleLabel(role). - AddAccessModeLabel(mode). AddControllerRevisionHashLabel(""). AddVolume(corev1.Volume{ Name: testapps.DataVolumeName, diff --git a/pkg/constant/labels.go b/pkg/constant/labels.go index fc568c9c186..4b2f7e40932 100644 --- a/pkg/constant/labels.go +++ b/pkg/constant/labels.go @@ -50,7 +50,6 @@ const ( RoleLabelKey = "kubeblocks.io/role" // RoleLabelKey consensusSet and replicationSet role label key KBAppServiceVersionKey = "apps.kubeblocks.io/service-version" - AccessModeLabelKey = "workloads.kubeblocks.io/access-mode" ReadyWithoutPrimaryKey = "kubeblocks.io/ready-without-primary" ) diff --git a/pkg/controller/component/lifecycle/lfa_member.go b/pkg/controller/component/lifecycle/lfa_member.go index 2d282bafce2..d39a833a417 100644 --- a/pkg/controller/component/lifecycle/lfa_member.go +++ b/pkg/controller/component/lifecycle/lfa_member.go @@ -172,6 +172,7 @@ func hackParameters4Switchover(ctx context.Context, cli client.Reader, namespace } func leaderRole(roles []appsv1.ReplicaRole) (string, error) { + // HACK: assume the role with highest priority to be leader highestPriority := 0 var role *appsv1.ReplicaRole for _, r := range roles { diff --git a/pkg/controller/instanceset/pod_role_event_handler.go b/pkg/controller/instanceset/pod_role_event_handler.go index e53142af9f5..c2157c945d6 100644 --- a/pkg/controller/instanceset/pod_role_event_handler.go +++ b/pkg/controller/instanceset/pod_role_event_handler.go @@ -250,11 +250,8 @@ func updatePodRoleLabel(cli client.Client, reqCtx intctrlutil.RequestCtx, switch ok { case true: pod.Labels[RoleLabelKey] = role.Name - // FIXME: find out why this label is needed - // pod.Labels[AccessModeLabelKey] = string(role.AccessMode) case false: delete(pod.Labels, RoleLabelKey) - delete(pod.Labels, AccessModeLabelKey) } if pod.Annotations == nil { diff --git a/pkg/controller/instanceset/pod_role_event_handler_test.go b/pkg/controller/instanceset/pod_role_event_handler_test.go index 49776c61d91..addb6f95c23 100644 --- a/pkg/controller/instanceset/pod_role_event_handler_test.go +++ b/pkg/controller/instanceset/pod_role_event_handler_test.go @@ -96,8 +96,6 @@ var _ = Describe("pod role label event handler test", func() { Expect(pd).ShouldNot(BeNil()) Expect(pd.Labels).ShouldNot(BeNil()) Expect(pd.Labels[RoleLabelKey]).Should(Equal(role.Name)) - // FIXME: find out why this label is needed - // Expect(pd.Labels[AccessModeLabelKey]).Should(BeEquivalentTo(role.AccessMode)) return nil }).Times(1) k8sMock.EXPECT(). diff --git a/pkg/controller/instanceset/types.go b/pkg/controller/instanceset/types.go index 4af6d8ab75b..7b179719494 100644 --- a/pkg/controller/instanceset/types.go +++ b/pkg/controller/instanceset/types.go @@ -23,8 +23,7 @@ const ( WorkloadsManagedByLabelKey = "workloads.kubeblocks.io/managed-by" WorkloadsInstanceLabelKey = "workloads.kubeblocks.io/instance" - RoleLabelKey = "kubeblocks.io/role" - AccessModeLabelKey = "workloads.kubeblocks.io/access-mode" + RoleLabelKey = "kubeblocks.io/role" LegacyRSMFinalizerName = "rsm.workloads.kubeblocks.io/finalizer" diff --git a/pkg/controller/plan/restore.go b/pkg/controller/plan/restore.go index 813ecab3db4..d3588c71310 100644 --- a/pkg/controller/plan/restore.go +++ b/pkg/controller/plan/restore.go @@ -33,7 +33,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" appsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" - appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" "github.com/apecloud/kubeblocks/pkg/constant" "github.com/apecloud/kubeblocks/pkg/controller/component" @@ -232,7 +231,16 @@ func (r *RestoreManager) DoPostReady(comp *component.SynthesizedComponent, backupObj *dpv1alpha1.Backup) error { jobActionLabels := constant.GetCompLabels(r.Cluster.Name, comp.Name) if len(comp.Roles) > 0 { - jobActionLabels[instanceset.AccessModeLabelKey] = string(appsv1alpha1.ReadWrite) + // HACK: assume the role with highest priority to be writable + highestPriority := 0 + var role *appsv1.ReplicaRole + for _, r := range comp.Roles { + if r.UpdatePriority > highestPriority { + highestPriority = r.UpdatePriority + role = &r + } + } + jobActionLabels[instanceset.RoleLabelKey] = role.Name } sourceTargetName := compObj.Annotations[constant.BackupSourceTargetAnnotationKey] restore := &dpv1alpha1.Restore{ diff --git a/pkg/testutil/apps/base_factory.go b/pkg/testutil/apps/base_factory.go index bca40a586e2..3df01b19070 100644 --- a/pkg/testutil/apps/base_factory.go +++ b/pkg/testutil/apps/base_factory.go @@ -104,10 +104,6 @@ func (factory *BaseFactory[T, PT, F]) AddAppManagedByLabel() *F { return factory.AddLabels(constant.AppManagedByLabelKey, constant.AppName) } -func (factory *BaseFactory[T, PT, F]) AddAccessModeLabel(value string) *F { - return factory.AddLabels(constant.AccessModeLabelKey, value) -} - func (factory *BaseFactory[T, PT, F]) AddRoleLabel(value string) *F { return factory.AddLabels(constant.RoleLabelKey, value) } diff --git a/pkg/testutil/apps/cluster_instance_set_test_util.go b/pkg/testutil/apps/cluster_instance_set_test_util.go index 93153ca47e6..4223f79dcd0 100644 --- a/pkg/testutil/apps/cluster_instance_set_test_util.go +++ b/pkg/testutil/apps/cluster_instance_set_test_util.go @@ -157,6 +157,7 @@ func MockInstanceSetPods( } // MockInstanceSetPod mocks to create the pod of the InstanceSet, just using in envTest +// TODO: remove accessMode func MockInstanceSetPod( testCtx *testutil.TestContext, its *workloads.InstanceSet, @@ -164,7 +165,8 @@ func MockInstanceSetPod( consensusCompName, podName, podRole, accessMode string, - resources ...corev1.ResourceRequirements) *corev1.Pod { + resources ...corev1.ResourceRequirements, +) *corev1.Pod { var stsUpdateRevision string if its != nil { stsUpdateRevision = its.Status.UpdateRevision @@ -183,7 +185,6 @@ func MockInstanceSetPod( AddAppComponentLabel(consensusCompName). AddAppManagedByLabel(). AddRoleLabel(podRole). - // AddAccessModeLabel(accessMode). AddControllerRevisionHashLabel(stsUpdateRevision). AddLabelsInMap(ml). AddVolume(corev1.Volume{ From 7ad342b87ade788dbdda57266e759556da91e52c Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Fri, 8 Nov 2024 16:38:01 +0800 Subject: [PATCH 14/34] fix unit test --- apis/workloads/v1/instanceset_types.go | 3 ++ controllers/apps/configuration/policy_util.go | 7 +++-- .../builder/builder_component_definition.go | 28 ++++++++++--------- pkg/controller/component/its_convertor.go | 15 +--------- pkg/controller/instanceset/utils.go | 16 +++++++++-- pkg/controller/plan/restore.go | 3 +- pkg/controller/plan/restore_test.go | 13 +++------ .../apps/cluster_instance_set_test_util.go | 5 +--- 8 files changed, 45 insertions(+), 45 deletions(-) diff --git a/apis/workloads/v1/instanceset_types.go b/apis/workloads/v1/instanceset_types.go index 320ef9ee044..d23cad5a84c 100644 --- a/apis/workloads/v1/instanceset_types.go +++ b/apis/workloads/v1/instanceset_types.go @@ -293,6 +293,9 @@ type InstanceSetStatus struct { // Indicates whether it is required for the InstanceSet to have at least one primary instance ready. // + // Deprecated: since instanceset no longer checks a "primary" role when doing ready check, this + // field is no longer needed and will be removed in the future. + // // +optional ReadyWithoutPrimary bool `json:"readyWithoutPrimary,omitempty"` diff --git a/controllers/apps/configuration/policy_util.go b/controllers/apps/configuration/policy_util.go index 3f92f3ff0df..f142f4d74f7 100644 --- a/controllers/apps/configuration/policy_util.go +++ b/controllers/apps/configuration/policy_util.go @@ -33,7 +33,6 @@ import ( "github.com/apecloud/kubeblocks/pkg/configuration/core" cfgproto "github.com/apecloud/kubeblocks/pkg/configuration/proto" "github.com/apecloud/kubeblocks/pkg/constant" - "github.com/apecloud/kubeblocks/pkg/controller/component" "github.com/apecloud/kubeblocks/pkg/controller/configuration" "github.com/apecloud/kubeblocks/pkg/controller/instanceset" intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil" @@ -85,7 +84,11 @@ func getPodsForOnlineUpdate(params reconfigureParams) ([]corev1.Pod, error) { } if params.SynthesizedComponent != nil { - instanceset.SortPods(pods, instanceset.ComposeRolePriorityMap(component.ConvertSynthesizeCompRoleToInstanceSetRole(params.SynthesizedComponent)), true) + instanceset.SortPods( + pods, + instanceset.ComposeRolePriorityMap(params.SynthesizedComponent.Roles), + true, + ) } return pods, nil } diff --git a/pkg/controller/builder/builder_component_definition.go b/pkg/controller/builder/builder_component_definition.go index b49b050ff67..e97c0e81cf1 100644 --- a/pkg/controller/builder/builder_component_definition.go +++ b/pkg/controller/builder/builder_component_definition.go @@ -210,19 +210,21 @@ func (builder *ComponentDefinitionBuilder) SetUpdateStrategy(strategy *appsv1.Up return builder } -// FIXME: currently not used -// func (builder *ComponentDefinitionBuilder) AddRole(name string, serviceable, writable bool) *ComponentDefinitionBuilder { -// role := appsv1.ReplicaRole{ -// Name: name, -// Serviceable: serviceable, -// Writable: writable, -// } -// if builder.get().Spec.Roles == nil { -// builder.get().Spec.Roles = make([]appsv1.ReplicaRole, 0) -// } -// builder.get().Spec.Roles = append(builder.get().Spec.Roles, role) -// return builder -// } +func (builder *ComponentDefinitionBuilder) AddRole( + name string, updatePriority int, participatesInQuorum bool, switchoverBeforeUpdate bool, +) *ComponentDefinitionBuilder { + role := appsv1.ReplicaRole{ + Name: name, + UpdatePriority: updatePriority, + ParticipatesInQuorum: participatesInQuorum, + SwitchoverBeforeUpdate: switchoverBeforeUpdate, + } + if builder.get().Spec.Roles == nil { + builder.get().Spec.Roles = make([]appsv1.ReplicaRole, 0) + } + builder.get().Spec.Roles = append(builder.get().Spec.Roles, role) + return builder +} func (builder *ComponentDefinitionBuilder) SetLifecycleAction(name string, val interface{}) *ComponentDefinitionBuilder { obj := &builder.get().Spec.LifecycleActions diff --git a/pkg/controller/component/its_convertor.go b/pkg/controller/component/its_convertor.go index 7857842f568..cca9eb40088 100644 --- a/pkg/controller/component/its_convertor.go +++ b/pkg/controller/component/its_convertor.go @@ -288,7 +288,7 @@ func (c *itsRolesConvertor) convert(args ...any) (any, error) { if err != nil { return nil, err } - return ConvertSynthesizeCompRoleToInstanceSetRole(synthesizeComp), nil + return synthesizeComp.Roles, nil } // itsRoleProbeConvertor converts the ComponentDefinition.Spec.LifecycleActions.RoleProbe into InstanceSet.Spec.RoleProbe. @@ -341,16 +341,3 @@ func (c *itsMembershipReconfigurationConvertor) convert(args ...any) (any, error // synthesizeComp, err := parseITSConvertorArgs(args...) return "", nil // TODO } - -// ConvertSynthesizeCompRoleToInstanceSetRole converts the component.SynthesizedComponent.Roles to workloads.ReplicaRole. -func ConvertSynthesizeCompRoleToInstanceSetRole(synthesizedComp *SynthesizedComponent) []workloads.ReplicaRole { - if synthesizedComp.Roles == nil { - return nil - } - - itsReplicaRoles := make([]workloads.ReplicaRole, 0) - for _, role := range synthesizedComp.Roles { - itsReplicaRoles = append(itsReplicaRoles, *role.DeepCopy()) - } - return itsReplicaRoles -} diff --git a/pkg/controller/instanceset/utils.go b/pkg/controller/instanceset/utils.go index dc178141a23..03f2d51f07d 100644 --- a/pkg/controller/instanceset/utils.go +++ b/pkg/controller/instanceset/utils.go @@ -99,9 +99,21 @@ func IsInstancesReady(its *workloads.InstanceSet) bool { return true } -// IsInstanceSetReady gives InstanceSet level 'ready' state when all instances are available +// IsInstanceSetReady gives InstanceSet level 'ready' state: +// 1. all instances are available +// 2. and all members have role set (if they are role-ful) func IsInstanceSetReady(its *workloads.InstanceSet) bool { - return IsInstancesReady(its) + instancesReady := IsInstancesReady(its) + if !instancesReady { + return false + } + + // check whether role probe has done + if len(its.Spec.Roles) == 0 && its.Spec.RoleProbe == nil { + return true + } + membersStatus := its.Status.MembersStatus + return len(membersStatus) == int(*its.Spec.Replicas) } // AddAnnotationScope will add AnnotationScope defined by 'scope' to all keys in map 'annotations'. diff --git a/pkg/controller/plan/restore.go b/pkg/controller/plan/restore.go index d3588c71310..15b584ecadf 100644 --- a/pkg/controller/plan/restore.go +++ b/pkg/controller/plan/restore.go @@ -22,6 +22,7 @@ package plan import ( "context" "fmt" + "math" "strings" corev1 "k8s.io/api/core/v1" @@ -232,7 +233,7 @@ func (r *RestoreManager) DoPostReady(comp *component.SynthesizedComponent, jobActionLabels := constant.GetCompLabels(r.Cluster.Name, comp.Name) if len(comp.Roles) > 0 { // HACK: assume the role with highest priority to be writable - highestPriority := 0 + highestPriority := math.MinInt var role *appsv1.ReplicaRole for _, r := range comp.Roles { if r.UpdatePriority > highestPriority { diff --git a/pkg/controller/plan/restore_test.go b/pkg/controller/plan/restore_test.go index f620ee5dec6..3581c4a1d02 100644 --- a/pkg/controller/plan/restore_test.go +++ b/pkg/controller/plan/restore_test.go @@ -157,17 +157,12 @@ var _ = Describe("Restore", func() { Replicas: 1, Roles: []appsv1.ReplicaRole{ { - Name: "leader", - // FIXME - // Serviceable: true, - // Writable: true, - // Votable: true, + Name: "leader", + UpdatePriority: 2, }, { - Name: "follower", - // Serviceable: true, - // Writable: false, - // Votable: true, + Name: "follower", + UpdatePriority: 1, }, }, } diff --git a/pkg/testutil/apps/cluster_instance_set_test_util.go b/pkg/testutil/apps/cluster_instance_set_test_util.go index 4223f79dcd0..832a6d5374c 100644 --- a/pkg/testutil/apps/cluster_instance_set_test_util.go +++ b/pkg/testutil/apps/cluster_instance_set_test_util.go @@ -138,14 +138,11 @@ func MockInstanceSetPods( if its != nil && len(its.Spec.Roles) > 0 { if i == 0 { podRole = leaderRole.Name - // accessMode = string(leaderRole.AccessMode) } else { podRole = noneLeaderRole.Name - // accessMode = string(noneLeaderRole.AccessMode) } } - // FIXME: is access mode label needed? - pod := MockInstanceSetPod(testCtx, its, cluster.Name, compName, pName, podRole, "foo") + pod := MockInstanceSetPod(testCtx, its, cluster.Name, compName, pName, podRole, "") annotations := pod.Annotations if annotations == nil { annotations = make(map[string]string) From 90b8f224ffd35d596d42c8bc289b39ec6e72e25a Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Fri, 8 Nov 2024 16:47:46 +0800 Subject: [PATCH 15/34] make --- .../crd/bases/workloads.kubeblocks.io_instancesets.yaml | 8 ++++++-- .../helm/crds/workloads.kubeblocks.io_instancesets.yaml | 8 ++++++-- docs/developer_docs/api-reference/cluster.md | 2 ++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml b/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml index 1dee1084225..5bba9ff8555 100644 --- a/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml +++ b/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml @@ -12557,8 +12557,12 @@ spec: format: int32 type: integer readyWithoutPrimary: - description: Indicates whether it is required for the InstanceSet - to have at least one primary instance ready. + description: |- + Indicates whether it is required for the InstanceSet to have at least one primary instance ready. + + + Deprecated: since instanceset no longer checks a "primary" role when doing ready check, this + field is no longer needed and will be removed in the future. type: boolean replicas: description: replicas is the number of instances created by the InstanceSet diff --git a/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml b/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml index 1dee1084225..5bba9ff8555 100644 --- a/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml +++ b/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml @@ -12557,8 +12557,12 @@ spec: format: int32 type: integer readyWithoutPrimary: - description: Indicates whether it is required for the InstanceSet - to have at least one primary instance ready. + description: |- + Indicates whether it is required for the InstanceSet to have at least one primary instance ready. + + + Deprecated: since instanceset no longer checks a "primary" role when doing ready check, this + field is no longer needed and will be removed in the future. type: boolean replicas: description: replicas is the number of instances created by the InstanceSet diff --git a/docs/developer_docs/api-reference/cluster.md b/docs/developer_docs/api-reference/cluster.md index 602b6d4b90a..6fdadfd8cc2 100644 --- a/docs/developer_docs/api-reference/cluster.md +++ b/docs/developer_docs/api-reference/cluster.md @@ -28970,6 +28970,8 @@ bool (Optional)

Indicates whether it is required for the InstanceSet to have at least one primary instance ready.

+

Deprecated: since instanceset no longer checks a “primary” role when doing ready check, this +field is no longer needed and will be removed in the future.

From 6fc8514cee47f93d600344daf3f719ce224b7866 Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Tue, 12 Nov 2024 11:00:23 +0800 Subject: [PATCH 16/34] comments --- pkg/operations/switchover_util.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/operations/switchover_util.go b/pkg/operations/switchover_util.go index fe7b27dff60..fae47a00607 100644 --- a/pkg/operations/switchover_util.go +++ b/pkg/operations/switchover_util.go @@ -133,7 +133,7 @@ func getPodByRole(ctx context.Context, cli client.Reader, synthesizeComp *compon return nil, err } if len(pods) != 1 { - return nil, errors.New("component pod list is empty or has more than one serviceable and writable pod") + return nil, errors.New("target pod list is empty or has more than one pod") } return pods[0], nil } @@ -146,7 +146,6 @@ func getTargetRoleName(roles []appsv1.ReplicaRole) (string, error) { return targetRole, errors.New("component has no roles definition, does not support switchover") } for _, role := range roles { - // FIXME: the assumption that only one role supports switchover may change in the future if role.SwitchoverBeforeUpdate { if targetRole != "" { return targetRole, errors.New("componentDefinition has more than role that needs switchover before, does not support switchover") From 6439d9d9af2967e914182df30970417543b13ae8 Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Tue, 12 Nov 2024 13:11:31 +0800 Subject: [PATCH 17/34] add a validation rule --- apis/base/v1/types.go | 7 ++++++- ...ps.kubeblocks.io_componentdefinitions.yaml | 10 +++++++++- .../workloads.kubeblocks.io_instancesets.yaml | 20 +++++++++++++++++-- ...ps.kubeblocks.io_componentdefinitions.yaml | 10 +++++++++- .../workloads.kubeblocks.io_instancesets.yaml | 20 +++++++++++++++++-- 5 files changed, 60 insertions(+), 7 deletions(-) diff --git a/apis/base/v1/types.go b/apis/base/v1/types.go index 318f810f06e..3c242e7a2be 100644 --- a/apis/base/v1/types.go +++ b/apis/base/v1/types.go @@ -20,6 +20,8 @@ along with this program. If not, see . package v1 // ReplicaRole represents a role that can be assigned to a component instance, defining its behavior and responsibilities. +// +// +kubebuilder:validation:XValidation:rule="self.filter(x, x.participatesInQuorum == true).map(x, x.updatePriority).min() > self.filter(x, x.participatesInQuorum == false).map(x, x.updatePriority).max()",message="Roles participate in quorum should have higher update priority than roles do not participate in quorum." type ReplicaRole struct { // Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label // on the corresponding object to identify its role. @@ -53,7 +55,9 @@ type ReplicaRole struct { UpdatePriority int `json:"updatePriority"` // ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. - // This affects update strategies that need to maintain quorum for availability. + // This affects update strategies that need to maintain quorum for availability. Roles participate + // in quorum should have higher update priority than roles do not participate in quorum. + // The default value is false. // // For example, in a 5-pod component where: // - 2 learner pods (participatesInQuorum=false) @@ -71,6 +75,7 @@ type ReplicaRole struct { // SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before // updating or scaling in pods with this role. This is typically used for leader roles to // ensure minimal disruption during updates. + // The default value is false. // // This field is immutable once set. // diff --git a/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml b/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml index 6461e8c1a3e..391296025f4 100644 --- a/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml +++ b/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml @@ -8495,7 +8495,9 @@ spec: default: false description: |- ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. - This affects update strategies that need to maintain quorum for availability. + This affects update strategies that need to maintain quorum for availability. Roles participate + in quorum should have higher update priority than roles do not participate in quorum. + The default value is false. For example, in a 5-pod component where: @@ -8514,6 +8516,7 @@ spec: SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before updating or scaling in pods with this role. This is typically used for leader roles to ensure minimal disruption during updates. + The default value is false. This field is immutable once set. @@ -8538,6 +8541,11 @@ spec: required: - name type: object + x-kubernetes-validations: + - message: Roles participate in quorum should have higher update + priority than roles do not participate in quorum. + rule: self.filter(c, c.participatesInQuorum == true).min() > self.filter(c, + c.participatesInQuorum == true).max() type: array runtime: description: |- diff --git a/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml b/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml index 5bba9ff8555..eb35e06d173 100644 --- a/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml +++ b/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml @@ -4279,7 +4279,9 @@ spec: default: false description: |- ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. - This affects update strategies that need to maintain quorum for availability. + This affects update strategies that need to maintain quorum for availability. Roles participate + in quorum should have higher update priority than roles do not participate in quorum. + The default value is false. For example, in a 5-pod component where: @@ -4298,6 +4300,7 @@ spec: SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before updating or scaling in pods with this role. This is typically used for leader roles to ensure minimal disruption during updates. + The default value is false. This field is immutable once set. @@ -4322,6 +4325,11 @@ spec: required: - name type: object + x-kubernetes-validations: + - message: Roles participate in quorum should have higher update + priority than roles do not participate in quorum. + rule: self.filter(c, c.participatesInQuorum == true).min() > self.filter(c, + c.participatesInQuorum == true).max() type: array selector: description: |- @@ -12491,7 +12499,9 @@ spec: default: false description: |- ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. - This affects update strategies that need to maintain quorum for availability. + This affects update strategies that need to maintain quorum for availability. Roles participate + in quorum should have higher update priority than roles do not participate in quorum. + The default value is false. For example, in a 5-pod component where: @@ -12510,6 +12520,7 @@ spec: SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before updating or scaling in pods with this role. This is typically used for leader roles to ensure minimal disruption during updates. + The default value is false. This field is immutable once set. @@ -12534,6 +12545,11 @@ spec: required: - name type: object + x-kubernetes-validations: + - message: Roles participate in quorum should have higher update + priority than roles do not participate in quorum. + rule: self.filter(c, c.participatesInQuorum == true).min() + > self.filter(c, c.participatesInQuorum == true).max() required: - podName type: object diff --git a/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml b/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml index 6461e8c1a3e..391296025f4 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml @@ -8495,7 +8495,9 @@ spec: default: false description: |- ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. - This affects update strategies that need to maintain quorum for availability. + This affects update strategies that need to maintain quorum for availability. Roles participate + in quorum should have higher update priority than roles do not participate in quorum. + The default value is false. For example, in a 5-pod component where: @@ -8514,6 +8516,7 @@ spec: SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before updating or scaling in pods with this role. This is typically used for leader roles to ensure minimal disruption during updates. + The default value is false. This field is immutable once set. @@ -8538,6 +8541,11 @@ spec: required: - name type: object + x-kubernetes-validations: + - message: Roles participate in quorum should have higher update + priority than roles do not participate in quorum. + rule: self.filter(c, c.participatesInQuorum == true).min() > self.filter(c, + c.participatesInQuorum == true).max() type: array runtime: description: |- diff --git a/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml b/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml index 5bba9ff8555..eb35e06d173 100644 --- a/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml +++ b/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml @@ -4279,7 +4279,9 @@ spec: default: false description: |- ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. - This affects update strategies that need to maintain quorum for availability. + This affects update strategies that need to maintain quorum for availability. Roles participate + in quorum should have higher update priority than roles do not participate in quorum. + The default value is false. For example, in a 5-pod component where: @@ -4298,6 +4300,7 @@ spec: SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before updating or scaling in pods with this role. This is typically used for leader roles to ensure minimal disruption during updates. + The default value is false. This field is immutable once set. @@ -4322,6 +4325,11 @@ spec: required: - name type: object + x-kubernetes-validations: + - message: Roles participate in quorum should have higher update + priority than roles do not participate in quorum. + rule: self.filter(c, c.participatesInQuorum == true).min() > self.filter(c, + c.participatesInQuorum == true).max() type: array selector: description: |- @@ -12491,7 +12499,9 @@ spec: default: false description: |- ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. - This affects update strategies that need to maintain quorum for availability. + This affects update strategies that need to maintain quorum for availability. Roles participate + in quorum should have higher update priority than roles do not participate in quorum. + The default value is false. For example, in a 5-pod component where: @@ -12510,6 +12520,7 @@ spec: SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before updating or scaling in pods with this role. This is typically used for leader roles to ensure minimal disruption during updates. + The default value is false. This field is immutable once set. @@ -12534,6 +12545,11 @@ spec: required: - name type: object + x-kubernetes-validations: + - message: Roles participate in quorum should have higher update + priority than roles do not participate in quorum. + rule: self.filter(c, c.participatesInQuorum == true).min() + > self.filter(c, c.participatesInQuorum == true).max() required: - podName type: object From d8e34de72ef3c59ed5089e7bab50affe53c9cdd4 Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Tue, 12 Nov 2024 14:33:46 +0800 Subject: [PATCH 18/34] comments --- apis/apps/v1/componentdefinition_types.go | 6 +++--- .../apps.kubeblocks.io_componentdefinitions.yaml | 10 +++++----- .../bases/workloads.kubeblocks.io_instancesets.yaml | 9 +++++---- .../apps.kubeblocks.io_componentdefinitions.yaml | 10 +++++----- .../crds/workloads.kubeblocks.io_instancesets.yaml | 9 +++++---- docs/developer_docs/api-reference/cluster.md | 12 ++++++------ 6 files changed, 29 insertions(+), 27 deletions(-) diff --git a/apis/apps/v1/componentdefinition_types.go b/apis/apps/v1/componentdefinition_types.go index 9c185b71057..a53491ce2ea 100644 --- a/apis/apps/v1/componentdefinition_types.go +++ b/apis/apps/v1/componentdefinition_types.go @@ -412,9 +412,9 @@ type ComponentDefinitionSpec struct { // Enumerate all possible roles assigned to each replica of the Component, influencing its behavior. // - // A replica can have zero to multiple roles. - // KubeBlocks operator determines the roles of each replica by invoking the `lifecycleActions.roleProbe` method. - // This action returns a list of roles for each replica, and the returned roles must be predefined in the `roles` field. + // A replica can have zero or one role. + // KubeBlocks operator determines the role of each replica by invoking the `lifecycleActions.roleProbe` method. + // This action returns the role for each replica, and the returned role must be predefined here. // // The roles assigned to a replica can influence various aspects of the Component's behavior, such as: // diff --git a/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml b/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml index 391296025f4..dcd88708879 100644 --- a/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml +++ b/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml @@ -8456,9 +8456,9 @@ spec: Enumerate all possible roles assigned to each replica of the Component, influencing its behavior. - A replica can have zero to multiple roles. - KubeBlocks operator determines the roles of each replica by invoking the `lifecycleActions.roleProbe` method. - This action returns a list of roles for each replica, and the returned roles must be predefined in the `roles` field. + A replica can have zero or one role. + KubeBlocks operator determines the role of each replica by invoking the `lifecycleActions.roleProbe` method. + This action returns the role for each replica, and the returned role must be predefined here. The roles assigned to a replica can influence various aspects of the Component's behavior, such as: @@ -8544,8 +8544,8 @@ spec: x-kubernetes-validations: - message: Roles participate in quorum should have higher update priority than roles do not participate in quorum. - rule: self.filter(c, c.participatesInQuorum == true).min() > self.filter(c, - c.participatesInQuorum == true).max() + rule: self.filter(x, x.participatesInQuorum == true).map(x, x.updatePriority).min() + > self.filter(x, x.participatesInQuorum == false).map(x, x.updatePriority).max() type: array runtime: description: |- diff --git a/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml b/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml index eb35e06d173..37cf39e37bb 100644 --- a/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml +++ b/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml @@ -4328,8 +4328,8 @@ spec: x-kubernetes-validations: - message: Roles participate in quorum should have higher update priority than roles do not participate in quorum. - rule: self.filter(c, c.participatesInQuorum == true).min() > self.filter(c, - c.participatesInQuorum == true).max() + rule: self.filter(x, x.participatesInQuorum == true).map(x, x.updatePriority).min() + > self.filter(x, x.participatesInQuorum == false).map(x, x.updatePriority).max() type: array selector: description: |- @@ -12548,8 +12548,9 @@ spec: x-kubernetes-validations: - message: Roles participate in quorum should have higher update priority than roles do not participate in quorum. - rule: self.filter(c, c.participatesInQuorum == true).min() - > self.filter(c, c.participatesInQuorum == true).max() + rule: self.filter(x, x.participatesInQuorum == true).map(x, + x.updatePriority).min() > self.filter(x, x.participatesInQuorum + == false).map(x, x.updatePriority).max() required: - podName type: object diff --git a/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml b/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml index 391296025f4..dcd88708879 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml @@ -8456,9 +8456,9 @@ spec: Enumerate all possible roles assigned to each replica of the Component, influencing its behavior. - A replica can have zero to multiple roles. - KubeBlocks operator determines the roles of each replica by invoking the `lifecycleActions.roleProbe` method. - This action returns a list of roles for each replica, and the returned roles must be predefined in the `roles` field. + A replica can have zero or one role. + KubeBlocks operator determines the role of each replica by invoking the `lifecycleActions.roleProbe` method. + This action returns the role for each replica, and the returned role must be predefined here. The roles assigned to a replica can influence various aspects of the Component's behavior, such as: @@ -8544,8 +8544,8 @@ spec: x-kubernetes-validations: - message: Roles participate in quorum should have higher update priority than roles do not participate in quorum. - rule: self.filter(c, c.participatesInQuorum == true).min() > self.filter(c, - c.participatesInQuorum == true).max() + rule: self.filter(x, x.participatesInQuorum == true).map(x, x.updatePriority).min() + > self.filter(x, x.participatesInQuorum == false).map(x, x.updatePriority).max() type: array runtime: description: |- diff --git a/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml b/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml index eb35e06d173..37cf39e37bb 100644 --- a/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml +++ b/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml @@ -4328,8 +4328,8 @@ spec: x-kubernetes-validations: - message: Roles participate in quorum should have higher update priority than roles do not participate in quorum. - rule: self.filter(c, c.participatesInQuorum == true).min() > self.filter(c, - c.participatesInQuorum == true).max() + rule: self.filter(x, x.participatesInQuorum == true).map(x, x.updatePriority).min() + > self.filter(x, x.participatesInQuorum == false).map(x, x.updatePriority).max() type: array selector: description: |- @@ -12548,8 +12548,9 @@ spec: x-kubernetes-validations: - message: Roles participate in quorum should have higher update priority than roles do not participate in quorum. - rule: self.filter(c, c.participatesInQuorum == true).min() - > self.filter(c, c.participatesInQuorum == true).max() + rule: self.filter(x, x.participatesInQuorum == true).map(x, + x.updatePriority).min() > self.filter(x, x.participatesInQuorum + == false).map(x, x.updatePriority).max() required: - podName type: object diff --git a/docs/developer_docs/api-reference/cluster.md b/docs/developer_docs/api-reference/cluster.md index 6fdadfd8cc2..84009ca3018 100644 --- a/docs/developer_docs/api-reference/cluster.md +++ b/docs/developer_docs/api-reference/cluster.md @@ -1352,9 +1352,9 @@ ComponentAvailable (Optional)

Enumerate all possible roles assigned to each replica of the Component, influencing its behavior.

-

A replica can have zero to multiple roles. -KubeBlocks operator determines the roles of each replica by invoking the lifecycleActions.roleProbe method. -This action returns a list of roles for each replica, and the returned roles must be predefined in the roles field.

+

A replica can have zero or one role. +KubeBlocks operator determines the role of each replica by invoking the lifecycleActions.roleProbe method. +This action returns the role for each replica, and the returned role must be predefined here.

The roles assigned to a replica can influence various aspects of the Component’s behavior, such as:

  • Service selection: The Component’s exposed Services may target replicas based on their roles using roleSelector.
    • @@ -5008,9 +5008,9 @@ ComponentAvailable (Optional)

    Enumerate all possible roles assigned to each replica of the Component, influencing its behavior.

    -

    A replica can have zero to multiple roles. -KubeBlocks operator determines the roles of each replica by invoking the lifecycleActions.roleProbe method. -This action returns a list of roles for each replica, and the returned roles must be predefined in the roles field.

    +

    A replica can have zero or one role. +KubeBlocks operator determines the role of each replica by invoking the lifecycleActions.roleProbe method. +This action returns the role for each replica, and the returned role must be predefined here.

    The roles assigned to a replica can influence various aspects of the Component’s behavior, such as:

    • Service selection: The Component’s exposed Services may target replicas based on their roles using roleSelector.
      • From 60456ae3e290a6ee64a40b4e2fee58832699fa7e Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Tue, 12 Nov 2024 15:07:28 +0800 Subject: [PATCH 19/34] fix cel --- apis/apps/v1/componentdefinition_types.go | 1 + apis/base/v1/types.go | 2 -- .../apps.kubeblocks.io_componentdefinitions.yaml | 10 +++++----- .../bases/workloads.kubeblocks.io_instancesets.yaml | 11 ----------- .../crds/apps.kubeblocks.io_componentdefinitions.yaml | 10 +++++----- .../crds/workloads.kubeblocks.io_instancesets.yaml | 11 ----------- 6 files changed, 11 insertions(+), 34 deletions(-) diff --git a/apis/apps/v1/componentdefinition_types.go b/apis/apps/v1/componentdefinition_types.go index a53491ce2ea..809bc5ba6b9 100644 --- a/apis/apps/v1/componentdefinition_types.go +++ b/apis/apps/v1/componentdefinition_types.go @@ -425,6 +425,7 @@ type ComponentDefinitionSpec struct { // // This field is immutable. // + // +kubebuilder:validation:XValidation:rule="self.filter(x, x.participatesInQuorum == true).map(x, x.updatePriority).min() > self.filter(x, x.participatesInQuorum == false).map(x, x.updatePriority).max()",message="Roles participate in quorum should have higher update priority than roles do not participate in quorum." // +optional Roles []ReplicaRole `json:"roles,omitempty"` diff --git a/apis/base/v1/types.go b/apis/base/v1/types.go index 3c242e7a2be..5ce8b01ea66 100644 --- a/apis/base/v1/types.go +++ b/apis/base/v1/types.go @@ -20,8 +20,6 @@ along with this program. If not, see . package v1 // ReplicaRole represents a role that can be assigned to a component instance, defining its behavior and responsibilities. -// -// +kubebuilder:validation:XValidation:rule="self.filter(x, x.participatesInQuorum == true).map(x, x.updatePriority).min() > self.filter(x, x.participatesInQuorum == false).map(x, x.updatePriority).max()",message="Roles participate in quorum should have higher update priority than roles do not participate in quorum." type ReplicaRole struct { // Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label // on the corresponding object to identify its role. diff --git a/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml b/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml index dcd88708879..8a98572ad76 100644 --- a/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml +++ b/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml @@ -8541,12 +8541,12 @@ spec: required: - name type: object - x-kubernetes-validations: - - message: Roles participate in quorum should have higher update - priority than roles do not participate in quorum. - rule: self.filter(x, x.participatesInQuorum == true).map(x, x.updatePriority).min() - > self.filter(x, x.participatesInQuorum == false).map(x, x.updatePriority).max() type: array + x-kubernetes-validations: + - message: Roles participate in quorum should have higher update priority + than roles do not participate in quorum. + rule: self.filter(x, x.participatesInQuorum == true).map(x, x.updatePriority).min() + > self.filter(x, x.participatesInQuorum == false).map(x, x.updatePriority).max() runtime: description: |- Specifies the PodSpec template used in the Component. diff --git a/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml b/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml index 37cf39e37bb..cd78f9b597f 100644 --- a/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml +++ b/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml @@ -4325,11 +4325,6 @@ spec: required: - name type: object - x-kubernetes-validations: - - message: Roles participate in quorum should have higher update - priority than roles do not participate in quorum. - rule: self.filter(x, x.participatesInQuorum == true).map(x, x.updatePriority).min() - > self.filter(x, x.participatesInQuorum == false).map(x, x.updatePriority).max() type: array selector: description: |- @@ -12545,12 +12540,6 @@ spec: required: - name type: object - x-kubernetes-validations: - - message: Roles participate in quorum should have higher update - priority than roles do not participate in quorum. - rule: self.filter(x, x.participatesInQuorum == true).map(x, - x.updatePriority).min() > self.filter(x, x.participatesInQuorum - == false).map(x, x.updatePriority).max() required: - podName type: object diff --git a/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml b/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml index dcd88708879..8a98572ad76 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml @@ -8541,12 +8541,12 @@ spec: required: - name type: object - x-kubernetes-validations: - - message: Roles participate in quorum should have higher update - priority than roles do not participate in quorum. - rule: self.filter(x, x.participatesInQuorum == true).map(x, x.updatePriority).min() - > self.filter(x, x.participatesInQuorum == false).map(x, x.updatePriority).max() type: array + x-kubernetes-validations: + - message: Roles participate in quorum should have higher update priority + than roles do not participate in quorum. + rule: self.filter(x, x.participatesInQuorum == true).map(x, x.updatePriority).min() + > self.filter(x, x.participatesInQuorum == false).map(x, x.updatePriority).max() runtime: description: |- Specifies the PodSpec template used in the Component. diff --git a/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml b/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml index 37cf39e37bb..cd78f9b597f 100644 --- a/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml +++ b/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml @@ -4325,11 +4325,6 @@ spec: required: - name type: object - x-kubernetes-validations: - - message: Roles participate in quorum should have higher update - priority than roles do not participate in quorum. - rule: self.filter(x, x.participatesInQuorum == true).map(x, x.updatePriority).min() - > self.filter(x, x.participatesInQuorum == false).map(x, x.updatePriority).max() type: array selector: description: |- @@ -12545,12 +12540,6 @@ spec: required: - name type: object - x-kubernetes-validations: - - message: Roles participate in quorum should have higher update - priority than roles do not participate in quorum. - rule: self.filter(x, x.participatesInQuorum == true).map(x, - x.updatePriority).min() > self.filter(x, x.participatesInQuorum - == false).map(x, x.updatePriority).max() required: - podName type: object From 9dadfa83c645b2f6a644e56393b1f5b412314e56 Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Tue, 12 Nov 2024 15:21:58 +0800 Subject: [PATCH 20/34] fix cel again --- apis/apps/v1/componentdefinition_types.go | 1 + apis/base/v1/types.go | 2 +- config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml | 3 ++- config/crd/bases/workloads.kubeblocks.io_instancesets.yaml | 4 ++-- deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml | 3 ++- deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml | 4 ++-- 6 files changed, 10 insertions(+), 7 deletions(-) diff --git a/apis/apps/v1/componentdefinition_types.go b/apis/apps/v1/componentdefinition_types.go index 809bc5ba6b9..2fcb4f90463 100644 --- a/apis/apps/v1/componentdefinition_types.go +++ b/apis/apps/v1/componentdefinition_types.go @@ -425,6 +425,7 @@ type ComponentDefinitionSpec struct { // // This field is immutable. // + // +kubebuilder:validation:MaxItems=128 // +kubebuilder:validation:XValidation:rule="self.filter(x, x.participatesInQuorum == true).map(x, x.updatePriority).min() > self.filter(x, x.participatesInQuorum == false).map(x, x.updatePriority).max()",message="Roles participate in quorum should have higher update priority than roles do not participate in quorum." // +optional Roles []ReplicaRole `json:"roles,omitempty"` diff --git a/apis/base/v1/types.go b/apis/base/v1/types.go index 5ce8b01ea66..ca5062495d1 100644 --- a/apis/base/v1/types.go +++ b/apis/base/v1/types.go @@ -68,7 +68,7 @@ type ReplicaRole struct { // // +kubebuilder:default=false // +optional - ParticipatesInQuorum bool `json:"participatesInQuorum bool"` + ParticipatesInQuorum bool `json:"participatesInQuorum"` // SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before // updating or scaling in pods with this role. This is typically used for leader roles to diff --git a/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml b/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml index 8a98572ad76..4bb47305950 100644 --- a/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml +++ b/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml @@ -8491,7 +8491,7 @@ spec: maxLength: 32 pattern: ^.*[^\s]+.*$ type: string - participatesInQuorum bool: + participatesInQuorum: default: false description: |- ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. @@ -8541,6 +8541,7 @@ spec: required: - name type: object + maxItems: 128 type: array x-kubernetes-validations: - message: Roles participate in quorum should have higher update priority diff --git a/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml b/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml index cd78f9b597f..4436892c2ed 100644 --- a/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml +++ b/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml @@ -4275,7 +4275,7 @@ spec: maxLength: 32 pattern: ^.*[^\s]+.*$ type: string - participatesInQuorum bool: + participatesInQuorum: default: false description: |- ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. @@ -12490,7 +12490,7 @@ spec: maxLength: 32 pattern: ^.*[^\s]+.*$ type: string - participatesInQuorum bool: + participatesInQuorum: default: false description: |- ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. diff --git a/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml b/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml index 8a98572ad76..4bb47305950 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml @@ -8491,7 +8491,7 @@ spec: maxLength: 32 pattern: ^.*[^\s]+.*$ type: string - participatesInQuorum bool: + participatesInQuorum: default: false description: |- ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. @@ -8541,6 +8541,7 @@ spec: required: - name type: object + maxItems: 128 type: array x-kubernetes-validations: - message: Roles participate in quorum should have higher update priority diff --git a/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml b/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml index cd78f9b597f..4436892c2ed 100644 --- a/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml +++ b/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml @@ -4275,7 +4275,7 @@ spec: maxLength: 32 pattern: ^.*[^\s]+.*$ type: string - participatesInQuorum bool: + participatesInQuorum: default: false description: |- ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. @@ -12490,7 +12490,7 @@ spec: maxLength: 32 pattern: ^.*[^\s]+.*$ type: string - participatesInQuorum bool: + participatesInQuorum: default: false description: |- ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. From 2e630a9e25f8e767603088840f6a02fb9289510b Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Tue, 12 Nov 2024 15:48:30 +0800 Subject: [PATCH 21/34] delete cel --- apis/apps/v1/componentdefinition_types.go | 6 +++++- .../crd/bases/apps.kubeblocks.io_componentdefinitions.yaml | 5 ----- .../helm/crds/apps.kubeblocks.io_componentdefinitions.yaml | 5 ----- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/apis/apps/v1/componentdefinition_types.go b/apis/apps/v1/componentdefinition_types.go index 2fcb4f90463..73ed6af8ce5 100644 --- a/apis/apps/v1/componentdefinition_types.go +++ b/apis/apps/v1/componentdefinition_types.go @@ -410,6 +410,11 @@ type ComponentDefinitionSpec struct { // +optional Available *ComponentAvailable `json:"available,omitempty"` + // FIXME: there's a bug in CEL's cost estimation when chaining .filter() and .map(). + // It was fixed in k8s 1.30, see: https://github.com/kubernetes/kubernetes/pull/123562. + // Maybe we can add this back later. + // TODO +kubebuilder:validation:XValidation:rule="self.filter(x, x.participatesInQuorum == true).map(x, x.updatePriority).min() > self.filter(x, x.participatesInQuorum == false).map(x, x.updatePriority).max()",message="Roles participate in quorum should have higher update priority than roles do not participate in quorum." + // Enumerate all possible roles assigned to each replica of the Component, influencing its behavior. // // A replica can have zero or one role. @@ -426,7 +431,6 @@ type ComponentDefinitionSpec struct { // This field is immutable. // // +kubebuilder:validation:MaxItems=128 - // +kubebuilder:validation:XValidation:rule="self.filter(x, x.participatesInQuorum == true).map(x, x.updatePriority).min() > self.filter(x, x.participatesInQuorum == false).map(x, x.updatePriority).max()",message="Roles participate in quorum should have higher update priority than roles do not participate in quorum." // +optional Roles []ReplicaRole `json:"roles,omitempty"` diff --git a/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml b/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml index 4bb47305950..bc1c1a491f9 100644 --- a/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml +++ b/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml @@ -8543,11 +8543,6 @@ spec: type: object maxItems: 128 type: array - x-kubernetes-validations: - - message: Roles participate in quorum should have higher update priority - than roles do not participate in quorum. - rule: self.filter(x, x.participatesInQuorum == true).map(x, x.updatePriority).min() - > self.filter(x, x.participatesInQuorum == false).map(x, x.updatePriority).max() runtime: description: |- Specifies the PodSpec template used in the Component. diff --git a/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml b/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml index 4bb47305950..bc1c1a491f9 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml @@ -8543,11 +8543,6 @@ spec: type: object maxItems: 128 type: array - x-kubernetes-validations: - - message: Roles participate in quorum should have higher update priority - than roles do not participate in quorum. - rule: self.filter(x, x.participatesInQuorum == true).map(x, x.updatePriority).min() - > self.filter(x, x.participatesInQuorum == false).map(x, x.updatePriority).max() runtime: description: |- Specifies the PodSpec template used in the Component. From b0e9c707d05dcb4b1a6fb9759fc60ace6f93818a Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Fri, 15 Nov 2024 11:23:30 +0800 Subject: [PATCH 22/34] make leaveMemberForPod cleaner --- .../apps/transformer_component_workload.go | 33 +++++++++++-------- .../apps/cluster_instance_set_test_util.go | 2 +- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/controllers/apps/transformer_component_workload.go b/controllers/apps/transformer_component_workload.go index 644fdc94dbc..688f03fe506 100644 --- a/controllers/apps/transformer_component_workload.go +++ b/controllers/apps/transformer_component_workload.go @@ -683,7 +683,7 @@ func (r *componentWorkloadOps) leaveMember4ScaleIn() error { } func (r *componentWorkloadOps) leaveMemberForPod(pod *corev1.Pod, pods []*corev1.Pod) error { - isLeader := func(pod *corev1.Pod) bool { + needSwtichover := func(pod *corev1.Pod) bool { if pod == nil || len(pod.Labels) == 0 { return false } @@ -700,12 +700,10 @@ func (r *componentWorkloadOps) leaveMemberForPod(pod *corev1.Pod, pods []*corev1 return false } - tryToSwitchover := func(lfa lifecycle.Lifecycle, pod *corev1.Pod) error { - // if pod is not leader/primary, no need to switchover - if !isLeader(pod) { + trySwitchover := func(lfa lifecycle.Lifecycle, pod *corev1.Pod) error { + if !needSwtichover(pod) { return nil } - // if HA functionality is not enabled, no need to switchover err := lfa.Switchover(r.reqCtx.Ctx, r.cli, nil, "") if err != nil && errors.Is(err, lifecycle.ErrActionNotDefined) { return nil @@ -716,8 +714,17 @@ func (r *componentWorkloadOps) leaveMemberForPod(pod *corev1.Pod, pods []*corev1 return err } - if !(isLeader(pod) || // if the pod is leader, it needs to call switchover - (r.synthesizeComp.LifecycleActions != nil && r.synthesizeComp.LifecycleActions.MemberLeave != nil)) { // if the memberLeave action is defined, it needs to call it + tryMemberLeave := func(lfa lifecycle.Lifecycle) error { + if r.synthesizeComp.LifecycleActions == nil || r.synthesizeComp.LifecycleActions.MemberLeave == nil { + return nil + } + + if err := lfa.MemberLeave(r.reqCtx.Ctx, r.cli, nil); err != nil { + if !errors.Is(err, lifecycle.ErrActionNotDefined) { + return err + } + } + return nil } @@ -726,16 +733,14 @@ func (r *componentWorkloadOps) leaveMemberForPod(pod *corev1.Pod, pods []*corev1 return err } - // switchover if the leaving pod is leader - if switchoverErr := tryToSwitchover(lfa, pod); switchoverErr != nil { - return switchoverErr + if err := trySwitchover(lfa, pod); err != nil { + return err } - if err = lfa.MemberLeave(r.reqCtx.Ctx, r.cli, nil); err != nil { - if !errors.Is(err, lifecycle.ErrActionNotDefined) && err == nil { - return err - } + if err := tryMemberLeave(lfa); err != nil { + return err } + return nil } diff --git a/pkg/testutil/apps/cluster_instance_set_test_util.go b/pkg/testutil/apps/cluster_instance_set_test_util.go index 832a6d5374c..51e05ca087b 100644 --- a/pkg/testutil/apps/cluster_instance_set_test_util.go +++ b/pkg/testutil/apps/cluster_instance_set_test_util.go @@ -314,7 +314,7 @@ func MockInstanceSetStatus(testCtx testutil.TestContext, cluster *appsv1.Cluster if _, ok := pod.Labels[constant.RoleLabelKey]; !ok { continue } - role := &workloads.ReplicaRole{} + var role *workloads.ReplicaRole for _, r := range its.Spec.Roles { if r.Name == pod.Labels[constant.RoleLabelKey] { role = r.DeepCopy() From 12ffec8cd3e1147a264c0fdd0cc956d5e2d26233 Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Fri, 15 Nov 2024 11:50:16 +0800 Subject: [PATCH 23/34] fix test --- .../transformer_component_workload_test.go | 24 ++++++++----------- .../component/lifecycle/lfa_member.go | 9 +++---- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/controllers/apps/transformer_component_workload_test.go b/controllers/apps/transformer_component_workload_test.go index 39349712e6b..e126aeeccfd 100644 --- a/controllers/apps/transformer_component_workload_test.go +++ b/controllers/apps/transformer_component_workload_test.go @@ -18,15 +18,11 @@ package apps import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/apimachinery/pkg/util/sets" - - kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" - workloads "github.com/apecloud/kubeblocks/apis/workloads/v1" - testk8s "github.com/apecloud/kubeblocks/pkg/testutil/k8s" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" "github.com/apecloud/kubeblocks/pkg/constant" "github.com/apecloud/kubeblocks/pkg/controller/component" @@ -34,6 +30,7 @@ import ( "github.com/apecloud/kubeblocks/pkg/controller/model" intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil" testapps "github.com/apecloud/kubeblocks/pkg/testutil/apps" + testk8s "github.com/apecloud/kubeblocks/pkg/testutil/k8s" ) var _ = Describe("Component Workload Operations Test", func() { @@ -49,6 +46,11 @@ var _ = Describe("Component Workload Operations Test", func() { synthesizeComp *component.SynthesizedComponent ) + roles := []kbappsv1.ReplicaRole{ + {Name: "leader", SwitchoverBeforeUpdate: true, UpdatePriority: 3}, + {Name: "follower", SwitchoverBeforeUpdate: false, UpdatePriority: 2}, + } + newDAG := func(graphCli model.GraphClient, comp *appsv1alpha1.Component) *graph.DAG { d := graph.NewDAG() graphCli.Root(d, comp, comp, model.ActionStatusPtr()) @@ -74,10 +76,7 @@ var _ = Describe("Component Workload Operations Test", func() { Namespace: testCtx.DefaultNamespace, ClusterName: clusterName, Name: compName, - Roles: []kbappsv1.ReplicaRole{ - {Name: "leader", Serviceable: true, Writable: true, Votable: true}, - {Name: "follower", Serviceable: false, Writable: false, Votable: false}, - }, + Roles: roles, LifecycleActions: &kbappsv1.ComponentLifecycleActions{ MemberJoin: &kbappsv1.Action{ Exec: &kbappsv1.ExecAction{ @@ -155,10 +154,7 @@ var _ = Describe("Component Workload Operations Test", func() { AddAppManagedByLabel(). AddAnnotations(constant.MemberJoinStatusAnnotationKey, ""). SetReplicas(2). - SetRoles([]workloads.ReplicaRole{ - {Name: "leader", AccessMode: workloads.ReadWriteMode, CanVote: true, IsLeader: true}, - {Name: "follower", AccessMode: workloads.ReadonlyMode, CanVote: true, IsLeader: false}, - }). + SetRoles(roles). GetObject() mockCluster := testapps.NewClusterFactory(testCtx.DefaultNamespace, "test-cluster", "test-def"). diff --git a/pkg/controller/component/lifecycle/lfa_member.go b/pkg/controller/component/lifecycle/lfa_member.go index d39a833a417..80d99b55cff 100644 --- a/pkg/controller/component/lifecycle/lfa_member.go +++ b/pkg/controller/component/lifecycle/lfa_member.go @@ -22,6 +22,7 @@ package lifecycle import ( "context" "fmt" + "math" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -173,13 +174,13 @@ func hackParameters4Switchover(ctx context.Context, cli client.Reader, namespace func leaderRole(roles []appsv1.ReplicaRole) (string, error) { // HACK: assume the role with highest priority to be leader - highestPriority := 0 - var role *appsv1.ReplicaRole + highestPriority := math.MinInt + var role string for _, r := range roles { if r.UpdatePriority > highestPriority { highestPriority = r.UpdatePriority - role = &r + role = r.Name } } - return role.Name, nil + return role, nil } From aa3010a0a7a6390024917228a8d11e143010a0f9 Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Fri, 22 Nov 2024 16:56:03 +0800 Subject: [PATCH 24/34] remove ReadyWithoutPrimary --- apis/workloads/v1/instanceset_types.go | 8 -------- pkg/constant/labels.go | 1 - pkg/controller/instanceset/reconciler_status.go | 16 ---------------- 3 files changed, 25 deletions(-) diff --git a/apis/workloads/v1/instanceset_types.go b/apis/workloads/v1/instanceset_types.go index d23cad5a84c..2fd5f3db167 100644 --- a/apis/workloads/v1/instanceset_types.go +++ b/apis/workloads/v1/instanceset_types.go @@ -291,14 +291,6 @@ type InstanceSetStatus struct { // +optional MembersStatus []MemberStatus `json:"membersStatus,omitempty"` - // Indicates whether it is required for the InstanceSet to have at least one primary instance ready. - // - // Deprecated: since instanceset no longer checks a "primary" role when doing ready check, this - // field is no longer needed and will be removed in the future. - // - // +optional - ReadyWithoutPrimary bool `json:"readyWithoutPrimary,omitempty"` - // currentRevisions, if not empty, indicates the old version of the InstanceSet used to generate the underlying workload. // key is the pod name, value is the revision. // diff --git a/pkg/constant/labels.go b/pkg/constant/labels.go index 5e0b1cd6f73..6fd48846de4 100644 --- a/pkg/constant/labels.go +++ b/pkg/constant/labels.go @@ -51,7 +51,6 @@ const ( RoleLabelKey = "kubeblocks.io/role" // RoleLabelKey consensusSet and replicationSet role label key KBAppServiceVersionKey = "apps.kubeblocks.io/service-version" - ReadyWithoutPrimaryKey = "kubeblocks.io/ready-without-primary" ) func GetClusterLabels(clusterName string, labels ...map[string]string) map[string]string { diff --git a/pkg/controller/instanceset/reconciler_status.go b/pkg/controller/instanceset/reconciler_status.go index 168a30b3cc5..bb8a302f539 100644 --- a/pkg/controller/instanceset/reconciler_status.go +++ b/pkg/controller/instanceset/reconciler_status.go @@ -31,7 +31,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" workloads "github.com/apecloud/kubeblocks/apis/workloads/v1" - "github.com/apecloud/kubeblocks/pkg/constant" "github.com/apecloud/kubeblocks/pkg/controller/kubebuilderx" "github.com/apecloud/kubeblocks/pkg/controller/model" intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil" @@ -190,10 +189,6 @@ func (r *statusReconciler) Reconcile(tree *kubebuilderx.ObjectTree) (kubebuilder // 4. set members status setMembersStatus(its, podList) - // 5. set readyWithoutPrimary - // TODO(free6om): should put this field to the spec - setReadyWithPrimary(its, podList) - if its.Spec.MinReadySeconds > 0 && availableReplicas != readyReplicas { return kubebuilderx.RetryAfter(time.Second), nil } @@ -292,17 +287,6 @@ func buildFailureCondition(its *workloads.InstanceSet, pods []*corev1.Pod) (*met }, nil } -func setReadyWithPrimary(its *workloads.InstanceSet, pods []*corev1.Pod) { - readyWithoutPrimary := false - for _, pod := range pods { - if value, ok := pod.Labels[constant.ReadyWithoutPrimaryKey]; ok && value == "true" { - readyWithoutPrimary = true - break - } - } - its.Status.ReadyWithoutPrimary = readyWithoutPrimary -} - func setMembersStatus(its *workloads.InstanceSet, pods []*corev1.Pod) { // no roles defined if its.Spec.Roles == nil { From 198ac42ac146f3fb6f3769cf52a938a239307114 Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Fri, 22 Nov 2024 16:57:48 +0800 Subject: [PATCH 25/34] remove legacy code --- .../component/lifecycle/lfa_member.go | 58 +------------------ 1 file changed, 1 insertion(+), 57 deletions(-) diff --git a/pkg/controller/component/lifecycle/lfa_member.go b/pkg/controller/component/lifecycle/lfa_member.go index 80d99b55cff..2c91b14fdfd 100644 --- a/pkg/controller/component/lifecycle/lfa_member.go +++ b/pkg/controller/component/lifecycle/lfa_member.go @@ -21,8 +21,6 @@ package lifecycle import ( "context" - "fmt" - "math" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -72,10 +70,7 @@ func (a *switchover) parameters(ctx context.Context, cli client.Reader) (map[str // // - KB_SWITCHOVER_CANDIDATE_NAME: The name of the pod for the new leader candidate, which may not be specified (empty). // - KB_SWITCHOVER_CANDIDATE_FQDN: The FQDN of the new leader candidate's pod, which may not be specified (empty). - m, err := hackParameters4Switchover(ctx, cli, a.namespace, a.clusterName, a.compName, a.roles) - if err != nil { - return nil, err - } + m := make(map[string]string) if len(a.candidate) > 0 { compName := constant.GenerateClusterComponentName(a.clusterName, a.compName) m[switchoverCandidateName] = a.candidate @@ -133,54 +128,3 @@ func (a *memberLeave) parameters(ctx context.Context, cli client.Reader) (map[st leaveMemberPodNameVar: a.pod.Name, }, nil } - -////////// hack for legacy Addons ////////// -// The container executing this action has access to following variables: -// -// - KB_LEADER_POD_IP: The IP address of the current leader's pod prior to the switchover. -// - KB_LEADER_POD_NAME: The name of the current leader's pod prior to the switchover. -// - KB_LEADER_POD_FQDN: The FQDN of the current leader's pod prior to the switchover. - -func hackParameters4Switchover(ctx context.Context, cli client.Reader, namespace, clusterName, compName string, roles []appsv1.ReplicaRole) (map[string]string, error) { - const ( - leaderPodName = "KB_LEADER_POD_NAME" - leaderPodFQDN = "KB_LEADER_POD_FQDN" - leaderPodIP = "KB_LEADER_POD_IP" - ) - - role, err := leaderRole(roles) - if err != nil { - return nil, err - } - - pods, err := component.ListOwnedPodsWithRole(ctx, cli, namespace, clusterName, compName, role) - if err != nil { - return nil, err - } - if len(pods) == 0 { - return nil, fmt.Errorf("has no pod with the leader role %s", role) - } - if len(pods) > 1 { - return nil, fmt.Errorf("more than one pod found as leader: %d, role: %s", len(pods), role) - } - - pod := pods[0] - return map[string]string{ - leaderPodName: pod.Name, - leaderPodFQDN: component.PodFQDN(namespace, constant.GenerateClusterComponentName(clusterName, compName), pod.Name), - leaderPodIP: pod.Status.PodIP, - }, nil -} - -func leaderRole(roles []appsv1.ReplicaRole) (string, error) { - // HACK: assume the role with highest priority to be leader - highestPriority := math.MinInt - var role string - for _, r := range roles { - if r.UpdatePriority > highestPriority { - highestPriority = r.UpdatePriority - role = r.Name - } - } - return role, nil -} From 2fe6d1f36b06757b4765bb9a0b056a0ce920b541 Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Fri, 22 Nov 2024 17:02:41 +0800 Subject: [PATCH 26/34] make manifests/api-doc --- .../workloads.kubeblocks.io_instancesets.yaml | 8 -------- .../crds/workloads.kubeblocks.io_instancesets.yaml | 8 -------- docs/developer_docs/api-reference/cluster.md | 14 -------------- 3 files changed, 30 deletions(-) diff --git a/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml b/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml index 4436892c2ed..14ecfaf9272 100644 --- a/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml +++ b/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml @@ -12562,14 +12562,6 @@ spec: this InstanceSet with a Ready Condition. format: int32 type: integer - readyWithoutPrimary: - description: |- - Indicates whether it is required for the InstanceSet to have at least one primary instance ready. - - - Deprecated: since instanceset no longer checks a "primary" role when doing ready check, this - field is no longer needed and will be removed in the future. - type: boolean replicas: description: replicas is the number of instances created by the InstanceSet controller. diff --git a/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml b/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml index 4436892c2ed..14ecfaf9272 100644 --- a/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml +++ b/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml @@ -12562,14 +12562,6 @@ spec: this InstanceSet with a Ready Condition. format: int32 type: integer - readyWithoutPrimary: - description: |- - Indicates whether it is required for the InstanceSet to have at least one primary instance ready. - - - Deprecated: since instanceset no longer checks a "primary" role when doing ready check, this - field is no longer needed and will be removed in the future. - type: boolean replicas: description: replicas is the number of instances created by the InstanceSet controller. diff --git a/docs/developer_docs/api-reference/cluster.md b/docs/developer_docs/api-reference/cluster.md index 4214c9fbaac..790ddb29d51 100644 --- a/docs/developer_docs/api-reference/cluster.md +++ b/docs/developer_docs/api-reference/cluster.md @@ -29413,20 +29413,6 @@ Used only when spec.roles set.

      -readyWithoutPrimary
      - -bool - - - -(Optional) -

      Indicates whether it is required for the InstanceSet to have at least one primary instance ready.

      -

      Deprecated: since instanceset no longer checks a “primary” role when doing ready check, this -field is no longer needed and will be removed in the future.

      - - - - currentRevisions
      map[string]string From 571e3b5b7c89d856bb109f8333b7b5b5957f7e67 Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Wed, 11 Dec 2024 12:03:50 +0800 Subject: [PATCH 27/34] make generate && make api-doc --- apis/workloads/v1/zz_generated.deepcopy.go | 22 ---- docs/developer_docs/api-reference/cluster.md | 113 ------------------- 2 files changed, 135 deletions(-) diff --git a/apis/workloads/v1/zz_generated.deepcopy.go b/apis/workloads/v1/zz_generated.deepcopy.go index c8e9b0fa858..2f97c2e9d4f 100644 --- a/apis/workloads/v1/zz_generated.deepcopy.go +++ b/apis/workloads/v1/zz_generated.deepcopy.go @@ -465,28 +465,6 @@ func (in *Range) DeepCopy() *Range { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RoleProbe) DeepCopyInto(out *RoleProbe) { - *out = *in - if in.CustomHandler != nil { - in, out := &in.CustomHandler, &out.CustomHandler - *out = make([]Action, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RoleProbe. -func (in *RoleProbe) DeepCopy() *RoleProbe { - if in == nil { - return nil - } - out := new(RoleProbe) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SchedulingPolicy) DeepCopyInto(out *SchedulingPolicy) { *out = *in diff --git a/docs/developer_docs/api-reference/cluster.md b/docs/developer_docs/api-reference/cluster.md index 7540168e7c7..56a5b792c72 100644 --- a/docs/developer_docs/api-reference/cluster.md +++ b/docs/developer_docs/api-reference/cluster.md @@ -30256,119 +30256,6 @@ int32 -

      RoleProbe -

      -

      -(Appears on:InstanceSetSpec) -

      -
      -

      RoleProbe defines how to observe role

      -
      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      FieldDescription
      -customHandler
      - - -[]Action - - -      		 -(Optional) -

      Defines a custom method for role probing. -Actions defined here are executed in series. -Upon completion of all actions, the final output should be a single string representing the role name defined in spec.Roles. -The latest BusyBox image will be used if Image is not configured. -Environment variables can be used in Command: -- v_KB_ITS_LASTSTDOUT: stdout from the last action, watch for ‘v’ prefix -- KB_ITS_USERNAME: username part of the credential -- KB_ITS_PASSWORD: password part of the credential

      -
      -initialDelaySeconds
      - -int32 - -      		 -(Optional) -

      Specifies the number of seconds to wait after the container has started before initiating role probing.

      -
      -timeoutSeconds
      - -int32 - -      		 -(Optional) -

      Specifies the number of seconds after which the probe times out.

      -
      -periodSeconds
      - -int32 - -      		 -(Optional) -

      Specifies the frequency (in seconds) of probe execution.

      -
      -successThreshold
      - -int32 - -      		 -(Optional) -

      Specifies the minimum number of consecutive successes for the probe to be considered successful after having failed.

      -
      -failureThreshold
      - -int32 - -      		 -(Optional) -

      Specifies the minimum number of consecutive failures for the probe to be considered failed after having succeeded.

      -
      -roleUpdateMechanism
      - - -RoleUpdateMechanism - - -      		 -(Optional) -

      Specifies the method for updating the pod role label.

      -

      RoleUpdateMechanism (string alias)

      From 0e5569b7742b64ff97241433c7c49521baf33260 Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Fri, 13 Dec 2024 14:21:51 +0800 Subject: [PATCH 28/34] remove swtichoverBeforeUpdate --- apis/base/v1/types.go | 11 -------- controllers/apps/component_controller_test.go | 21 ++++++-------- .../apps/configuration/policy_util_test.go | 14 ++++------ .../transformer_component_workload_test.go | 4 +-- controllers/k8score/event_controller_test.go | 14 ++++------ .../builder/builder_component_definition.go | 9 +++--- .../builder/builder_instance_set_test.go | 7 ++--- .../pod_role_event_handler_test.go | 1 - pkg/controller/instanceset/suite_test.go | 28 ++++++++----------- pkg/controllerutil/pod_utils_test.go | 1 - .../apps/cluster_instance_set_test_util.go | 14 ++++------ pkg/testutil/apps/constant.go | 21 ++++++-------- 12 files changed, 57 insertions(+), 88 deletions(-) diff --git a/apis/base/v1/types.go b/apis/base/v1/types.go index ca5062495d1..a1d55a01862 100644 --- a/apis/base/v1/types.go +++ b/apis/base/v1/types.go @@ -69,15 +69,4 @@ type ReplicaRole struct { // +kubebuilder:default=false // +optional ParticipatesInQuorum bool `json:"participatesInQuorum"` - - // SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before - // updating or scaling in pods with this role. This is typically used for leader roles to - // ensure minimal disruption during updates. - // The default value is false. - // - // This field is immutable once set. - // - // +kubebuilder:default=false - // +optional - SwitchoverBeforeUpdate bool `json:"switchoverBeforeUpdate"` } diff --git a/controllers/apps/component_controller_test.go b/controllers/apps/component_controller_test.go index b94753e67d1..e0152e5e26b 100644 --- a/controllers/apps/component_controller_test.go +++ b/controllers/apps/component_controller_test.go @@ -1247,22 +1247,19 @@ var _ = Describe("Component Controller", func() { By("check default component roles") targetRoles := []workloads.ReplicaRole{ { - Name: "leader", - SwitchoverBeforeUpdate: true, - ParticipatesInQuorum: true, - UpdatePriority: 5, + Name: "leader", + ParticipatesInQuorum: true, + UpdatePriority: 5, }, { - Name: "follower", - SwitchoverBeforeUpdate: false, - ParticipatesInQuorum: true, - UpdatePriority: 4, + Name: "follower", + ParticipatesInQuorum: true, + UpdatePriority: 4, }, { - Name: "learner", - SwitchoverBeforeUpdate: false, - ParticipatesInQuorum: false, - UpdatePriority: 2, + Name: "learner", + ParticipatesInQuorum: false, + UpdatePriority: 2, }, } itsKey := types.NamespacedName{ diff --git a/controllers/apps/configuration/policy_util_test.go b/controllers/apps/configuration/policy_util_test.go index 6f5793ce486..4859b8e1a4d 100644 --- a/controllers/apps/configuration/policy_util_test.go +++ b/controllers/apps/configuration/policy_util_test.go @@ -164,16 +164,14 @@ func newMockReconfigureParams(testName string, cli client.Client, paramOps ...Pa MinReadySeconds: 5, Roles: []appsv1.ReplicaRole{ { - Name: "leader", - SwitchoverBeforeUpdate: true, - ParticipatesInQuorum: true, - UpdatePriority: 5, + Name: "leader", + ParticipatesInQuorum: true, + UpdatePriority: 5, }, { - Name: "follower", - SwitchoverBeforeUpdate: false, - ParticipatesInQuorum: true, - UpdatePriority: 4, + Name: "follower", + ParticipatesInQuorum: true, + UpdatePriority: 4, }, }, }, diff --git a/controllers/apps/transformer_component_workload_test.go b/controllers/apps/transformer_component_workload_test.go index 2cc4ad7887d..967f85b7e0f 100644 --- a/controllers/apps/transformer_component_workload_test.go +++ b/controllers/apps/transformer_component_workload_test.go @@ -47,8 +47,8 @@ var _ = Describe("Component Workload Operations Test", func() { ) roles := []appsv1.ReplicaRole{ - {Name: "leader", SwitchoverBeforeUpdate: true, UpdatePriority: 3}, - {Name: "follower", SwitchoverBeforeUpdate: false, UpdatePriority: 2}, + {Name: "leader", UpdatePriority: 3}, + {Name: "follower", UpdatePriority: 2}, } newDAG := func(graphCli model.GraphClient, comp *appsv1.Component) *graph.DAG { diff --git a/controllers/k8score/event_controller_test.go b/controllers/k8score/event_controller_test.go index e7269abfc00..a716ac6201b 100644 --- a/controllers/k8score/event_controller_test.go +++ b/controllers/k8score/event_controller_test.go @@ -143,16 +143,14 @@ var _ = Describe("Event Controller", func() { Expect(testapps.GetAndChangeObj(&testCtx, client.ObjectKeyFromObject(its), func(tmpITS *workloads.InstanceSet) { tmpITS.Spec.Roles = []workloads.ReplicaRole{ { - Name: "leader", - SwitchoverBeforeUpdate: true, - ParticipatesInQuorum: true, - UpdatePriority: 5, + Name: "leader", + ParticipatesInQuorum: true, + UpdatePriority: 5, }, { - Name: "follower", - SwitchoverBeforeUpdate: false, - ParticipatesInQuorum: true, - UpdatePriority: 4, + Name: "follower", + ParticipatesInQuorum: true, + UpdatePriority: 4, }, } })()).Should(Succeed()) diff --git a/pkg/controller/builder/builder_component_definition.go b/pkg/controller/builder/builder_component_definition.go index e97c0e81cf1..d1fda3b6a6c 100644 --- a/pkg/controller/builder/builder_component_definition.go +++ b/pkg/controller/builder/builder_component_definition.go @@ -211,13 +211,12 @@ func (builder *ComponentDefinitionBuilder) SetUpdateStrategy(strategy *appsv1.Up } func (builder *ComponentDefinitionBuilder) AddRole( - name string, updatePriority int, participatesInQuorum bool, switchoverBeforeUpdate bool, + name string, updatePriority int, participatesInQuorum bool, ) *ComponentDefinitionBuilder { role := appsv1.ReplicaRole{ - Name: name, - UpdatePriority: updatePriority, - ParticipatesInQuorum: participatesInQuorum, - SwitchoverBeforeUpdate: switchoverBeforeUpdate, + Name: name, + UpdatePriority: updatePriority, + ParticipatesInQuorum: participatesInQuorum, } if builder.get().Spec.Roles == nil { builder.get().Spec.Roles = make([]appsv1.ReplicaRole, 0) diff --git a/pkg/controller/builder/builder_instance_set_test.go b/pkg/controller/builder/builder_instance_set_test.go index 0a5d6867979..85aa5289d77 100644 --- a/pkg/controller/builder/builder_instance_set_test.go +++ b/pkg/controller/builder/builder_instance_set_test.go @@ -50,10 +50,9 @@ var _ = Describe("instance_set builder", func() { parallelPodManagementConcurrency := &intstr.IntOrString{Type: intstr.String, StrVal: "100%"} selectors := map[string]string{selectorKey4: selectorValue4} role := workloads.ReplicaRole{ - Name: "foo", - SwitchoverBeforeUpdate: true, - ParticipatesInQuorum: true, - UpdatePriority: 1, + Name: "foo", + ParticipatesInQuorum: true, + UpdatePriority: 1, } reconfiguration := workloads.MembershipReconfiguration{ SwitchoverAction: &workloads.Action{ diff --git a/pkg/controller/instanceset/pod_role_event_handler_test.go b/pkg/controller/instanceset/pod_role_event_handler_test.go index d63c8d178f2..f2b4bfdd53f 100644 --- a/pkg/controller/instanceset/pod_role_event_handler_test.go +++ b/pkg/controller/instanceset/pod_role_event_handler_test.go @@ -56,7 +56,6 @@ var _ = Describe("pod role label event handler test", func() { } role := workloads.ReplicaRole{ Name: "leader", - SwitchoverBeforeUpdate: true, ParticipatesInQuorum: true, UpdatePriority: 5, } diff --git a/pkg/controller/instanceset/suite_test.go b/pkg/controller/instanceset/suite_test.go index f348b0913a3..d98c9aeadcf 100644 --- a/pkg/controller/instanceset/suite_test.go +++ b/pkg/controller/instanceset/suite_test.go @@ -75,28 +75,24 @@ var ( } roles = []workloads.ReplicaRole{ { - Name: "leader", - SwitchoverBeforeUpdate: true, - ParticipatesInQuorum: true, - UpdatePriority: 5, + Name: "leader", + ParticipatesInQuorum: true, + UpdatePriority: 5, }, { - Name: "follower", - SwitchoverBeforeUpdate: false, - ParticipatesInQuorum: true, - UpdatePriority: 4, + Name: "follower", + ParticipatesInQuorum: true, + UpdatePriority: 4, }, { - Name: "logger", - SwitchoverBeforeUpdate: false, - ParticipatesInQuorum: false, - UpdatePriority: 3, + Name: "logger", + ParticipatesInQuorum: false, + UpdatePriority: 3, }, { - Name: "learner", - SwitchoverBeforeUpdate: false, - ParticipatesInQuorum: false, - UpdatePriority: 2, + Name: "learner", + ParticipatesInQuorum: false, + UpdatePriority: 2, }, } pod = builder.NewPodBuilder("", ""). diff --git a/pkg/controllerutil/pod_utils_test.go b/pkg/controllerutil/pod_utils_test.go index d009f434325..4cdf3506c4c 100644 --- a/pkg/controllerutil/pod_utils_test.go +++ b/pkg/controllerutil/pod_utils_test.go @@ -626,7 +626,6 @@ var _ = Describe("pod utils", func() { AddAppComponentLabel(compName). AddAppManagedByLabel(). AddRoleLabel(role). - AddAccessModeLabel(mode). AddControllerRevisionHashLabel(""). AddVolume(corev1.Volume{ Name: testapps.DataVolumeName, diff --git a/pkg/testutil/apps/cluster_instance_set_test_util.go b/pkg/testutil/apps/cluster_instance_set_test_util.go index a596f8a54c2..3e49615fe7c 100644 --- a/pkg/testutil/apps/cluster_instance_set_test_util.go +++ b/pkg/testutil/apps/cluster_instance_set_test_util.go @@ -77,16 +77,14 @@ func MockInstanceSetComponent( AddContainer(corev1.Container{Name: DefaultMySQLContainerName, Image: ApeCloudMySQLImage}). SetRoles([]workloads.ReplicaRole{ { - Name: "leader", - SwitchoverBeforeUpdate: true, - ParticipatesInQuorum: true, - UpdatePriority: 5, + Name: "leader", + ParticipatesInQuorum: true, + UpdatePriority: 5, }, { - Name: "follower", - SwitchoverBeforeUpdate: false, - ParticipatesInQuorum: true, - UpdatePriority: 4, + Name: "follower", + ParticipatesInQuorum: true, + UpdatePriority: 4, }, }).Create(testCtx).GetObject() } diff --git a/pkg/testutil/apps/constant.go b/pkg/testutil/apps/constant.go index 86816fac3ea..531a7721d4c 100644 --- a/pkg/testutil/apps/constant.go +++ b/pkg/testutil/apps/constant.go @@ -209,22 +209,19 @@ var ( UpdateStrategy: &[]appsv1.UpdateStrategy{appsv1.BestEffortParallelStrategy}[0], Roles: []appsv1.ReplicaRole{ { - Name: "leader", - SwitchoverBeforeUpdate: true, - ParticipatesInQuorum: true, - UpdatePriority: 5, + Name: "leader", + ParticipatesInQuorum: true, + UpdatePriority: 5, }, { - Name: "follower", - SwitchoverBeforeUpdate: false, - ParticipatesInQuorum: true, - UpdatePriority: 4, + Name: "follower", + ParticipatesInQuorum: true, + UpdatePriority: 4, }, { - Name: "learner", - SwitchoverBeforeUpdate: false, - ParticipatesInQuorum: false, - UpdatePriority: 2, + Name: "learner", + ParticipatesInQuorum: false, + UpdatePriority: 2, }, }, Exporter: &appsv1.Exporter{ From 6fc0fdac74ee4b77b56075337a192419d3c98b1b Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Fri, 13 Dec 2024 14:22:09 +0800 Subject: [PATCH 29/34] switchover refactor --- apis/apps/v1/componentdefinition_types.go | 15 +-- apis/operations/v1alpha1/opsrequest_types.go | 18 ++-- .../v1alpha1/opsrequest_validation.go | 67 ++++++------- apis/workloads/v1/instanceset_types.go | 2 +- ...ps.kubeblocks.io_componentdefinitions.yaml | 24 ++--- .../workloads.kubeblocks.io_instancesets.yaml | 22 ----- .../apps/transformer_component_workload.go | 13 +-- ...ps.kubeblocks.io_componentdefinitions.yaml | 24 ++--- .../workloads.kubeblocks.io_instancesets.yaml | 22 ----- pkg/controller/component/lifecycle/kbagent.go | 16 ++- .../component/lifecycle/lfa_member.go | 24 +++-- pkg/operations/switchover.go | 28 +++--- pkg/operations/switchover_util.go | 98 ++++++------------- 13 files changed, 144 insertions(+), 229 deletions(-) diff --git a/apis/apps/v1/componentdefinition_types.go b/apis/apps/v1/componentdefinition_types.go index 4594a09f086..bad1faf2125 100644 --- a/apis/apps/v1/componentdefinition_types.go +++ b/apis/apps/v1/componentdefinition_types.go @@ -506,7 +506,7 @@ type ComponentDefinitionSpec struct { // `Immediately`, `RuntimeReady`, `ComponentReady`, and `ClusterReady`. // - `preTerminate`: Defines the hook to be executed before terminating a Component. // - `roleProbe`: Defines the procedure which is invoked regularly to assess the role of replicas. - // - `switchover`: Defines the procedure for a controlled transition of leadership from the current leader to a new replica. + // - `switchover`: Defines the procedure for a controlled transition of a role to a new replica. // This approach aims to minimize downtime and maintain availability in systems with a leader-follower topology, // such as before planned maintenance or upgrades on the current leader node. // - `memberJoin`: Defines the procedure to add a new replica to the replication group. @@ -1550,15 +1550,18 @@ type ComponentLifecycleActions struct { // +optional AvailableProbe *Probe `json:"availableProbe,omitempty"` - // Defines the procedure for a controlled transition of leadership from the current leader to a new replica. - // This approach aims to minimize downtime and maintain availability in systems with a leader-follower topology, + // Defines the procedure for a controlled transition of a role to a new replica. + // This approach aims to minimize downtime and maintain availability // during events such as planned maintenance or when performing stop, shutdown, restart, or upgrade operations // involving the current leader node. // // The container executing this action has access to following variables: // - // - KB_SWITCHOVER_CANDIDATE_NAME: The name of the pod for the new leader candidate, which may not be specified (empty). - // - KB_SWITCHOVER_CANDIDATE_FQDN: The FQDN of the new leader candidate's pod, which may not be specified (empty). + // - KB_SWITCHOVER_CANDIDATE_NAME: The name of the pod of the new role's candidate, which may not be specified (empty). + // - KB_SWITCHOVER_CANDIDATE_FQDN: The FQDN of the pod of the new role's candidate, which may not be specified (empty). + // - KB_SWITCHOVER_CURRENT_NAME: The name of the pod of the current role. + // - KB_SWITCHOVER_CURRENT_FQDN: The FQDN of the pod of the current role. + // - KB_SWITCHOVER_ROLE: The role that will be transferred to another replica. // // Note: This field is immutable once it has been set. // @@ -1746,7 +1749,7 @@ type ComponentLifecycleActions struct { // `Immediately`, `RuntimeReady`, `ComponentReady`, and `ClusterReady`. // - `preTerminate`: Defines the hook to be executed before terminating a Component. // - `roleProbe`: Defines the procedure which is invoked regularly to assess the role of replicas. -// - `switchover`: Defines the procedure for a controlled transition of leadership from the current leader to a new replica. +// - `switchover`: Defines the procedure for a controlled transition of a role to a new replica. // This approach aims to minimize downtime and maintain availability in systems with a leader-follower topology, // such as during planned maintenance or upgrades on the current leader node. // - `memberJoin`: Defines the procedure to add a new replica to the replication group. diff --git a/apis/operations/v1alpha1/opsrequest_types.go b/apis/operations/v1alpha1/opsrequest_types.go index 656540374ab..84b587adae0 100644 --- a/apis/operations/v1alpha1/opsrequest_types.go +++ b/apis/operations/v1alpha1/opsrequest_types.go @@ -302,19 +302,17 @@ type Switchover struct { // Specifies the name of the Component. ComponentOps `json:",inline"` - // Specifies the instance to become the primary or leader during a switchover operation. - // - // The value of `instanceName` can be either: - // - // 1. "*" (wildcard value): - // - Indicates no specific instance is designated as the primary or leader. - // - // 2. A valid instance name (pod name): - // - Designates a specific instance (pod) as the primary or leader. - // - The name must match one of the pods in the component. Any non-valid pod name is considered invalid. + // Specifies the instance whose role will be transferred. A typical usage is to transfer the leader role + // in a consensus system. // // +kubebuilder:validation:Required InstanceName string `json:"instanceName"` + + // If CandidateName is specified, the role will be transferred to this instance. + // The name must match one of the pods in the component. + // Refer to ComponentDefinition's Swtichover lifecycle action for more details. + // +optional + CandidateName string `json:"candidateName,omitempty"` } // Upgrade defines the parameters for an upgrade operation. diff --git a/apis/operations/v1alpha1/opsrequest_validation.go b/apis/operations/v1alpha1/opsrequest_validation.go index eec11800b18..cc614d43960 100644 --- a/apis/operations/v1alpha1/opsrequest_validation.go +++ b/apis/operations/v1alpha1/opsrequest_validation.go @@ -793,31 +793,12 @@ func GetRunningOpsByOpsType(ctx context.Context, cli client.Client, // validateSwitchoverResourceList checks if switchover resourceList is legal. func validateSwitchoverResourceList(ctx context.Context, cli client.Client, cluster *appsv1.Cluster, switchoverList []Switchover) error { - var ( - targetRole string - ) for _, switchover := range switchoverList { if switchover.InstanceName == "" { return notEmptyError("switchover.instanceName") } validateBaseOnCompDef := func(compDef string) error { - getTargetRole := func(roles []appsv1.ReplicaRole) (string, error) { - targetRole = "" - if len(roles) == 0 { - return targetRole, errors.New("component has no roles definition, does not support switchover") - } - for _, role := range roles { - // FIXME: the assumption that only one role supports switchover may change in the future - if role.SwitchoverBeforeUpdate { - if targetRole != "" { - return targetRole, errors.New("componentDefinition has more than role that needs switchover before, does not support switchover") - } - targetRole = role.Name - } - } - return targetRole, nil - } compDefObj, err := getComponentDefByName(ctx, cli, compDef) if err != nil { return err @@ -832,27 +813,47 @@ func validateSwitchoverResourceList(ctx context.Context, cli client.Client, clus if switchover.InstanceName == KBSwitchoverCandidateInstanceForAnyPod { return nil } - targetRole, err = getTargetRole(compDefObj.Spec.Roles) + if len(compDefObj.Spec.Roles) == 0 { + return errors.New("component has no roles definition, does not support switchover") + } + + getPod := func(name string) (*corev1.Pod, error) { + pod := &corev1.Pod{} + if err := cli.Get(ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: name}, pod); err != nil { + return nil, fmt.Errorf("get instanceName %s failed, err: %s, and check the validity of the instanceName using \"kbcli cluster list-instances\"", name, err.Error()) + } + return pod, nil + } + + checkOwnership := func(pod *corev1.Pod) error { + if !strings.HasPrefix(pod.Name, fmt.Sprintf("%s-%s", cluster.Name, switchover.ComponentName)) { + return fmt.Errorf("instanceName %s does not belong to the current component, please check the validity of the instance using \"kbcli cluster list-instances\"", switchover.InstanceName) + } + return nil + } + + pod, err := getPod(switchover.InstanceName) if err != nil { return err } - if targetRole == "" { - return errors.New("componentDefinition has no role is serviceable and writable, does not support switchover") - } - pod := &corev1.Pod{} - if err := cli.Get(ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: switchover.InstanceName}, pod); err != nil { - return fmt.Errorf("get instanceName %s failed, err: %s, and check the validity of the instanceName using \"kbcli cluster list-instances\"", switchover.InstanceName, err.Error()) + if err := checkOwnership(pod); err != nil { + return err } - v, ok := pod.Labels[constant.RoleLabelKey] - if !ok || v == "" { + roleName, ok := pod.Labels[constant.RoleLabelKey] + if !ok || roleName == "" { return fmt.Errorf("instanceName %s cannot be promoted because it had a invalid role label", switchover.InstanceName) } - if v == targetRole { - return fmt.Errorf("instanceName %s cannot be promoted because it is already the primary or leader instance", switchover.InstanceName) - } - if !strings.HasPrefix(pod.Name, fmt.Sprintf("%s-%s", cluster.Name, switchover.ComponentName)) { - return fmt.Errorf("instanceName %s does not belong to the current component, please check the validity of the instance using \"kbcli cluster list-instances\"", switchover.InstanceName) + + if switchover.CandidateName != "" { + candidatePod, err := getPod(switchover.InstanceName) + if err != nil { + return err + } + if err := checkOwnership(candidatePod); err != nil { + return err + } } + return nil } diff --git a/apis/workloads/v1/instanceset_types.go b/apis/workloads/v1/instanceset_types.go index 336eeac34ab..0ba618c6f77 100644 --- a/apis/workloads/v1/instanceset_types.go +++ b/apis/workloads/v1/instanceset_types.go @@ -192,7 +192,7 @@ type InstanceSetSpec struct { // Note: This field will be removed in future version. UpdateStrategy appsv1.StatefulSetUpdateStrategy `json:"updateStrategy,omitempty"` - // A list of roles defined in the system. + // A list of roles defined in the system. Instanceset obtains role through pods' role label `kubeblocks.io/role`. // // +optional Roles []ReplicaRole `json:"roles,omitempty"` diff --git a/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml b/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml index 21ed4bdee21..52e98fb6cce 100644 --- a/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml +++ b/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml @@ -4513,7 +4513,7 @@ spec: `Immediately`, `RuntimeReady`, `ComponentReady`, and `ClusterReady`. - `preTerminate`: Defines the hook to be executed before terminating a Component. - `roleProbe`: Defines the procedure which is invoked regularly to assess the role of replicas. - - `switchover`: Defines the procedure for a controlled transition of leadership from the current leader to a new replica. + - `switchover`: Defines the procedure for a controlled transition of a role to a new replica. This approach aims to minimize downtime and maintain availability in systems with a leader-follower topology, such as before planned maintenance or upgrades on the current leader node. - `memberJoin`: Defines the procedure to add a new replica to the replication group. @@ -7992,8 +7992,8 @@ spec: type: object switchover: description: |- - Defines the procedure for a controlled transition of leadership from the current leader to a new replica. - This approach aims to minimize downtime and maintain availability in systems with a leader-follower topology, + Defines the procedure for a controlled transition of a role to a new replica. + This approach aims to minimize downtime and maintain availability during events such as planned maintenance or when performing stop, shutdown, restart, or upgrade operations involving the current leader node. @@ -8001,8 +8001,11 @@ spec: The container executing this action has access to following variables: - - KB_SWITCHOVER_CANDIDATE_NAME: The name of the pod for the new leader candidate, which may not be specified (empty). - - KB_SWITCHOVER_CANDIDATE_FQDN: The FQDN of the new leader candidate's pod, which may not be specified (empty). + - KB_SWITCHOVER_CANDIDATE_NAME: The name of the pod of the new role's candidate, which may not be specified (empty). + - KB_SWITCHOVER_CANDIDATE_FQDN: The FQDN of the pod of the new role's candidate, which may not be specified (empty). + - KB_SWITCHOVER_CURRENT_NAME: The name of the pod of the current role. + - KB_SWITCHOVER_CURRENT_FQDN: The FQDN of the pod of the current role. + - KB_SWITCHOVER_ROLE: The role that will be transferred to another replica. Note: This field is immutable once it has been set. @@ -8514,17 +8517,6 @@ spec: of 2 learners and 1 follower while maintaining quorum. - This field is immutable once set. - type: boolean - switchoverBeforeUpdate: - default: false - description: |- - SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before - updating or scaling in pods with this role. This is typically used for leader roles to - ensure minimal disruption during updates. - The default value is false. - - This field is immutable once set. type: boolean updatePriority: diff --git a/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml b/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml index 0451d732107..55aa6c4feab 100644 --- a/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml +++ b/config/crd/bases/workloads.kubeblocks.io_instancesets.yaml @@ -4213,17 +4213,6 @@ spec: of 2 learners and 1 follower while maintaining quorum. - This field is immutable once set. - type: boolean - switchoverBeforeUpdate: - default: false - description: |- - SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before - updating or scaling in pods with this role. This is typically used for leader roles to - ensure minimal disruption during updates. - The default value is false. - - This field is immutable once set. type: boolean updatePriority: @@ -12428,17 +12417,6 @@ spec: of 2 learners and 1 follower while maintaining quorum. - This field is immutable once set. - type: boolean - switchoverBeforeUpdate: - default: false - description: |- - SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before - updating or scaling in pods with this role. This is typically used for leader roles to - ensure minimal disruption during updates. - The default value is false. - - This field is immutable once set. type: boolean updatePriority: diff --git a/controllers/apps/transformer_component_workload.go b/controllers/apps/transformer_component_workload.go index 7d96628b1d0..d80121ee00d 100644 --- a/controllers/apps/transformer_component_workload.go +++ b/controllers/apps/transformer_component_workload.go @@ -684,17 +684,9 @@ func (r *componentWorkloadOps) leaveMemberForPod(pod *corev1.Pod, pods []*corev1 if pod == nil || len(pod.Labels) == 0 { return false } - roleName, ok := pod.Labels[constant.RoleLabelKey] - if !ok { - return false - } + _, ok := pod.Labels[constant.RoleLabelKey] - for _, replicaRole := range r.runningITS.Spec.Roles { - if roleName == replicaRole.Name && replicaRole.SwitchoverBeforeUpdate { - return true - } - } - return false + return ok } trySwitchover := func(lfa lifecycle.Lifecycle, pod *corev1.Pod) error { @@ -706,6 +698,7 @@ func (r *componentWorkloadOps) leaveMemberForPod(pod *corev1.Pod, pods []*corev1 return nil } if err == nil { + // FIXME: compare role labels after switchover succeeds return fmt.Errorf("switchover succeed, wait role label to be updated") } return err diff --git a/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml b/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml index 21ed4bdee21..52e98fb6cce 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml @@ -4513,7 +4513,7 @@ spec: `Immediately`, `RuntimeReady`, `ComponentReady`, and `ClusterReady`. - `preTerminate`: Defines the hook to be executed before terminating a Component. - `roleProbe`: Defines the procedure which is invoked regularly to assess the role of replicas. - - `switchover`: Defines the procedure for a controlled transition of leadership from the current leader to a new replica. + - `switchover`: Defines the procedure for a controlled transition of a role to a new replica. This approach aims to minimize downtime and maintain availability in systems with a leader-follower topology, such as before planned maintenance or upgrades on the current leader node. - `memberJoin`: Defines the procedure to add a new replica to the replication group. @@ -7992,8 +7992,8 @@ spec: type: object switchover: description: |- - Defines the procedure for a controlled transition of leadership from the current leader to a new replica. - This approach aims to minimize downtime and maintain availability in systems with a leader-follower topology, + Defines the procedure for a controlled transition of a role to a new replica. + This approach aims to minimize downtime and maintain availability during events such as planned maintenance or when performing stop, shutdown, restart, or upgrade operations involving the current leader node. @@ -8001,8 +8001,11 @@ spec: The container executing this action has access to following variables: - - KB_SWITCHOVER_CANDIDATE_NAME: The name of the pod for the new leader candidate, which may not be specified (empty). - - KB_SWITCHOVER_CANDIDATE_FQDN: The FQDN of the new leader candidate's pod, which may not be specified (empty). + - KB_SWITCHOVER_CANDIDATE_NAME: The name of the pod of the new role's candidate, which may not be specified (empty). + - KB_SWITCHOVER_CANDIDATE_FQDN: The FQDN of the pod of the new role's candidate, which may not be specified (empty). + - KB_SWITCHOVER_CURRENT_NAME: The name of the pod of the current role. + - KB_SWITCHOVER_CURRENT_FQDN: The FQDN of the pod of the current role. + - KB_SWITCHOVER_ROLE: The role that will be transferred to another replica. Note: This field is immutable once it has been set. @@ -8514,17 +8517,6 @@ spec: of 2 learners and 1 follower while maintaining quorum. - This field is immutable once set. - type: boolean - switchoverBeforeUpdate: - default: false - description: |- - SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before - updating or scaling in pods with this role. This is typically used for leader roles to - ensure minimal disruption during updates. - The default value is false. - - This field is immutable once set. type: boolean updatePriority: diff --git a/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml b/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml index 0451d732107..55aa6c4feab 100644 --- a/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml +++ b/deploy/helm/crds/workloads.kubeblocks.io_instancesets.yaml @@ -4213,17 +4213,6 @@ spec: of 2 learners and 1 follower while maintaining quorum. - This field is immutable once set. - type: boolean - switchoverBeforeUpdate: - default: false - description: |- - SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before - updating or scaling in pods with this role. This is typically used for leader roles to - ensure minimal disruption during updates. - The default value is false. - - This field is immutable once set. type: boolean updatePriority: @@ -12428,17 +12417,6 @@ spec: of 2 learners and 1 follower while maintaining quorum. - This field is immutable once set. - type: boolean - switchoverBeforeUpdate: - default: false - description: |- - SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before - updating or scaling in pods with this role. This is typically used for leader roles to - ensure minimal disruption during updates. - The default value is false. - - This field is immutable once set. type: boolean updatePriority: diff --git a/pkg/controller/component/lifecycle/kbagent.go b/pkg/controller/component/lifecycle/kbagent.go index 571177c2aa4..ed7c39d8dd8 100644 --- a/pkg/controller/component/lifecycle/kbagent.go +++ b/pkg/controller/component/lifecycle/kbagent.go @@ -78,12 +78,18 @@ func (a *kbagent) RoleProbe(ctx context.Context, cli client.Reader, opts *Option } func (a *kbagent) Switchover(ctx context.Context, cli client.Reader, opts *Options, candidate string) error { + roleName, ok := a.pod.Labels[constant.RoleLabelKey] + if !ok { + return errors.Errorf("pod %s/%s has no role label", a.pod.Namespace, a.pod.Name) + } lfa := &switchover{ - namespace: a.synthesizedComp.Namespace, - clusterName: a.synthesizedComp.ClusterName, - compName: a.synthesizedComp.Name, - roles: a.synthesizedComp.Roles, - candidate: candidate, + namespace: a.synthesizedComp.Namespace, + clusterName: a.synthesizedComp.ClusterName, + compName: a.synthesizedComp.Name, + roles: a.synthesizedComp.Roles, + role: roleName, + currentPod: a.pod.Name, + candidatePod: candidate, } return a.ignoreOutput(a.checkedCallAction(ctx, cli, a.synthesizedComp.LifecycleActions.Switchover, lfa, opts)) } diff --git a/pkg/controller/component/lifecycle/lfa_member.go b/pkg/controller/component/lifecycle/lfa_member.go index 2c91b14fdfd..11fe377d994 100644 --- a/pkg/controller/component/lifecycle/lfa_member.go +++ b/pkg/controller/component/lifecycle/lfa_member.go @@ -33,6 +33,9 @@ import ( const ( switchoverCandidateName = "KB_SWITCHOVER_CANDIDATE_NAME" switchoverCandidateFQDN = "KB_SWITCHOVER_CANDIDATE_FQDN" + switchoverCurrentName = "KB_SWITCHOVER_CURRENT_NAME" + switchoverCurrentFQDN = "KB_SWITCHOVER_CURRENT_FQDN" + switchoverRole = "KB_SWITCHOVER_ROLE" joinMemberPodFQDNVar = "KB_JOIN_MEMBER_POD_FQDN" joinMemberPodNameVar = "KB_JOIN_MEMBER_POD_NAME" leaveMemberPodFQDNVar = "KB_LEAVE_MEMBER_POD_FQDN" @@ -52,11 +55,13 @@ func (a *roleProbe) parameters(ctx context.Context, cli client.Reader) (map[stri } type switchover struct { - namespace string - clusterName string - compName string - roles []appsv1.ReplicaRole - candidate string + namespace string + clusterName string + compName string + roles []appsv1.ReplicaRole + role string // the role that will be transferred to another replica. + currentPod string + candidatePod string } var _ lifecycleAction = &switchover{} @@ -71,11 +76,14 @@ func (a *switchover) parameters(ctx context.Context, cli client.Reader) (map[str // - KB_SWITCHOVER_CANDIDATE_NAME: The name of the pod for the new leader candidate, which may not be specified (empty). // - KB_SWITCHOVER_CANDIDATE_FQDN: The FQDN of the new leader candidate's pod, which may not be specified (empty). m := make(map[string]string) - if len(a.candidate) > 0 { + if len(a.candidatePod) > 0 { compName := constant.GenerateClusterComponentName(a.clusterName, a.compName) - m[switchoverCandidateName] = a.candidate - m[switchoverCandidateFQDN] = component.PodFQDN(a.namespace, compName, a.candidate) + m[switchoverCandidateName] = a.candidatePod + m[switchoverCandidateFQDN] = component.PodFQDN(a.namespace, compName, a.candidatePod) } + m[switchoverCurrentName] = a.currentPod + m[switchoverCurrentFQDN] = component.PodFQDN(a.namespace, a.compName, a.currentPod) + m[switchoverRole] = a.role return m, nil } diff --git a/pkg/operations/switchover.go b/pkg/operations/switchover.go index b1e3082b977..37a57fb3f0e 100644 --- a/pkg/operations/switchover.go +++ b/pkg/operations/switchover.go @@ -27,8 +27,10 @@ import ( "time" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" @@ -48,6 +50,7 @@ var _ OpsHandler = switchoverOpsHandler{} type SwitchoverMessage struct { opsv1alpha1.Switchover OldPod string + Role string Cluster string } @@ -72,13 +75,18 @@ func (r switchoverOpsHandler) ActionStartedCondition(reqCtx intctrlutil.RequestC if err != nil { return nil, err } - pod, err := getPodToPerformSwitchover(reqCtx.Ctx, cli, synthesizedComp) - if err != nil { - return nil, err + pod := &corev1.Pod{} + if err := cli.Get(reqCtx.Ctx, types.NamespacedName{Namespace: synthesizedComp.Namespace, Name: switchover.InstanceName}, pod); err != nil { + return nil, fmt.Errorf("get pod %v/%v failed, err: %v", synthesizedComp.Namespace, switchover.InstanceName, err.Error()) + } + roleName, ok := pod.Labels[constant.RoleLabelKey] + if !ok || roleName == "" { + return nil, fmt.Errorf("pod %s does not have a invalid role label", switchover.InstanceName) } switchoverMessageMap[switchover.ComponentName] = SwitchoverMessage{ Switchover: switchover, OldPod: pod.Name, + Role: roleName, Cluster: opsRes.Cluster.Name, } } @@ -239,13 +247,14 @@ func handleSwitchover(reqCtx intctrlutil.RequestCtx, cli client.Client, opsRes * return nil } +// We consider a switchover action succeeds if the role label of the pod to perform switchover has changed. func doSwitchover(ctx context.Context, cli client.Reader, synthesizedComp *component.SynthesizedComponent, switchover *opsv1alpha1.Switchover, switchoverCondition *metav1.Condition) error { - consistency, err := checkPodRoleLabelConsistency(ctx, cli, *synthesizedComp, switchover, switchoverCondition) + changed, err := roleLabelChanged(ctx, cli, *synthesizedComp, switchover, switchoverCondition) if err != nil { return err } - if consistency { + if changed { return nil } @@ -259,13 +268,8 @@ func doSwitchover(ctx context.Context, cli client.Reader, synthesizedComp *compo return err } - var candidate string - if switchover.InstanceName == KBSwitchoverCandidateInstanceForAnyPod { - candidate = "" - } else { - candidate = switchover.InstanceName - } - err = lfa.Switchover(ctx, cli, nil, candidate) + // NOTE: switchover is a blocking action currently. May change to non-blocking for better performance. + err = lfa.Switchover(ctx, cli, nil, switchover.CandidateName) if err != nil { return err } else { diff --git a/pkg/operations/switchover_util.go b/pkg/operations/switchover_util.go index fae47a00607..aa95e4ca131 100644 --- a/pkg/operations/switchover_util.go +++ b/pkg/operations/switchover_util.go @@ -28,9 +28,9 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - appsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" opsv1alpha1 "github.com/apecloud/kubeblocks/apis/operations/v1alpha1" "github.com/apecloud/kubeblocks/pkg/constant" "github.com/apecloud/kubeblocks/pkg/controller/component" @@ -49,13 +49,13 @@ func needDoSwitchover(ctx context.Context, cli client.Client, synthesizedComp *component.SynthesizedComponent, switchover *opsv1alpha1.Switchover) (bool, error) { - pod, err := getPodToPerformSwitchover(ctx, cli, synthesizedComp) - if err != nil { - return false, err - } - if pod == nil { - return false, nil - } + // pod, err := getPodToPerformSwitchover(ctx, cli, synthesizedComp) + // if err != nil { + // return false, err + // } + // if pod == nil { + // return false, nil + // } switch switchover.InstanceName { case KBSwitchoverCandidateInstanceForAnyPod: return true, nil @@ -71,87 +71,49 @@ func needDoSwitchover(ctx context.Context, return false, controllerutil.NewFatalError(fmt.Sprintf(`the pod "%s" not belongs to the component "%s"`, switchover.InstanceName, switchover.ComponentName)) } // If the current instance is already the primary, then no switchover will be performed. - if pod.Name == switchover.InstanceName { - return false, nil - } + // if pod.Name == switchover.InstanceName { + // return false, nil + // } } return true, nil } -// checkPodRoleLabelConsistency checks whether the pod role label is consistent with the specified role label after switchover. -func checkPodRoleLabelConsistency(ctx context.Context, +func roleLabelChanged(ctx context.Context, cli client.Reader, synthesizedComp component.SynthesizedComponent, switchover *opsv1alpha1.Switchover, switchoverCondition *metav1.Condition) (bool, error) { - if switchover == nil || switchoverCondition == nil { - return false, nil - } - pod, err := getPodToPerformSwitchover(ctx, cli, &synthesizedComp) - if err != nil { - return false, err - } - if pod == nil { - return false, nil + pod := &corev1.Pod{} + if err := cli.Get(ctx, types.NamespacedName{Namespace: synthesizedComp.Namespace, Name: switchover.InstanceName}, pod); err != nil { + return false, fmt.Errorf("get pod %v/%v failed, err: %v", synthesizedComp.Namespace, switchover.InstanceName, err.Error()) } var switchoverMessageMap map[string]SwitchoverMessage if err := json.Unmarshal([]byte(switchoverCondition.Message), &switchoverMessageMap); err != nil { return false, err } + role, err := getRoleName(pod) + if err != nil { + return false, err + } + for _, switchoverMessage := range switchoverMessageMap { if switchoverMessage.ComponentName != synthesizedComp.Name { continue } - switch switchoverMessage.Switchover.InstanceName { - case KBSwitchoverCandidateInstanceForAnyPod: - if pod.Name != switchoverMessage.OldPod { - return true, nil - } - default: - if pod.Name == switchoverMessage.Switchover.InstanceName { - return true, nil - } + if switchoverMessage.Role != role { + return true, nil + } else { + return false, nil } } - return false, nil -} - -// get the Pod object whose current role needs to be transferred to another pod -func getPodToPerformSwitchover(ctx context.Context, cli client.Reader, synthesizedComp *component.SynthesizedComponent) (*corev1.Pod, error) { - role, err := getTargetRoleName(synthesizedComp.Roles) - if err != nil { - return nil, err - } - pod, err := getPodByRole(ctx, cli, synthesizedComp, role) - return pod, err -} - -func getPodByRole(ctx context.Context, cli client.Reader, synthesizeComp *component.SynthesizedComponent, targetRole string) (*corev1.Pod, error) { - pods, err := component.ListOwnedPodsWithRole(ctx, cli, synthesizeComp.Namespace, synthesizeComp.ClusterName, synthesizeComp.Name, targetRole) - if err != nil { - return nil, err - } - if len(pods) != 1 { - return nil, errors.New("target pod list is empty or has more than one pod") - } - return pods[0], nil + return false, errors.New("invalid switchover message") } -// getTargetRole returns the role on which the switchover is performed -// FIXME: the assumption that only one role supports switchover may change in the future -func getTargetRoleName(roles []appsv1.ReplicaRole) (string, error) { - targetRole := "" - if len(roles) == 0 { - return targetRole, errors.New("component has no roles definition, does not support switchover") - } - for _, role := range roles { - if role.SwitchoverBeforeUpdate { - if targetRole != "" { - return targetRole, errors.New("componentDefinition has more than role that needs switchover before, does not support switchover") - } - targetRole = role.Name - } +func getRoleName(pod *corev1.Pod) (string, error) { + roleName, ok := pod.Labels[constant.RoleLabelKey] + if !ok || roleName == "" { + return "", fmt.Errorf("pod %s/%s does not have a invalid role label", pod.Namespace, pod.Name) } - return targetRole, nil + return roleName, nil } From b492aa81e69dacffcf688d04a0fc517cd45d046b Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Mon, 23 Dec 2024 19:00:51 +0800 Subject: [PATCH 30/34] comments --- pkg/controller/instanceset/update_plan.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/controller/instanceset/update_plan.go b/pkg/controller/instanceset/update_plan.go index a9201da76ed..432cc193add 100644 --- a/pkg/controller/instanceset/update_plan.go +++ b/pkg/controller/instanceset/update_plan.go @@ -140,7 +140,7 @@ func (p *realUpdatePlan) build() { } } -// unknown & empty & learner & 1/2 followers -> 1/2 followers -> leader +// unknown & empty & roles that do not participate in quorum & 1/2 followers -> 1/2 followers -> leader func (p *realUpdatePlan) buildBestEffortParallelUpdatePlan(rolePriorityMap map[string]int) { currentVertex, _ := model.FindRootVertex(p.dag) preVertex := currentVertex @@ -156,7 +156,7 @@ func (p *realUpdatePlan) buildBestEffortParallelUpdatePlan(rolePriorityMap map[s } } - // append unknown, empty and learner + // append unknown, empty and roles that do not participate in quorum index := 0 podList := p.pods for i, pod := range podList { @@ -206,7 +206,7 @@ func (p *realUpdatePlan) buildBestEffortParallelUpdatePlan(rolePriorityMap map[s } } -// unknown & empty & leader & followers & learner +// unknown & empty & all roles func (p *realUpdatePlan) buildParallelUpdatePlan() { root, _ := model.FindRootVertex(p.dag) for i := range p.pods { @@ -215,7 +215,7 @@ func (p *realUpdatePlan) buildParallelUpdatePlan() { } } -// unknown -> empty -> learner -> followers(none->readonly->readwrite) -> leader +// update according to role update priority func (p *realUpdatePlan) buildSerialUpdatePlan() { preVertex, _ := model.FindRootVertex(p.dag) for i := range p.pods { From bc08373397b9231021ef295ca137a103800a858b Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Mon, 23 Dec 2024 19:07:13 +0800 Subject: [PATCH 31/34] go fmt --- pkg/controller/instanceset/pod_role_event_handler_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/controller/instanceset/pod_role_event_handler_test.go b/pkg/controller/instanceset/pod_role_event_handler_test.go index f2b4bfdd53f..acc462e7e0f 100644 --- a/pkg/controller/instanceset/pod_role_event_handler_test.go +++ b/pkg/controller/instanceset/pod_role_event_handler_test.go @@ -55,9 +55,9 @@ var _ = Describe("pod role label event handler test", func() { FieldPath: lorryEventFieldPath, } role := workloads.ReplicaRole{ - Name: "leader", - ParticipatesInQuorum: true, - UpdatePriority: 5, + Name: "leader", + ParticipatesInQuorum: true, + UpdatePriority: 5, } By("build an expected message") From 5adf4506459d28678e7f7a7a5bfed2b5447b47f5 Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Tue, 24 Dec 2024 11:45:06 +0800 Subject: [PATCH 32/34] remove accessMode Revert "use contant's rolelabelkey" --- controllers/operations/opsrequest_controller_test.go | 4 ++-- pkg/controllerutil/pod_utils_test.go | 3 +-- pkg/operations/custom_test.go | 2 +- pkg/operations/horizontal_scaling_test.go | 4 ++-- pkg/operations/ops_progress_util_test.go | 4 ++-- pkg/operations/ops_util_test.go | 2 +- pkg/operations/rebuild_instance_test.go | 4 ++-- pkg/operations/vertical_scaling_test.go | 2 +- pkg/testutil/apps/cluster_instance_set_test_util.go | 5 ++--- 9 files changed, 14 insertions(+), 16 deletions(-) diff --git a/controllers/operations/opsrequest_controller_test.go b/controllers/operations/opsrequest_controller_test.go index 4f4ea7a57c1..f8a58aad1be 100644 --- a/controllers/operations/opsrequest_controller_test.go +++ b/controllers/operations/opsrequest_controller_test.go @@ -147,7 +147,7 @@ var _ = Describe("OpsRequest Controller", func() { By("mock pods are available and wait for cluster enter running phase") podName := fmt.Sprintf("%s-%s-0", clusterObj.Name, mysqlCompName) pod := testapps.MockInstanceSetPod(&testCtx, nil, clusterObj.Name, mysqlCompName, - podName, "leader", "ReadWrite") + podName, "leader") // the opsRequest will use startTime to check some condition. // if there is no sleep for 1 second, unstable error may occur. @@ -194,7 +194,7 @@ var _ = Describe("OpsRequest Controller", func() { testk8s.MockPodIsTerminating(ctx, testCtx, pod) testk8s.RemovePodFinalizer(ctx, testCtx, pod) testapps.MockInstanceSetPod(&testCtx, nil, clusterObj.Name, mysqlCompName, - pod.Name, "leader", "ReadWrite", scalingCtx.target) + pod.Name, "leader", scalingCtx.target) Expect(testapps.ChangeObj(&testCtx, verticalScalingOpsRequest, func(lopsReq *opsv1alpha1.OpsRequest) { if lopsReq.Annotations == nil { lopsReq.Annotations = map[string]string{} diff --git a/pkg/controllerutil/pod_utils_test.go b/pkg/controllerutil/pod_utils_test.go index 4cdf3506c4c..89686f6dfa2 100644 --- a/pkg/controllerutil/pod_utils_test.go +++ b/pkg/controllerutil/pod_utils_test.go @@ -618,9 +618,8 @@ var _ = Describe("pod utils", func() { compName = "component" podName = "pod" role = "leader" - mode = "ReadWrite" ) - pod := testapps.MockInstanceSetPod(&testCtx, nil, clusterName, compName, podName, role, mode) + pod := testapps.MockInstanceSetPod(&testCtx, nil, clusterName, compName, podName, role) ppod := testapps.NewPodFactory(testCtx.DefaultNamespace, "pod"). AddAppInstanceLabel(clusterName). AddAppComponentLabel(compName). diff --git a/pkg/operations/custom_test.go b/pkg/operations/custom_test.go index 16c38e4fa49..808f63f95fc 100644 --- a/pkg/operations/custom_test.go +++ b/pkg/operations/custom_test.go @@ -367,7 +367,7 @@ var _ = Describe("CustomOps", func() { GetObject() // create a pod which belongs to the sharding component - pod := testapps.MockInstanceSetPod(&testCtx, nil, cluster.Name, defaultCompName, fmt.Sprintf(shardingCompName+"-0"), "", "") + pod := testapps.MockInstanceSetPod(&testCtx, nil, cluster.Name, defaultCompName, fmt.Sprintf(shardingCompName+"-0"), "") Expect(testapps.ChangeObj(&testCtx, pod, func(obj *corev1.Pod) { pod.Labels[constant.KBAppShardingNameLabelKey] = defaultCompName })).Should(Succeed()) diff --git a/pkg/operations/horizontal_scaling_test.go b/pkg/operations/horizontal_scaling_test.go index 95fe13d0eb6..66d5409c093 100644 --- a/pkg/operations/horizontal_scaling_test.go +++ b/pkg/operations/horizontal_scaling_test.go @@ -180,7 +180,7 @@ var _ = Describe("HorizontalScaling OpsRequest", func() { } for i := range ordinals { podName := fmt.Sprintf("%s-%s%s-%d", clusterName, defaultCompName, prefix, ordinals[i]) - pod := testapps.MockInstanceSetPod(&testCtx, nil, clusterName, defaultCompName, podName, "follower", "Readonly") + pod := testapps.MockInstanceSetPod(&testCtx, nil, clusterName, defaultCompName, podName, "follower") pods = append(pods, pod) } return pods @@ -388,7 +388,7 @@ var _ = Describe("HorizontalScaling OpsRequest", func() { }, }, []string{}, func(podList []*corev1.Pod) { By("create the specified pod " + offlineInstanceName) - testapps.MockInstanceSetPod(&testCtx, nil, clusterName, defaultCompName, offlineInstanceName, "follower", "Readonly") + testapps.MockInstanceSetPod(&testCtx, nil, clusterName, defaultCompName, offlineInstanceName, "follower") }, false) Expect(opsRes.OpsRequest.Status.Progress).Should(Equal("1/1")) By("expect replicas to 4") diff --git a/pkg/operations/ops_progress_util_test.go b/pkg/operations/ops_progress_util_test.go index 28c0d6c351f..f27f722b6fd 100644 --- a/pkg/operations/ops_progress_util_test.go +++ b/pkg/operations/ops_progress_util_test.go @@ -89,7 +89,7 @@ var _ = Describe("Ops ProgressDetails", func() { By("mock one pod of InstanceSet to update successfully") testk8s.RemovePodFinalizer(ctx, testCtx, pod) testapps.MockInstanceSetPod(&testCtx, nil, clusterName, defaultCompName, - pod.Name, "leader", "ReadWrite") + pod.Name, "leader") _, _ = GetOpsManager().Reconcile(reqCtx, k8sClient, opsRes) Expect(getProgressDetailStatus(opsRes, defaultCompName, pod)).Should(Equal(opsv1alpha1.SucceedProgressStatus)) @@ -204,7 +204,7 @@ var _ = Describe("Ops ProgressDetails", func() { tokens := strings.Split(podList[2].Name, "-") targetPodName := fmt.Sprintf("%s-3", strings.Join(tokens[0:len(tokens)-1], "-")) testapps.MockInstanceSetPod(&testCtx, nil, clusterName, defaultCompName, - targetPodName, "follower", "Readonly") + targetPodName, "follower") targetPod := &corev1.Pod{} Expect(k8sClient.Get(ctx, client.ObjectKey{Name: targetPodName, Namespace: testCtx.DefaultNamespace}, targetPod)).Should(Succeed()) testapps.MockInstanceSetStatus(testCtx, opsRes.Cluster, defaultCompName) diff --git a/pkg/operations/ops_util_test.go b/pkg/operations/ops_util_test.go index 53ce59e9eb0..b0201d474c9 100644 --- a/pkg/operations/ops_util_test.go +++ b/pkg/operations/ops_util_test.go @@ -137,7 +137,7 @@ var _ = Describe("OpsUtil functions", func() { testk8s.MockPodIsTerminating(ctx, testCtx, pods[2]) testk8s.RemovePodFinalizer(ctx, testCtx, pods[2]) // recreate it - pod := testapps.MockInstanceSetPod(&testCtx, nil, clusterName, defaultCompName, pods[2].Name, "follower", "Readonly") + pod := testapps.MockInstanceSetPod(&testCtx, nil, clusterName, defaultCompName, pods[2].Name, "follower") // mock pod is failed testk8s.MockPodIsFailed(ctx, testCtx, pod) opsPhase, _, err = compOpsHelper.reconcileActionWithComponentOps(reqCtx, k8sClient, opsRes, "test", handleRestartProgress) diff --git a/pkg/operations/rebuild_instance_test.go b/pkg/operations/rebuild_instance_test.go index 3a2d5980a38..31739a96633 100644 --- a/pkg/operations/rebuild_instance_test.go +++ b/pkg/operations/rebuild_instance_test.go @@ -483,8 +483,8 @@ var _ = Describe("OpsUtil functions", func() { })).Should(Succeed()) By("mock the new pods to available") - testapps.MockInstanceSetPod(&testCtx, nil, clusterName, defaultCompName, podPrefix+"-3", "follower", "Readonly") - testapps.MockInstanceSetPod(&testCtx, nil, clusterName, defaultCompName, podPrefix+"-4", "follower", "Readonly") + testapps.MockInstanceSetPod(&testCtx, nil, clusterName, defaultCompName, podPrefix+"-3", "follower") + testapps.MockInstanceSetPod(&testCtx, nil, clusterName, defaultCompName, podPrefix+"-4", "follower") By("expect specified instances to take offline") _, _ = GetOpsManager().Reconcile(reqCtx, k8sClient, opsRes) diff --git a/pkg/operations/vertical_scaling_test.go b/pkg/operations/vertical_scaling_test.go index b5d4bbba49d..18139c5cf56 100644 --- a/pkg/operations/vertical_scaling_test.go +++ b/pkg/operations/vertical_scaling_test.go @@ -208,7 +208,7 @@ var _ = Describe("VerticalScaling OpsRequest", func() { testk8s.MockPodIsTerminating(ctx, testCtx, pod) testk8s.RemovePodFinalizer(ctx, testCtx, pod) testapps.MockInstanceSetPod(&testCtx, nil, clusterName, defaultCompName, - pod.Name, "leader", "ReadWrite", ops.Spec.VerticalScalingList[0].ResourceRequirements) + pod.Name, "leader", ops.Spec.VerticalScalingList[0].ResourceRequirements) } By("mock podList[0] rolling update successfully by re-creating it") diff --git a/pkg/testutil/apps/cluster_instance_set_test_util.go b/pkg/testutil/apps/cluster_instance_set_test_util.go index 3e49615fe7c..b508741f411 100644 --- a/pkg/testutil/apps/cluster_instance_set_test_util.go +++ b/pkg/testutil/apps/cluster_instance_set_test_util.go @@ -140,7 +140,7 @@ func MockInstanceSetPods( podRole = noneLeaderRole.Name } } - pod := MockInstanceSetPod(testCtx, its, cluster.Name, compName, pName, podRole, "") + pod := MockInstanceSetPod(testCtx, its, cluster.Name, compName, pName, podRole) annotations := pod.Annotations if annotations == nil { annotations = make(map[string]string) @@ -152,14 +152,13 @@ func MockInstanceSetPods( } // MockInstanceSetPod mocks to create the pod of the InstanceSet, just using in envTest -// TODO: remove accessMode func MockInstanceSetPod( testCtx *testutil.TestContext, its *workloads.InstanceSet, clusterName, consensusCompName, podName, - podRole, accessMode string, + podRole string, resources ...corev1.ResourceRequirements, ) *corev1.Pod { var stsUpdateRevision string From f032589ae64943c19f8ce1e16f58fb0e7dfff054 Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Tue, 24 Dec 2024 15:25:24 +0800 Subject: [PATCH 33/34] fix ut --- pkg/testutil/apps/cluster_instance_set_test_util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/testutil/apps/cluster_instance_set_test_util.go b/pkg/testutil/apps/cluster_instance_set_test_util.go index b508741f411..5382306e8b0 100644 --- a/pkg/testutil/apps/cluster_instance_set_test_util.go +++ b/pkg/testutil/apps/cluster_instance_set_test_util.go @@ -318,7 +318,7 @@ func MockInstanceSetStatus(testCtx testutil.TestContext, cluster *appsv1.Cluster break } } - gomega.Expect(role).ShouldNot(gomega.BeNil()) + // role can be nil memberStatus := workloads.MemberStatus{ PodName: pod.Name, ReplicaRole: role, From 253a2369a35ec559a132ea43808e97eb2f6220fb Mon Sep 17 00:00:00 2001 From: Harold Cheng Date: Thu, 9 Jan 2025 15:29:03 +0800 Subject: [PATCH 34/34] remove basev1 --- apis/apps/v1/componentdefinition_types.go | 54 +++++++- apis/apps/v1/zz_generated.deepcopy.go | 18 ++- apis/base/v1/groupversion_info.go | 39 ------ apis/base/v1/types.go | 72 ---------- apis/base/v1/zz_generated.deepcopy.go | 41 ------ apis/workloads/v1/instanceset_types.go | 19 +-- apis/workloads/v1/zz_generated.deepcopy.go | 5 +- docs/developer_docs/api-reference/cluster.md | 136 +++++++++++++------ 8 files changed, 162 insertions(+), 222 deletions(-) delete mode 100644 apis/base/v1/groupversion_info.go delete mode 100644 apis/base/v1/types.go delete mode 100644 apis/base/v1/zz_generated.deepcopy.go diff --git a/apis/apps/v1/componentdefinition_types.go b/apis/apps/v1/componentdefinition_types.go index 6f458a52ea0..90aabaf6dca 100644 --- a/apis/apps/v1/componentdefinition_types.go +++ b/apis/apps/v1/componentdefinition_types.go @@ -26,8 +26,6 @@ import ( corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - basev1 "github.com/apecloud/kubeblocks/apis/base/v1" ) // +genclient @@ -1428,8 +1426,56 @@ type ComponentAvailableProbeAssertion struct { } // ReplicaRole represents a role that can be assigned to a component instance, defining its behavior and responsibilities. -// +kubebuilder:object:generate=false -type ReplicaRole = basev1.ReplicaRole +type ReplicaRole struct { + // Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label + // on the corresponding object to identify its role. + // + // For example, common role names include: + // - "leader": The primary/master instance that handles write operations + // - "follower": Secondary/replica instances that replicate data from the leader + // - "learner": Read-only instances that don't participate in elections + // + // This field is immutable once set. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MaxLength=32 + // +kubebuilder:validation:Pattern=`^.*[^\s]+.*$` + Name string `json:"name"` + + // UpdatePriority determines the order in which pods with different roles are updated. + // Pods are sorted by this priority (higher numbers = higher priority) and updated accordingly. + // Roles with the highest priority will be updated last. + // The default priority is 0. + // + // For example: + // - Leader role may have priority 2 (updated last) + // - Follower role may have priority 1 (updated before leader) + // - Learner role may have priority 0 (updated first) + // + // This field is immutable once set. + // + // +kubebuilder:default=0 + // +optional + UpdatePriority int `json:"updatePriority"` + + // ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. + // This affects update strategies that need to maintain quorum for availability. Roles participate + // in quorum should have higher update priority than roles do not participate in quorum. + // The default value is false. + // + // For example, in a 5-pod component where: + // - 2 learner pods (participatesInQuorum=false) + // - 2 follower pods (participatesInQuorum=true) + // - 1 leader pod (participatesInQuorum=true) + // The quorum size would be 3 (based on the 3 participating pods), allowing parallel updates + // of 2 learners and 1 follower while maintaining quorum. + // + // This field is immutable once set. + // + // +kubebuilder:default=false + // +optional + ParticipatesInQuorum bool `json:"participatesInQuorum"` +} // UpdateStrategy defines the update strategy for cluster components. This strategy determines how updates are applied // across the cluster. diff --git a/apis/apps/v1/zz_generated.deepcopy.go b/apis/apps/v1/zz_generated.deepcopy.go index 6e5ee5f09e3..4524d020a5e 100644 --- a/apis/apps/v1/zz_generated.deepcopy.go +++ b/apis/apps/v1/zz_generated.deepcopy.go @@ -24,7 +24,6 @@ along with this program. If not, see . package v1 import ( - basev1 "github.com/apecloud/kubeblocks/apis/base/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -1189,7 +1188,7 @@ func (in *ComponentDefinitionSpec) DeepCopyInto(out *ComponentDefinitionSpec) { } if in.Roles != nil { in, out := &in.Roles, &out.Roles - *out = make([]basev1.ReplicaRole, len(*in)) + *out = make([]ReplicaRole, len(*in)) copy(*out, *in) } if in.UpdateStrategy != nil { @@ -2411,6 +2410,21 @@ func (in *Range) DeepCopy() *Range { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReplicaRole) DeepCopyInto(out *ReplicaRole) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicaRole. +func (in *ReplicaRole) DeepCopy() *ReplicaRole { + if in == nil { + return nil + } + out := new(ReplicaRole) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ReplicasLimit) DeepCopyInto(out *ReplicasLimit) { *out = *in diff --git a/apis/base/v1/groupversion_info.go b/apis/base/v1/groupversion_info.go deleted file mode 100644 index 2fd928f9bd9..00000000000 --- a/apis/base/v1/groupversion_info.go +++ /dev/null @@ -1,39 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -// Package v1 contains API Schema definitions for the base v1 API group -// +kubebuilder:object:generate=true -// +groupName=base.kubeblocks.io -package v1 - -import ( - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/scheme" -) - -var ( - // GroupVersion is group version used to register these objects - GroupVersion = schema.GroupVersion{Group: "base.kubeblocks.io", Version: "v1"} - - // SchemeBuilder is used to add go types to the GroupVersionKind scheme - SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} - - // AddToScheme adds the types in this group-version to the given scheme. - AddToScheme = SchemeBuilder.AddToScheme -) diff --git a/apis/base/v1/types.go b/apis/base/v1/types.go deleted file mode 100644 index a1d55a01862..00000000000 --- a/apis/base/v1/types.go +++ /dev/null @@ -1,72 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package v1 - -// ReplicaRole represents a role that can be assigned to a component instance, defining its behavior and responsibilities. -type ReplicaRole struct { - // Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label - // on the corresponding object to identify its role. - // - // For example, common role names include: - // - "leader": The primary/master instance that handles write operations - // - "follower": Secondary/replica instances that replicate data from the leader - // - "learner": Read-only instances that don't participate in elections - // - // This field is immutable once set. - // - // +kubebuilder:validation:Required - // +kubebuilder:validation:MaxLength=32 - // +kubebuilder:validation:Pattern=`^.*[^\s]+.*$` - Name string `json:"name"` - - // UpdatePriority determines the order in which pods with different roles are updated. - // Pods are sorted by this priority (higher numbers = higher priority) and updated accordingly. - // Roles with the highest priority will be updated last. - // The default priority is 0. - // - // For example: - // - Leader role may have priority 2 (updated last) - // - Follower role may have priority 1 (updated before leader) - // - Learner role may have priority 0 (updated first) - // - // This field is immutable once set. - // - // +kubebuilder:default=0 - // +optional - UpdatePriority int `json:"updatePriority"` - - // ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. - // This affects update strategies that need to maintain quorum for availability. Roles participate - // in quorum should have higher update priority than roles do not participate in quorum. - // The default value is false. - // - // For example, in a 5-pod component where: - // - 2 learner pods (participatesInQuorum=false) - // - 2 follower pods (participatesInQuorum=true) - // - 1 leader pod (participatesInQuorum=true) - // The quorum size would be 3 (based on the 3 participating pods), allowing parallel updates - // of 2 learners and 1 follower while maintaining quorum. - // - // This field is immutable once set. - // - // +kubebuilder:default=false - // +optional - ParticipatesInQuorum bool `json:"participatesInQuorum"` -} diff --git a/apis/base/v1/zz_generated.deepcopy.go b/apis/base/v1/zz_generated.deepcopy.go deleted file mode 100644 index dfd135819e5..00000000000 --- a/apis/base/v1/zz_generated.deepcopy.go +++ /dev/null @@ -1,41 +0,0 @@ -//go:build !ignore_autogenerated - -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -// Code generated by controller-gen. DO NOT EDIT. - -package v1 - -import () - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ReplicaRole) DeepCopyInto(out *ReplicaRole) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicaRole. -func (in *ReplicaRole) DeepCopy() *ReplicaRole { - if in == nil { - return nil - } - out := new(ReplicaRole) - in.DeepCopyInto(out) - return out -} diff --git a/apis/workloads/v1/instanceset_types.go b/apis/workloads/v1/instanceset_types.go index 1818cbd3691..1a824f1479d 100644 --- a/apis/workloads/v1/instanceset_types.go +++ b/apis/workloads/v1/instanceset_types.go @@ -25,8 +25,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - basev1 "github.com/apecloud/kubeblocks/apis/base/v1" - kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" ) @@ -329,7 +327,7 @@ const ( // ReplicaRole represents a role that can be assigned to a component instance, defining its behavior and responsibilities. // +kubebuilder:object:generate=false -type ReplicaRole = basev1.ReplicaRole +type ReplicaRole = kbappsv1.ReplicaRole // AccessMode defines SVC access mode enums. // +enum @@ -577,18 +575,5 @@ func (r *InstanceSet) IsInstanceSetReady() bool { return true } membersStatus := r.Status.MembersStatus - if len(membersStatus) != int(*r.Spec.Replicas) { - return false - } - if r.Status.ReadyWithoutPrimary { - return true - } - hasLeader := false - for _, status := range membersStatus { - if status.ReplicaRole != nil && status.ReplicaRole.IsLeader { - hasLeader = true - break - } - } - return hasLeader + return len(membersStatus) == int(*r.Spec.Replicas) } diff --git a/apis/workloads/v1/zz_generated.deepcopy.go b/apis/workloads/v1/zz_generated.deepcopy.go index e8e30a2aa22..d015fa73ef1 100644 --- a/apis/workloads/v1/zz_generated.deepcopy.go +++ b/apis/workloads/v1/zz_generated.deepcopy.go @@ -24,7 +24,6 @@ along with this program. If not, see . package v1 import ( - basev1 "github.com/apecloud/kubeblocks/apis/base/v1" appsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -195,7 +194,7 @@ func (in *InstanceSetSpec) DeepCopyInto(out *InstanceSetSpec) { in.UpdateStrategy.DeepCopyInto(&out.UpdateStrategy) if in.Roles != nil { in, out := &in.Roles, &out.Roles - *out = make([]basev1.ReplicaRole, len(*in)) + *out = make([]appsv1.ReplicaRole, len(*in)) copy(*out, *in) } if in.MembershipReconfiguration != nil { @@ -300,7 +299,7 @@ func (in *MemberStatus) DeepCopyInto(out *MemberStatus) { *out = *in if in.ReplicaRole != nil { in, out := &in.ReplicaRole, &out.ReplicaRole - *out = new(basev1.ReplicaRole) + *out = new(appsv1.ReplicaRole) **out = **in } } diff --git a/docs/developer_docs/api-reference/cluster.md b/docs/developer_docs/api-reference/cluster.md index eeaa06bb8fe..01ac599f590 100644 --- a/docs/developer_docs/api-reference/cluster.md +++ b/docs/developer_docs/api-reference/cluster.md @@ -1377,7 +1377,9 @@ ComponentAvailable roles
      -[]github.com/apecloud/kubeblocks/apis/base/v1.ReplicaRole + +[]ReplicaRole + @@ -5210,7 +5212,9 @@ ComponentAvailable roles
      -[]github.com/apecloud/kubeblocks/apis/base/v1.ReplicaRole + +[]ReplicaRole + @@ -8791,6 +8795,83 @@ int32 +

      ReplicaRole +

      +

      +(Appears on:ComponentDefinitionSpec, InstanceSetSpec, MemberStatus) +

      +
      +

      ReplicaRole represents a role that can be assigned to a component instance, defining its behavior and responsibilities.

      +
      + + + + + + + + + + + + + + + + + + + + + +
      FieldDescription
      +name
      + +string + +      		 +

      Name defines the role’s unique identifier. This value is used to set the “apps.kubeblocks.io/role” label +on the corresponding object to identify its role.

      +

      For example, common role names include: +- “leader”: The primary/master instance that handles write operations +- “follower”: Secondary/replica instances that replicate data from the leader +- “learner”: Read-only instances that don’t participate in elections

      +

      This field is immutable once set.

      +
      +updatePriority
      + +int + +      		 +(Optional) +

      UpdatePriority determines the order in which pods with different roles are updated. +Pods are sorted by this priority (higher numbers = higher priority) and updated accordingly. +Roles with the highest priority will be updated last. +The default priority is 0.

      +

      For example: +- Leader role may have priority 2 (updated last) +- Follower role may have priority 1 (updated before leader) +- Learner role may have priority 0 (updated first)

      +

      This field is immutable once set.

      +
      +participatesInQuorum
      + +bool + +      		 +(Optional) +

      ParticipatesInQuorum indicates if pods with this role are counted when determining quorum. +This affects update strategies that need to maintain quorum for availability. Roles participate +in quorum should have higher update priority than roles do not participate in quorum. +The default value is false.

      +

      For example, in a 5-pod component where: +- 2 learner pods (participatesInQuorum=false) +- 2 follower pods (participatesInQuorum=true) +- 1 leader pod (participatesInQuorum=true) +The quorum size would be 3 (based on the 3 participating pods), allowing parallel updates +of 2 learners and 1 follower while maintaining quorum.

      +

      This field is immutable once set.

      +

      ReplicasLimit

      @@ -28958,7 +29039,9 @@ UpdateStrategy.Type will be set to appsv1.OnDeleteStatefulSetStrategyType if Mem roles
      -[]github.com/apecloud/kubeblocks/apis/base/v1.ReplicaRole + +[]ReplicaRole + @@ -29482,7 +29565,9 @@ UpdateStrategy.Type will be set to appsv1.OnDeleteStatefulSetStrategyType if Mem roles
      -[]github.com/apecloud/kubeblocks/apis/base/v1.ReplicaRole + +[]ReplicaRole + @@ -29893,7 +29978,9 @@ string role
      -github.com/apecloud/kubeblocks/apis/base/v1.ReplicaRole + +ReplicaRole + @@ -30062,45 +30149,6 @@ Any attempt to modify other fields will be rejected.

      -

      Range -

      -

      -(Appears on:Ordinals) -

      -
      -

      Range represents a range with a start and an end value. -It is used to define a continuous segment.

      -
      - - - - - - - - - - - - - - - - - -
      FieldDescription
      -start
      - -int32 - -      		 -
      -end
      - -int32 - -      		 -

      RoleUpdateMechanism (string alias)