Skip to content

Commit

Permalink
feat(controller): fix inconsistency between etcd cluster and stateful…
Browse files Browse the repository at this point in the history
…lset

Signed-off-by: soma00333 <[email protected]>
  • Loading branch information
soma00333 committed Jan 25, 2025
1 parent c2343a0 commit 13a76bf
Showing 1 changed file with 22 additions and 11 deletions.
33 changes: 22 additions & 11 deletions internal/controller/etcdcluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,21 +142,32 @@ func (r *EtcdClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
}
targetReplica := *sts.Spec.Replicas // Start with the current size of the stateful set

// TODO: finish the logic later
// The number of replicas in the StatefulSet doesn't match the number of etcd members in the cluster.
if int(targetReplica) != memberCnt {
// TODO: finish the logic later
// nolint:staticcheck // Temporarily disable staticcheck
logger.Info("The expected number of replicas doesn't match the number of etcd members in the cluster", "targetReplica", targetReplica, "memberCnt", memberCnt)
if int(targetReplica) < memberCnt {
// a new added learner hasn't started yet

// re-generate configuration for the new learner member;
// increase statefulsets's replica by 1
// A new member has been added to the etcd cluster
// but the corresponding Pod hasn't been created yet in the StatefulSet.
// Increase the StatefulSet replicas by 1 to match the new cluster member.
newReplicaCount := targetReplica + 1
logger.Info("Increasing StatefulSet replicas to match the new etcd learner.", "oldReplicaCount", targetReplica, "newReplicaCount", newReplicaCount)
_, err = reconcileStatefulSet(ctx, logger, etcdCluster, r.Client, newReplicaCount, r.Scheme)
if err != nil {
return ctrl.Result{}, err
}
} else {
// an already removed member hasn't stopped yet.

// Decrease the statefulsets's replica by 1
// A member has been removed from the etcd cluster
// but the corresponding Pod is still running.
// Decrease the StatefulSet replicas by 1 to remove the unneeded Pod.
logger.Info("An etcd member was removed from the cluster, but the StatefulSet hasn't scaled down yet.")
newReplicaCount := targetReplica - 1
logger.Info("Decreasing StatefulSet replicas to remove the unneeded Pod.", "oldReplicaCount", targetReplica, "newReplicaCount", newReplicaCount)
_, err = reconcileStatefulSet(ctx, logger, etcdCluster, r.Client, newReplicaCount, r.Scheme)
if err != nil {
return ctrl.Result{}, err
}
}
// return
return ctrl.Result{RequeueAfter: requeueDuration}, nil
}

var (
Expand Down

0 comments on commit 13a76bf

Please sign in to comment.