chore: support provisioning clusters by sharding definitions (#8276)

apis/apps/v1/cluster_types.go

@@ -146,7 +146,7 @@ type ClusterSpec struct {
 	// Specifies a list of ClusterComponentSpec objects used to define the individual Components that make up a Cluster.
 	// This field allows for detailed configuration of each Component within the Cluster.
 	//
-	// Note: `shardingSpecs` and `componentSpecs` cannot both be empty; at least one must be defined to configure a Cluster.
+	// Note: `shardings` and `componentSpecs` cannot both be empty; at least one must be defined to configure a Cluster.
 	//
 	// +kubebuilder:validation:MinItems=1
 	// +kubebuilder:validation:MaxItems=128
@@ -155,14 +155,14 @@ type ClusterSpec struct {
 	// +optional
 	ComponentSpecs []ClusterComponentSpec `json:"componentSpecs,omitempty" patchStrategy:"merge,retainKeys" patchMergeKey:"name"`
 
-	// Specifies a list of ShardingSpec objects that manage the sharding topology for Cluster Components.
-	// Each ShardingSpec organizes components into shards, with each shard corresponding to a Component.
+	// Specifies a list of ClusterSharding objects that manage the sharding topology for Cluster Components.
+	// Each ClusterSharding organizes components into shards, with each shard corresponding to a Component.
 	// Components within a shard are all based on a common ClusterComponentSpec template, ensuring uniform configurations.
 	//
 	// This field supports dynamic resharding by facilitating the addition or removal of shards
-	// through the `shards` field in ShardingSpec.
+	// through the `shards` field in ClusterSharding.
 	//
-	// Note: `shardingSpecs` and `componentSpecs` cannot both be empty; at least one must be defined to configure a Cluster.
+	// Note: `shardings` and `componentSpecs` cannot both be empty; at least one must be defined to configure a Cluster.
 	//
 	// +patchMergeKey=name
 	// +patchStrategy=merge,retainKeys
@@ -171,7 +171,7 @@ type ClusterSpec struct {
 	// +kubebuilder:validation:MinItems=1
 	// +kubebuilder:validation:MaxItems=128
 	// +optional
-	ShardingSpecs []ShardingSpec `json:"shardingSpecs,omitempty"`
+	Shardings []ClusterSharding `json:"shardings,omitempty"`
 
 	// Specifies runtimeClassName for all Pods managed by this Cluster.
 	//
@@ -184,7 +184,7 @@ type ClusterSpec struct {
 	SchedulingPolicy *SchedulingPolicy `json:"schedulingPolicy,omitempty"`
 
 	// Defines a list of additional Services that are exposed by a Cluster.
-	// This field allows Services of selected Components, either from `componentSpecs` or `shardingSpecs` to be exposed,
+	// This field allows Services of selected Components, either from `componentSpecs` or `shardings` to be exposed,
 	// alongside Services defined with ComponentService.
 	//
 	// Services defined here can be referenced by other clusters using the ServiceRefClusterSelector.
@@ -222,10 +222,10 @@ type ClusterStatus struct {
 	// +optional
 	Components map[string]ClusterComponentStatus `json:"components,omitempty"`
 
-	// Represents the generation number of the referenced ClusterDefinition.
+	// Records the current status information of all shardings within the Cluster.
 	//
 	// +optional
-	ClusterDefGeneration int64 `json:"clusterDefGeneration,omitempty"`
+	Shardings map[string]ClusterComponentStatus `json:"shardings,omitempty"`
 
 	// Represents a list of detailed status of the Cluster object.
 	// Each condition in the list provides real-time information about certain aspect of the Cluster object.
@@ -259,7 +259,7 @@ const (
 type ClusterComponentSpec struct {
 	// Specifies the Component's name.
 	// It's part of the Service DNS name and must comply with the IANA service naming rule.
-	// The name is optional when ClusterComponentSpec is used as a template (e.g., in `shardingSpec`),
+	// The name is optional when ClusterComponentSpec is used as a template (e.g., in `clusterSharding`),
 	// but required otherwise.
 	//
 	// +kubebuilder:validation:MaxLength=22
@@ -542,33 +542,43 @@ type ClusterComponentService struct {
 	PodService *bool `json:"podService,omitempty"`
 }
 
-// ShardingSpec defines how KubeBlocks manage dynamic provisioned shards.
+// ClusterSharding defines how KubeBlocks manage dynamic provisioned shards.
 // A typical design pattern for distributed databases is to distribute data across multiple shards,
 // with each shard consisting of multiple replicas.
 // Therefore, KubeBlocks supports representing a shard with a Component and dynamically instantiating Components
 // using a template when shards are added.
 // When shards are removed, the corresponding Components are also deleted.
-type ShardingSpec struct {
+type ClusterSharding struct {
 	// Represents the common parent part of all shard names.
+	//
 	// This identifier is included as part of the Service DNS name and must comply with IANA service naming rules.
-	// It is used to generate the names of underlying Components following the pattern `$(shardingSpec.name)-$(ShardID)`.
+	// It is used to generate the names of underlying Components following the pattern `$(clusterSharding.name)-$(ShardID)`.
 	// ShardID is a random string that is appended to the Name to generate unique identifiers for each shard.
 	// For example, if the sharding specification name is "my-shard" and the ShardID is "abc", the resulting Component name
 	// would be "my-shard-abc".
 	//
-	// Note that the name defined in Component template(`shardingSpec.template.name`) will be disregarded
-	// when generating the Component names of the shards. The `shardingSpec.name` field takes precedence.
+	// Note that the name defined in Component template(`clusterSharding.template.name`) will be disregarded
+	// when generating the Component names of the shards. The `clusterSharding.name` field takes precedence.
 	//
 	// +kubebuilder:validation:Required
 	// +kubebuilder:validation:MaxLength=15
 	// +kubebuilder:validation:Pattern:=`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$`
 	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="name is immutable"
 	Name string `json:"name"`
 
+	// Specifies the ShardingDefinition custom resource (CR) that defines the sharding's characteristics and behavior.
+	//
+	// The full name or regular expression is supported to match the ShardingDefinition.
+	//
+	// +kubebuilder:validation:MaxLength=64
+	// +optional
+	ShardingDef string `json:"shardingDef,omitempty"`
+
 	// The template for generating Components for shards, where each shard consists of one Component.
+	//
 	// This field is of type ClusterComponentSpec, which encapsulates all the required details and
 	// definitions for creating and managing the Components.
-	// KubeBlocks uses this template to generate a set of identical Components or shards.
+	// KubeBlocks uses this template to generate a set of identical Components of shards.
 	// All the generated Components will have the same specifications and definitions as specified in the `template` field.
 	//
 	// This allows for the creation of multiple Components with consistent configurations,
@@ -578,20 +588,21 @@ type ShardingSpec struct {
 	Template ClusterComponentSpec `json:"template"`
 
 	// Specifies the desired number of shards.
+	//
 	// Users can declare the desired number of shards through this field.
 	// KubeBlocks dynamically creates and deletes Components based on the difference
 	// between the desired and actual number of shards.
 	// KubeBlocks provides lifecycle management for sharding, including:
 	//
-	// - Executing the postProvision Action defined in the ComponentDefinition when the number of shards increases.
+	// - Executing the shardProvision Action defined in the ShardingDefinition when the number of shards increases.
 	//   This allows for custom actions to be performed after a new shard is provisioned.
-	// - Executing the preTerminate Action defined in the ComponentDefinition when the number of shards decreases.
+	// - Executing the shardTerminate Action defined in the ShardingDefinition when the number of shards decreases.
 	//   This enables custom cleanup or data migration tasks to be executed before a shard is terminated.
 	//   Resources and data associated with the corresponding Component will also be deleted.
 	//
-	// +kubebuilder:validation:Required
 	// +kubebuilder:validation:Minimum=0
 	// +kubebuilder:validation:Maximum=2048
+	// +kubebuilder:validation:Required
 	Shards int32 `json:"shards,omitempty"`
 }
 

apis/apps/v1/clusterdefinition_types.go

@@ -37,12 +37,13 @@ import (
 // ClusterDefinition defines the topology for databases or storage systems,
 // offering a variety of topological configurations to meet diverse deployment needs and scenarios.
 //
-// It includes a list of Components, each linked to a ComponentDefinition, which enhances reusability and reduce redundancy.
+// It includes a list of Components and/or Shardings, each linked to a ComponentDefinition or a ShardingDefinition,
+// which enhances reusability and reduce redundancy.
 // For example, widely used components such as etcd and Zookeeper can be defined once and reused across multiple ClusterDefinitions,
 // simplifying the setup of new systems.
 //
-// Additionally, ClusterDefinition also specifies the sequence of startup, upgrade, and shutdown for Components,
-// ensuring a controlled and predictable management of component lifecycles.
+// Additionally, ClusterDefinition also specifies the sequence of startup, upgrade, and shutdown between Components and/or Shardings,
+// ensuring a controlled and predictable management of cluster lifecycles.
 type ClusterDefinition struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
@@ -107,10 +108,15 @@ type ClusterTopology struct {
 
 	// Components specifies the components in the topology.
 	//
-	// +kubebuilder:validation:Required
-	// +kubebuilder:validation:MinItems=1
 	// +kubebuilder:validation:MaxItems=128
-	Components []ClusterTopologyComponent `json:"components"`
+	// +optional
+	Components []ClusterTopologyComponent `json:"components,omitempty"`
+
+	// Shardings specifies the shardings in the topology.
+	//
+	// +kubebuilder:validation:MaxItems=128
+	// +optional
+	Shardings []ClusterTopologySharding `json:"shardings,omitempty"`
 
 	// Specifies the sequence in which components within a cluster topology are
 	// started, stopped, and upgraded.
@@ -157,34 +163,64 @@ type ClusterTopologyComponent struct {
 	CompDef string `json:"compDef"`
 }
 
+// ClusterTopologySharding defines a sharding within a ClusterTopology.
+type ClusterTopologySharding struct {
+	// Defines the unique identifier of the sharding within the cluster topology.
+	// It follows IANA Service naming rules and is used as part of the Service's DNS name.
+	// The name must start with a lowercase letter, can contain lowercase letters, numbers,
+	// and hyphens, and must end with a lowercase letter or number.
+	//
+	// Cannot be updated once set.
+	//
+	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:MaxLength=16
+	// +kubebuilder:validation:Pattern:=`^[a-z]([a-z0-9\-]*[a-z0-9])?$`
+	Name string `json:"name"`
+
+	// Specifies the sharding definition that defines the characteristics and behavior of the sharding.
+	//
+	// The system selects the ShardingDefinition CR with the latest version that matches the pattern.
+	// This approach allows:
+	//
+	// 1. Precise selection by providing the exact name of a ShardingDefinition CR.
+	// 2. Flexible and automatic selection of the most up-to-date ShardingDefinition CR
+	// by specifying a regular expression pattern.
+	//
+	// Once set, this field cannot be updated.
+	//
+	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:MaxLength=64
+	ShardingDef string `json:"shardingDef"`
+}
+
 // ClusterTopologyOrders manages the lifecycle of components within a cluster by defining their provisioning,
 // terminating, and updating sequences.
 // It organizes components into stages or groups, where each group indicates a set of components
 // that can be managed concurrently.
 // These groups are processed sequentially, allowing precise control based on component dependencies and requirements.
 type ClusterTopologyOrders struct {
-	// Specifies the order for creating and initializing components.
-	// This is designed for components that depend on one another. Components without dependencies can be grouped together.
+	// Specifies the order for creating and initializing entities.
+	// This is designed for entities that depend on one another. Entities without dependencies can be grouped together.
 	//
-	// Components that can be provisioned independently or have no dependencies can be listed together in the same stage,
+	// Entities that can be provisioned independently or have no dependencies can be listed together in the same stage,
 	// separated by commas.
 	//
 	// +optional
 	Provision []string `json:"provision,omitempty"`
 
-	// Outlines the order for stopping and deleting components.
-	// This sequence is designed for components that require a graceful shutdown or have interdependencies.
+	// Outlines the order for stopping and deleting entities.
+	// This sequence is designed for entities that require a graceful shutdown or have interdependencies.
 	//
-	// Components that can be terminated independently or have no dependencies can be listed together in the same stage,
+	// Entities that can be terminated independently or have no dependencies can be listed together in the same stage,
 	// separated by commas.
 	//
 	// +optional
 	Terminate []string `json:"terminate,omitempty"`
 
-	// Update determines the order for updating components' specifications, such as image upgrades or resource scaling.
-	// This sequence is designed for components that have dependencies or require specific update procedures.
+	// Update determines the order for updating entities' specifications, such as image upgrades or resource scaling.
+	// This sequence is designed for entities that have dependencies or require specific update procedures.
 	//
-	// Components that can be updated independently or have no dependencies can be listed together in the same stage,
+	// Entities that can be updated independently or have no dependencies can be listed together in the same stage,
 	// separated by commas.
 	//
 	// +optional

apis/apps/v1/deprecated.go

@@ -117,8 +117,8 @@ func (r *ClusterSpec) GetComponentByName(componentName string) *ClusterComponent
 	return nil
 }
 
-func (r *ClusterSpec) GetShardingByName(shardingName string) *ShardingSpec {
-	for _, v := range r.ShardingSpecs {
+func (r *ClusterSpec) GetShardingByName(shardingName string) *ClusterSharding {
+	for _, v := range r.Shardings {
 		if v.Name == shardingName {
 			return &v
 		}

apis/apps/v1/types.go

@@ -80,8 +80,7 @@ const (
 const (
 	ConditionTypeProvisioningStarted = "ProvisioningStarted" // ConditionTypeProvisioningStarted the operator starts resource provisioning to create or change the cluster
 	ConditionTypeApplyResources      = "ApplyResources"      // ConditionTypeApplyResources the operator start to apply resources to create or change the cluster
-	ConditionTypeReplicasReady       = "ReplicasReady"       // ConditionTypeReplicasReady all pods of components are ready
-	ConditionTypeReady               = "Ready"               // ConditionTypeReady all components are running
+	ConditionTypeReady               = "Ready"               // ConditionTypeReady all components and shardings are running
 )
 
 type ServiceRef struct {