Skip to content

Commit

Permalink
feat: lorry supports more actions (#6720)
Browse files Browse the repository at this point in the history
  • Loading branch information
@xuriwuyun
xuriwuyun authored Mar 13, 2024
1 parent bf5b01b commit fcbf339
Show file tree
Hide file tree
Showing 40 changed files with 1,286 additions and 161 deletions.
34 changes: 31 additions & 3 deletions apis/apps/v1alpha1/componentdefinition_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -686,19 +686,34 @@ type ComponentLifecycleActions struct {

// Defines the method to add a new replica to the replication group.
// This action is typically invoked when a new replica needs to be added, such as during scale-out.
// It may involve updating configuration, notifying other members, and ensuring data consistency.
// The function does not specify or constrain the role of the new member. The role assignment
// is handled by the scripts implemented in the action commands. This provides flexibility
// as the new member can be automatically scaled and assigned a role based on the cluster's needs.
//
// The following dedicated environment variables are available for the action:
//
// - KB_SERVICE_PORT: The port on which the DB service listens.
// - KB_SERVICE_USER: The username used to access the DB service with sufficient privileges.
// - KB_SERVICE_PASSWORD: The password of the user used to access the DB service .
// - KB_PRIMARY_POD_FQDN: The FQDN of the original primary Pod before switchover.
// - KB_PRIMARY_POD_FQDN: The FQDN of the original primary Pod.
// - KB_MEMBER_ADDRESSES: The addresses of all members.
// - KB_NEW_MEMBER_POD_NAME: The name of the new member's Pod.
// - KB_NEW_MEMBER_POD_IP: The name of the new member's Pod.
//
// Output of the action:
// - ERROR: Any error message if the action fails.
//
// For example, the following command can be used to add a new OBServer to the OceanBase Cluster in zone1:
// command:
// - bash
// - -c
// - |
// ADDRESS=$(KB_MEMBER_ADDRESSES%%,*)
// HOST=$(echo $ADDRESS | cut -d ':' -f 1)
// PORT=$(echo $ADDRESS | cut -d ':' -f 2)
// CLIENT="mysql -u $KB_SERVICE_USER -p$KB_SERVICE_PASSWORD -P $PORT -h $HOST -e"
// $CLIENT "ALTER SYSTEM ADD SERVER '$KB_NEW_MEMBER_POD_IP:$KB_SERVICE_PORT' ZONE 'zone1'"
//
// This field cannot be updated.
//
// +optional
Expand All @@ -714,12 +729,25 @@ type ComponentLifecycleActions struct {
// - KB_SERVICE_PORT: The port on which the DB service listens.
// - KB_SERVICE_USER: The username used to access the DB service with sufficient privileges.
// - KB_SERVICE_PASSWORD: The password of the user used to access the DB service.
// - KB_PRIMARY_POD_FQDN: The FQDN of the original primary Pod before switchover.
// - KB_PRIMARY_POD_FQDN: The FQDN of the original primary Pod.
// - KB_MEMBER_ADDRESSES: The addresses of all members.
// - KB_LEAVE_MEMBER_POD_NAME: The name of the leave member's Pod.
// - KB_LEAVE_MEMBER_POD_IP: The IP of the leave member's Pod.
//
// Output of the action:
// - ERROR: Any error message if the action fails.
//
// For example, the following command can be used to delete a OBServer from the OceanBase Cluster in zone1:
// command:
// - bash
// - -c
// - |
// ADDRESS=$(KB_MEMBER_ADDRESSES%%,*)
// HOST=$(echo $ADDRESS | cut -d ':' -f 1)
// PORT=$(echo $ADDRESS | cut -d ':' -f 2)
// CLIENT="mysql -u $KB_SERVICE_USER -p$KB_SERVICE_PASSWORD -P $PORT -h $HOST -e"
// $CLIENT "ALTER SYSTEM DELETE SERVER '$KB_NEW_MEMBER_POD_IP:$KB_SERVICE_PORT' ZONE 'zone1'"
//
// This field cannot be updated.
//
// +optional
Expand Down
40 changes: 29 additions & 11 deletions config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1115,17 +1115,27 @@ spec:
memberJoin:
description: "Defines the method to add a new replica to the replication
group. This action is typically invoked when a new replica needs
to be added, such as during scale-out. It may involve updating
configuration, notifying other members, and ensuring data consistency.
\n The following dedicated environment variables are available
to be added, such as during scale-out. The function does not
specify or constrain the role of the new member. The role assignment
is handled by the scripts implemented in the action commands.
This provides flexibility as the new member can be automatically
scaled and assigned a role based on the cluster's needs. \n
The following dedicated environment variables are available
for the action: \n - KB_SERVICE_PORT: The port on which the
DB service listens. - KB_SERVICE_USER: The username used to
access the DB service with sufficient privileges. - KB_SERVICE_PASSWORD:
The password of the user used to access the DB service . - KB_PRIMARY_POD_FQDN:
The FQDN of the original primary Pod before switchover. - KB_NEW_MEMBER_POD_NAME:
The name of the new member's Pod. \n Output of the action: -
ERROR: Any error message if the action fails. \n This field
cannot be updated."
The FQDN of the original primary Pod. - KB_MEMBER_ADDRESSES:
The addresses of all members. - KB_NEW_MEMBER_POD_NAME: The
name of the new member's Pod. - KB_NEW_MEMBER_POD_IP: The name
of the new member's Pod. \n Output of the action: - ERROR: Any
error message if the action fails. \n For example, the following
command can be used to add a new OBServer to the OceanBase Cluster
in zone1: command: - bash - -c - | ADDRESS=$(KB_MEMBER_ADDRESSES%%,*)
HOST=$(echo $ADDRESS | cut -d ':' -f 1) PORT=$(echo $ADDRESS
| cut -d ':' -f 2) CLIENT=\"mysql -u $KB_SERVICE_USER -p$KB_SERVICE_PASSWORD
-P $PORT -h $HOST -e\" $CLIENT \"ALTER SYSTEM ADD SERVER '$KB_NEW_MEMBER_POD_IP:$KB_SERVICE_PORT'
ZONE 'zone1'\" \n This field cannot be updated."
properties:
builtinHandler:
description: BuiltinHandler specifies the builtin action handler
Expand Down Expand Up @@ -1425,10 +1435,18 @@ spec:
used to access the DB service with sufficient privileges. -
KB_SERVICE_PASSWORD: The password of the user used to access
the DB service. - KB_PRIMARY_POD_FQDN: The FQDN of the original
primary Pod before switchover. - KB_LEAVE_MEMBER_POD_NAME: The
name of the leave member's Pod. \n Output of the action: - ERROR:
Any error message if the action fails. \n This field cannot
be updated."
primary Pod. - KB_MEMBER_ADDRESSES: The addresses of all members.
- KB_LEAVE_MEMBER_POD_NAME: The name of the leave member's Pod.
- KB_LEAVE_MEMBER_POD_IP: The IP of the leave member's Pod.
\n Output of the action: - ERROR: Any error message if the action
fails. \n For example, the following command can be used to
delete a OBServer from the OceanBase Cluster in zone1: command:
- bash - -c - | ADDRESS=$(KB_MEMBER_ADDRESSES%%,*) HOST=$(echo
$ADDRESS | cut -d ':' -f 1) PORT=$(echo $ADDRESS | cut -d ':'
-f 2) CLIENT=\"mysql -u $KB_SERVICE_USER -p$KB_SERVICE_PASSWORD
-P $PORT -h $HOST -e\" $CLIENT \"ALTER SYSTEM DELETE SERVER
'$KB_NEW_MEMBER_POD_IP:$KB_SERVICE_PORT' ZONE 'zone1'\" \n This
field cannot be updated."
properties:
builtinHandler:
description: BuiltinHandler specifies the builtin action handler
Expand Down
40 changes: 29 additions & 11 deletions deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1115,17 +1115,27 @@ spec:
memberJoin:
description: "Defines the method to add a new replica to the replication
group. This action is typically invoked when a new replica needs
to be added, such as during scale-out. It may involve updating
configuration, notifying other members, and ensuring data consistency.
\n The following dedicated environment variables are available
to be added, such as during scale-out. The function does not
specify or constrain the role of the new member. The role assignment
is handled by the scripts implemented in the action commands.
This provides flexibility as the new member can be automatically
scaled and assigned a role based on the cluster's needs. \n
The following dedicated environment variables are available
for the action: \n - KB_SERVICE_PORT: The port on which the
DB service listens. - KB_SERVICE_USER: The username used to
access the DB service with sufficient privileges. - KB_SERVICE_PASSWORD:
The password of the user used to access the DB service . - KB_PRIMARY_POD_FQDN:
The FQDN of the original primary Pod before switchover. - KB_NEW_MEMBER_POD_NAME:
The name of the new member's Pod. \n Output of the action: -
ERROR: Any error message if the action fails. \n This field
cannot be updated."
The FQDN of the original primary Pod. - KB_MEMBER_ADDRESSES:
The addresses of all members. - KB_NEW_MEMBER_POD_NAME: The
name of the new member's Pod. - KB_NEW_MEMBER_POD_IP: The name
of the new member's Pod. \n Output of the action: - ERROR: Any
error message if the action fails. \n For example, the following
command can be used to add a new OBServer to the OceanBase Cluster
in zone1: command: - bash - -c - | ADDRESS=$(KB_MEMBER_ADDRESSES%%,*)
HOST=$(echo $ADDRESS | cut -d ':' -f 1) PORT=$(echo $ADDRESS
| cut -d ':' -f 2) CLIENT=\"mysql -u $KB_SERVICE_USER -p$KB_SERVICE_PASSWORD
-P $PORT -h $HOST -e\" $CLIENT \"ALTER SYSTEM ADD SERVER '$KB_NEW_MEMBER_POD_IP:$KB_SERVICE_PORT'
ZONE 'zone1'\" \n This field cannot be updated."
properties:
builtinHandler:
description: BuiltinHandler specifies the builtin action handler
Expand Down Expand Up @@ -1425,10 +1435,18 @@ spec:
used to access the DB service with sufficient privileges. -
KB_SERVICE_PASSWORD: The password of the user used to access
the DB service. - KB_PRIMARY_POD_FQDN: The FQDN of the original
primary Pod before switchover. - KB_LEAVE_MEMBER_POD_NAME: The
name of the leave member's Pod. \n Output of the action: - ERROR:
Any error message if the action fails. \n This field cannot
be updated."
primary Pod. - KB_MEMBER_ADDRESSES: The addresses of all members.
- KB_LEAVE_MEMBER_POD_NAME: The name of the leave member's Pod.
- KB_LEAVE_MEMBER_POD_IP: The IP of the leave member's Pod.
\n Output of the action: - ERROR: Any error message if the action
fails. \n For example, the following command can be used to
delete a OBServer from the OceanBase Cluster in zone1: command:
- bash - -c - | ADDRESS=$(KB_MEMBER_ADDRESSES%%,*) HOST=$(echo
$ADDRESS | cut -d ':' -f 1) PORT=$(echo $ADDRESS | cut -d ':'
-f 2) CLIENT=\"mysql -u $KB_SERVICE_USER -p$KB_SERVICE_PASSWORD
-P $PORT -h $HOST -e\" $CLIENT \"ALTER SYSTEM DELETE SERVER
'$KB_NEW_MEMBER_POD_IP:$KB_SERVICE_PORT' ZONE 'zone1'\" \n This
field cannot be updated."
properties:
builtinHandler:
description: BuiltinHandler specifies the builtin action handler
Expand Down
32 changes: 29 additions & 3 deletions docs/developer_docs/api-reference/cluster.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7255,17 +7255,31 @@ LifecycleActionHandler
<em>(Optional)</em>
<p>Defines the method to add a new replica to the replication group.
This action is typically invoked when a new replica needs to be added, such as during scale-out.
It may involve updating configuration, notifying other members, and ensuring data consistency.</p>
The function does not specify or constrain the role of the new member. The role assignment
is handled by the scripts implemented in the action commands. This provides flexibility
as the new member can be automatically scaled and assigned a role based on the cluster&rsquo;s needs.</p>
<p>The following dedicated environment variables are available for the action:</p>
<ul>
<li>KB_SERVICE_PORT: The port on which the DB service listens.</li>
<li>KB_SERVICE_USER: The username used to access the DB service with sufficient privileges.</li>
<li>KB_SERVICE_PASSWORD: The password of the user used to access the DB service .</li>
<li>KB_PRIMARY_POD_FQDN: The FQDN of the original primary Pod before switchover.</li>
<li>KB_PRIMARY_POD_FQDN: The FQDN of the original primary Pod.</li>
<li>KB_MEMBER_ADDRESSES: The addresses of all members.</li>
<li>KB_NEW_MEMBER_POD_NAME: The name of the new member&rsquo;s Pod.</li>
<li>KB_NEW_MEMBER_POD_IP: The name of the new member&rsquo;s Pod.</li>
</ul>
<p>Output of the action:
- ERROR: Any error message if the action fails.</p>
<p>For example, the following command can be used to add a new OBServer to the OceanBase Cluster in zone1:
command:
- bash
- -c
- |
ADDRESS=$(KB_MEMBER_ADDRESSES%%,*)
HOST=$(echo $ADDRESS | cut -d &lsquo;:&rsquo; -f 1)
PORT=$(echo $ADDRESS | cut -d &lsquo;:&rsquo; -f 2)
CLIENT=&ldquo;mysql -u $KB_SERVICE_USER -p$KB_SERVICE_PASSWORD -P $PORT -h $HOST -e&rdquo;
$CLIENT &ldquo;ALTER SYSTEM ADD SERVER &lsquo;$KB_NEW_MEMBER_POD_IP:$KB_SERVICE_PORT&rsquo; ZONE &lsquo;zone1&rsquo;&rdquo;</p>
<p>This field cannot be updated.</p>
</td>
</tr>
Expand All @@ -7289,11 +7303,23 @@ but it is advisable to avoid performing data migration within this action.</p>
<li>KB_SERVICE_PORT: The port on which the DB service listens.</li>
<li>KB_SERVICE_USER: The username used to access the DB service with sufficient privileges.</li>
<li>KB_SERVICE_PASSWORD: The password of the user used to access the DB service.</li>
<li>KB_PRIMARY_POD_FQDN: The FQDN of the original primary Pod before switchover.</li>
<li>KB_PRIMARY_POD_FQDN: The FQDN of the original primary Pod.</li>
<li>KB_MEMBER_ADDRESSES: The addresses of all members.</li>
<li>KB_LEAVE_MEMBER_POD_NAME: The name of the leave member&rsquo;s Pod.</li>
<li>KB_LEAVE_MEMBER_POD_IP: The IP of the leave member&rsquo;s Pod.</li>
</ul>
<p>Output of the action:
- ERROR: Any error message if the action fails.</p>
<p>For example, the following command can be used to delete a OBServer from the OceanBase Cluster in zone1:
command:
- bash
- -c
- |
ADDRESS=$(KB_MEMBER_ADDRESSES%%,*)
HOST=$(echo $ADDRESS | cut -d &lsquo;:&rsquo; -f 1)
PORT=$(echo $ADDRESS | cut -d &lsquo;:&rsquo; -f 2)
CLIENT=&ldquo;mysql -u $KB_SERVICE_USER -p$KB_SERVICE_PASSWORD -P $PORT -h $HOST -e&rdquo;
$CLIENT &ldquo;ALTER SYSTEM DELETE SERVER &lsquo;$KB_NEW_MEMBER_POD_IP:$KB_SERVICE_PORT&rsquo; ZONE &lsquo;zone1&rsquo;&rdquo;</p>
<p>This field cannot be updated.</p>
</td>
</tr>
Expand Down
14 changes: 14 additions & 0 deletions pkg/constant/lorry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,17 @@ const (
ETCDCharacterType = "etcd"
PolarDBXCharacterType = "polardbx"
)

// action keys
const (
RoleProbeAction = "roleProbe"
MemberJoinAction = "memberJoin"
MemberLeaveAction = "memberLeave"
ReadonlyAction = "readonly"
ReadWriteAction = "readwrite"
PostProvisionAction = "postProvision"
PreTerminateAction = "preTerminate"
)

// action envs
const ()
16 changes: 8 additions & 8 deletions pkg/controller/component/lorry_utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ func buildLorryContainers(reqCtx intctrlutil.RequestCtx, synthesizeComp *Synthes
compRoleProbe = synthesizeComp.LifecycleActions.RoleProbe
}
if compRoleProbe != nil {
reqCtx.Log.V(3).Info("lorry", "settings", compRoleProbe)
reqCtx.Log.V(3).Info("lorry", "role probe settings", compRoleProbe)
roleChangedContainer := container.DeepCopy()
buildRoleProbeContainer(roleChangedContainer, compRoleProbe, int(lorryHTTPPort))
lorryContainers = append(lorryContainers, *roleChangedContainer)
Expand Down Expand Up @@ -408,20 +408,20 @@ func getActionCommandsWithExecImageOrContainerName(synthesizeComp *SynthesizedCo
}

actions := map[string]*appsv1alpha1.LifecycleActionHandler{
// "postProvision": synthesizeComp.LifecycleActions.PostProvision,
// "preTerminate": synthesizeComp.LifecycleActions.PreTerminate,
"memberJoin": synthesizeComp.LifecycleActions.MemberJoin,
"memberLeave": synthesizeComp.LifecycleActions.MemberLeave,
"readonly": synthesizeComp.LifecycleActions.Readonly,
"readwrite": synthesizeComp.LifecycleActions.Readwrite,
constant.PostProvisionAction: synthesizeComp.LifecycleActions.PostProvision,
constant.PreTerminateAction: synthesizeComp.LifecycleActions.PreTerminate,
constant.MemberJoinAction: synthesizeComp.LifecycleActions.MemberJoin,
constant.MemberLeaveAction: synthesizeComp.LifecycleActions.MemberLeave,
constant.ReadonlyAction: synthesizeComp.LifecycleActions.Readonly,
constant.ReadWriteAction: synthesizeComp.LifecycleActions.Readwrite,
// "dataPopulate": synthesizeComp.LifecycleActions.DataPopulate,
// "dataAssemble": synthesizeComp.LifecycleActions.DataAssemble,
// "reconfigure": synthesizeComp.LifecycleActions.Reconfigure,
// "accountProvision": synthesizeComp.LifecycleActions.AccountProvision,
}

if synthesizeComp.LifecycleActions.RoleProbe != nil {
actions["roleProbe"] = &synthesizeComp.LifecycleActions.RoleProbe.LifecycleActionHandler
actions[constant.RoleProbeAction] = &synthesizeComp.LifecycleActions.RoleProbe.LifecycleActionHandler
}

var toolImage string
Expand Down
32 changes: 32 additions & 0 deletions pkg/lorry/client/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,38 @@ func (cli *lorryClient) LeaveMember(ctx context.Context) error {
return err
}

// Lock sends a set readonly request to Lorry.
func (cli *lorryClient) Lock(ctx context.Context) error {
_, err := cli.Request(ctx, string(LockOperation), http.MethodPost, nil)
return err
}

// Unlock sends a set readwrite request to Lorry.
func (cli *lorryClient) Unlock(ctx context.Context) error {
_, err := cli.Request(ctx, string(UnlockOperation), http.MethodPost, nil)
return err
}

// PostProvision sends a component post provision request to Lorry.
func (cli *lorryClient) PostProvision(ctx context.Context, componentNames, podNames, podIPs, podHostNames, podHostIPs string) error {
parameters := map[string]any{
"componentNames": componentNames,
"podNames": podNames,
"podIPs": podIPs,
"podHostNames": podHostNames,
"podHostIPs": podHostIPs,
}
req := map[string]any{"parameters": parameters}
_, err := cli.Request(ctx, string(PostProvisionOperation), http.MethodPost, req)
return err
}

// PreTerminate sends a pre terminate request to Lorry.
func (cli *lorryClient) PreTerminate(ctx context.Context) error {
_, err := cli.Request(ctx, string(PreTerminateOperation), http.MethodPost, nil)
return err
}

func (cli *lorryClient) Request(ctx context.Context, operation, method string, req map[string]any) (map[string]any, error) {
if cli.requester == nil {
return nil, errors.New("lorry client's requester must be set")
Expand Down
Loading

0 comments on commit fcbf339

Please sign in to comment.