bugfix: wescale errors (#1350)

(cherry picked from commit 279bbac)
apecloud · Dec 20, 2024 · f35f526 · f35f526
1 parent 635e1d7
commit f35f526
Show file tree

Hide file tree

Showing 13 changed files with 297 additions and 158 deletions.
diff --git a/addons-cluster/apecloud-mysql/templates/_helpers.tpl b/addons-cluster/apecloud-mysql/templates/_helpers.tpl
@@ -13,7 +13,10 @@ The minimum proxy cpu cores is 0.5 and the maximum cpu cores is 64.
 {{- $proxyCPU = 64 }}
 {{- end }}
 - name: wescale-ctrl
-  {{- if not .Values.localEtcdEnabled }}
+  env:
+    - name: ETCDCTL_API
+      value: "{{ .Values.etcd.local.etcdctlApi }}"
+  {{- if eq .Values.etcd.mode "serviceRef" }}
   serviceRefs:
     {{ include "apecloud-mysql-cluster.serviceRef" . | indent 4 }}
   {{- end }}
@@ -32,7 +35,10 @@ The minimum proxy cpu cores is 0.5 and the maximum cpu cores is 64.
       cpu: 500m
       memory: 128Mi
 - name: wescale
-  {{- if not .Values.localEtcdEnabled }}
+  env:
+    - name: ETCDCTL_API
+      value: "{{ .Values.etcd.local.etcdctlApi }}"
+  {{- if eq .Values.etcd.mode "serviceRef" }}
   serviceRefs:
     {{ include "apecloud-mysql-cluster.serviceRef" . | indent 4 }}
   {{- end }}
@@ -61,7 +67,7 @@ raftGroup mode: max(replicas, 3)
 
 {{- define "apecloud-mysql-cluster.topology" }}
 {{- if and (eq .Values.mode "raftGroup") .Values.proxyEnabled }}
-  {{- if .Values.localEtcdEnabled }}
+  {{- if  eq .Values.etcd.mode "local" }}
     {{- "apecloud-mysql-proxy-etcd" }}
   {{- else }}
     {{- "apecloud-mysql-proxy" }}
@@ -73,12 +79,34 @@ raftGroup mode: max(replicas, 3)
 
 {{- define "apecloud-mysql-cluster.serviceRef" }}
 - name: etcd
-  namespace: {{ .Release.Namespace }}
-  serviceDescriptor: {{ include "kblib.clusterName" . }}-etcd-descriptor
+  namespace: {{ .Values.etcd.serviceRef.namespace }}
+  {{- if .Values.etcd.serviceRef.cluster }}
+  clusterServiceSelector:
+    cluster: {{ .Values.etcd.serviceRef.cluster.name }}
+    service:
+      component: {{ .Values.etcd.serviceRef.cluster.component }}
+      service: {{ .Values.etcd.serviceRef.cluster.service }}
+      port: {{ .Values.etcd.serviceRef.cluster.port }}
+    {{- if .Values.etcd.serviceRef.cluster.credential }}
+    credential:
+      component: {{ .Values.etcd.serviceRef.cluster.component }}
+      name: {{ .Values.etcd.serviceRef.cluster.credential }}
+    {{- end }}
+  {{- end }}
+  serviceDescriptor: {{ .Values.etcd.serviceRef.serviceDescriptor }}
 {{- end -}}
 
 {{- define "apecloud-mysql-cluster.etcdComponents" }}
 - name: etcd
+  serviceVersion: {{ .Values.etcd.local.serviceVersion | default "3.5.15" | quote }}
+  replicas: {{ .Values.etcd.local.replicas }}
+  resources:
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    limits:
+      cpu: 500m
+      memory: 500Mi
   volumeClaimTemplates:
     - name: data
       spec:
@@ -87,15 +115,7 @@ raftGroup mode: max(replicas, 3)
           - ReadWriteOnce
         resources:
           requests:
-            storage: {{ .Values.etcd.resources.storage }}
-  replicas: {{ .Values.etcd.replicas }}
-  resources:
-    requests:
-      cpu: 500m
-      memory: 500Mi
-    limits:
-      cpu: 500m
-      memory: 500Mi
+            storage: {{ .Values.etcd.local.resources.storage }}
 {{- end -}}
 
 {{- define "apecloud-mysql-cluster.schedulingPolicy" }}
@@ -116,4 +136,4 @@ schedulingPolicy:
             app.kubernetes.io/instance: {{ include "kblib.clusterName" . }}
             apps.kubeblocks.io/component-name: mysql
         topologyKey: kubernetes.io/hostname
-{{- end -}}
+{{- end -}}
diff --git a/addons-cluster/apecloud-mysql/templates/cluster.yaml b/addons-cluster/apecloud-mysql/templates/cluster.yaml
@@ -5,30 +5,31 @@ metadata:
   namespace: {{ .Release.Namespace }}
   labels:
     {{ include "kblib.clusterLabels" . | nindent 4 }}
-  {{- if and (eq .Values.mode "raftGroup") .Values.proxyEnabled }}
-  annotations:
-    kubeblocks.io/extra-env: '{"KB_PROXY_ENABLED":"on","ETCDCTL_API": "{{ .Values.etcd.etcdctlApi}}"}'
-  {{- end }}
 spec:
   terminationPolicy: {{ .Values.extra.terminationPolicy }}
   clusterDef: apecloud-mysql # ref clusterdefinition.name
   topology: {{ include "apecloud-mysql-cluster.topology" . }}
   componentSpecs:
     - name: mysql
+      serviceVersion: {{ .Values.version }}
+      {{- if and (eq .Values.mode "raftGroup") .Values.proxyEnabled }}
+      env:
+        - name: KB_PROXY_ENABLED
+          value: "on"
+      {{- end }}
       {{- include "kblib.componentMonitor" . | indent 6 }}
       {{- include "apecloud-mysql-cluster.schedulingPolicy" . | indent 6 }}
-      {{- if and (eq .Values.mode "raftGroup") .Values.proxyEnabled (not .Values.localEtcdEnabled) }}
-      serviceVersion: {{ .Values.version }}
+      {{- if and (eq .Values.mode "raftGroup") .Values.proxyEnabled (eq .Values.etcd.mode "serviceRef") }}
       serviceRefs:
         {{ include "apecloud-mysql-cluster.serviceRef" . | indent 8 }}
       {{- end }}
       replicas: {{ include "apecloud-mysql-cluster.replicas" . }}
       {{- include "kblib.componentResources" . | indent 6 }}
       {{- include "kblib.componentStorages" . | indent 6 }}
       {{- if and (eq .Values.mode "raftGroup") .Values.proxyEnabled }}
-      {{- include "apecloud-mysql-cluster.proxyComponents" . | indent 4 }}
-        {{- if .Values.localEtcdEnabled }}
-      {{- include "apecloud-mysql-cluster.etcdComponents" . | indent 4 }}
+        {{- include "apecloud-mysql-cluster.proxyComponents" . | indent 4 }}
+        {{- if eq .Values.etcd.mode "local" }}
+          {{- include "apecloud-mysql-cluster.etcdComponents" . | indent 4 }}
         {{- end }}
       {{- end }}
 
diff --git a/addons-cluster/apecloud-mysql/templates/etcd-service-descriptor.yaml b/addons-cluster/apecloud-mysql/templates/etcd-service-descriptor.yaml
diff --git a/addons-cluster/apecloud-mysql/values.schema.json b/addons-cluster/apecloud-mysql/values.schema.json
@@ -70,6 +70,79 @@
       "description": "Enable proxy or not.",
       "type": "boolean",
       "default": false
+    },
+    "etcd": {
+      "type": "object",
+      "properties": {
+        "mode": {
+          "type": "string",
+          "enum": ["serviceRef", "local"],
+          "default": "local"
+        },
+        "local": {
+          "type": "object",
+          "properties": {
+            "serviceVersion": {
+              "type": "string",
+              "default": "3.5.6"
+            },
+            "etcdctlApi": {
+              "type": "string",
+              "default": "3"
+            },
+            "replicas": {
+              "type": "integer",
+              "default": 3,
+              "minimum": 1,
+              "maximum": 3
+            },
+            "resources": {
+              "type": "object",
+              "properties": {
+                "storage": {
+                  "type": "string",
+                  "default": "20Gi"
+                }
+              }
+            }
+          }
+        },
+        "serviceRef": {
+          "type": "object",
+          "properties": {
+            "namespace": {
+              "type": "string",
+              "default": "default"
+            },
+            "cluster": {
+              "type": "object",
+              "properties": {
+                "name": {
+                  "type": "string"
+                },
+                "component": {
+                  "type": "string",
+                  "default": "etcd"
+                },
+                "service": {
+                  "type": "string",
+                  "default": "headless"
+                },
+                "port": {
+                  "type": "string",
+                  "default": "client"
+                },
+                "credential": {
+                  "type": "string"
+                }
+              }
+            },
+            "serviceDescriptor": {
+              "type": "string"
+            }
+          }
+        }
+      }
     }
   }
 }
diff --git a/addons-cluster/apecloud-mysql/values.yaml b/addons-cluster/apecloud-mysql/values.yaml
@@ -39,26 +39,47 @@ storage: 20
 ##
 proxyEnabled: false
 
-## @param localEtcdEnabled if true, use the local etcd
-## if mode is standalone, localEtcdEnabled is always false
-## if mode is raftGroup, localEtcdEnabled can be true
-localEtcdEnabled: false
-
 proxy:
   storageClassName: ""
 
 ## customized default values to override kblib chart's values
 extra:
   disableExporter: true
-  rbacEnabled: true
+  rbacEnabled: false
   terminationPolicy: Delete
 
 
 etcd:
-  serviceReference:
-    endpoint: "" # ex: etcd-cluster-etcd-headless.default.svc.cluster.local:2379
-  serviceVersion: 3.5.6
-  etcdctlApi: "3" # the etcd tool etcdctl api version
-  replicas: 3 # no support other num
-  resources:
-    storage: 20Gi
+  # etcd mode
+  #
+  # @param mode The deployment mode of etcd
+  # @valid serviceRef, local
+  # @default local
+  mode: local
+  local:
+    serviceVersion: 3.5.6
+    etcdctlApi: "3" # the etcd tool etcdctl api version
+    replicas: 3 # no support other num
+    resources:
+      storage: 20Gi
+  serviceRef:
+    # -- Kubernetes namespace for etcd
+    # @default default
+    namespace: default
+    cluster:
+      # -- Name of the etcd cluster
+      # @required
+      name: etcdm-cluster
+      # -- Component name for etcd
+      # @default etcd
+      component: etcd
+      # -- Service type for etcd
+      # @default headless
+      service: headless
+      # -- Port name for etcd client
+      # @default client
+      port: client
+      # -- Credentials for etcd
+      # @optional
+      credential: ""
+    serviceDescriptor: ""
diff --git a/addons/apecloud-mysql/scripts/vttablet.sh b/addons/apecloud-mysql/scripts/vttablet.sh
@@ -19,7 +19,7 @@ vtctld_host=${VTCTLD_HOST:-'127.0.0.1'}
 vtctld_web_port=${VTCTLD_WEB_PORT:-'15000'}
 printf -v alias '%s-%010d' $cell $uid
 printf -v tablet_dir 'vt_%010d' $uid
-tablet_hostname="$POD_NAME.$COMPONENT_NAME-headless.$CLUSTER_NAMESPACE.svc.cluster.local"
+tablet_hostname="$POD_NAME.$CLUSTER_COMPONENT_NAME-headless.$CLUSTER_NAMESPACE.svc.cluster.local"
 printf -v tablet_logfile 'vttablet_%010d_querylog.txt' $uid
 
 tablet_type=replica

diff --git a/addons/apecloud-mysql/templates/_helpers.tpl b/addons/apecloud-mysql/templates/_helpers.tpl
@@ -181,7 +181,7 @@ lifecycleActions:
           eval statement=\"${KB_ACCOUNT_STATEMENT}\"
           mysql -u${MYSQL_ROOT_USER} -p${MYSQL_ROOT_PASSWORD} -P3306 -h127.0.0.1 -e "${statement}"
       targetPodSelector: Role
-      matchingKey: leader 
+      matchingKey: leader
   dataDump:
     exec:
       image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:8.0.30-5.beta3.20240330.g94d1caf.15
@@ -272,7 +272,7 @@ vars:
         compDef: {{ .Values.etcd.etcdCmpdName }}
         name: headless
         optional: true
-        port: 
+        port:
           name: client
           option: Optional
   - name: COMPONENT_POD_LIST

diff --git a/addons/apecloud-mysql/templates/clusterdefinition.yaml b/addons/apecloud-mysql/templates/clusterdefinition.yaml
@@ -26,6 +26,10 @@ spec:
       - mysql
       - wescale-ctrl
       - wescale
+      terminate:
+      - wescale
+      - wescale-ctrl
+      - mysql
   - name: apecloud-mysql-proxy-etcd
     components:
     - name: mysql
@@ -42,3 +46,8 @@ spec:
       - mysql
       - wescale-ctrl
       - wescale
+      terminate:
+      - wescale
+      - wescale-ctrl
+      - mysql
+      - etcd
diff --git a/addons/apecloud-mysql/templates/cmpd-wescale-controller.yaml b/addons/apecloud-mysql/templates/cmpd-wescale-controller.yaml
@@ -26,6 +26,7 @@ spec:
   lifecycleActions:
     preTerminate:
       exec:
+        container: vtctld
         image: {{ .Values.wesqlscale.image.registry | default ( .Values.image.registry | default "docker.io" ) }}/{{ .Values.wesqlscale.image.repository }}:{{ .Values.wesqlscale.image.tag }}
         command: ["/scripts/etcd-clean.sh"]
   runtime:
@@ -100,6 +101,16 @@ spec:
           compDef: {{ .Values.etcd.etcdCmpdName }}
           name: headless
           optional: true
-          port: 
+          port:
             name: client
-            option: Optional
+            option: Optional
+    - name: COMPONENT_NAME
+      valueFrom:
+        componentVarRef:
+          optional: false
+          shortName: Required
+    - name: CLUSTER_COMPONENT_NAME
+      valueFrom:
+        componentVarRef:
+          optional: false
+          componentName: Required