chore: etcd powered by rsm (#5272)

(cherry picked from commit 8f6e37f)
apecloud · Oct 31, 2023 · bd86e26 · bd86e26
1 parent 41555c1
commit bd86e26
Show file tree

Hide file tree

Showing 7 changed files with 33 additions and 112 deletions.
diff --git a/deploy/etcd-cluster/templates/NOTES.txt b/deploy/etcd-cluster/templates/NOTES.txt
@@ -1,22 +1,2 @@
 1. Get the application URL by running these commands:
-{{- if .Values.ingress.enabled }}
-{{- range $host := .Values.ingress.hosts }}
-  {{- range .paths }}
-  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
-  {{- end }}
-{{- end }}
-{{- else if contains "NodePort" .Values.service.type }}
-  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "etcd-cluster.fullname" . }})
-  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
-  echo http://$NODE_IP:$NODE_PORT
-{{- else if contains "LoadBalancer" .Values.service.type }}
-     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
-           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "etcd-cluster.fullname" . }}'
-  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "etcd-cluster.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
-  echo http://$SERVICE_IP:{{ .Values.service.port }}
-{{- else if contains "ClusterIP" .Values.service.type }}
-  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "etcd-cluster.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
-  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
-  echo "Visit http://127.0.0.1:8080 to use your application"
-  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
-{{- end }}
+kubectl get svc --namespace {{ .Release.Namespace }}
diff --git a/deploy/etcd-cluster/templates/ingress.yaml b/deploy/etcd-cluster/templates/ingress.yaml
diff --git a/deploy/etcd-cluster/templates/tests/test-connection.yaml b/deploy/etcd-cluster/templates/tests/test-connection.yaml
diff --git a/deploy/etcd-cluster/values.yaml b/deploy/etcd-cluster/values.yaml
@@ -57,7 +57,7 @@ resources: { }
 persistence:
   ## @param shard[*].persistence.enabled Enable persistence using Persistent Volume Claims
   ##
-  enabled: false
+  enabled: true
   ## `data` volume settings
   ##
   data:
@@ -71,7 +71,7 @@ persistence:
     storageClassName:
     ## @param shard[*].persistence.size Size of data volume
     ##
-    size: 10Gi
+    size: 1Gi
 
 ingress:
   ## @param ingress.enabled Enable ingress record generation for etcd

diff --git a/deploy/etcd/templates/clusterdefinition.yaml b/deploy/etcd/templates/clusterdefinition.yaml
@@ -10,18 +10,27 @@ spec:
     - name: etcd
       characterType: etcd
       workloadType: Consensus
-      consensusSpec:
-        leader:
-          name: "leader"
-          accessMode: ReadWrite
-        followers:
-          - name: "follower"
+      rsmSpec:
+        roles:
+          - name: leader
             accessMode: ReadWrite
-        updateStrategy: BestEffortParallel
-      probes:
+            isLeader: true
+            canVote: true
+          - name: follower
+            accessMode: ReadWrite
+            isLeader: false
+            canVote: true
         roleProbe:
-          periodSeconds: 1
-          failureThreshold: 3
+          customHandler:
+            - image: quay.io/coreos/etcd:v3.5.6
+              command:
+                - |
+                  Status=$(etcdctl --endpoints=127.0.0.1:2379 endpoint status -w simple --command-timeout=300ms --dial-timeout=100m) &&
+                  IsLeader=$(echo $Status | awk -F ', ' '{print $5}') &&
+                  IsLearner=$(echo $Status | awk -F ', ' '{print $6}') &&
+                  if [ "true" = "$IsLeader" ]; then echo -n "leader"; elif [ "true" = "$IsLearner" ]; then echo -n "learner"; else echo -n "follower"; fi
+          roleUpdateMechanism: DirectAPIServerEventUpdate
+        memberUpdateStrategy: BestEffortParallel
       service:
         ports:
           - name: client
@@ -61,13 +70,12 @@ spec:
                 # TODO: clusterDomain 'cluster.local' requires configurable
                 MY_PEER=$KB_POD_FQDN{{ .Values.clusterDomain }}
                 exec etcd --name ${HOSTNAME} \
+                --experimental-initial-corrupt-check=true \
                 --listen-peer-urls http://0.0.0.0:2380 \
                 --listen-client-urls http://0.0.0.0:2379 \
                 --advertise-client-urls http://${MY_PEER}:2379 \
                 --initial-advertise-peer-urls http://${MY_PEER}:2380 \
-                --initial-cluster-token etcd-cluster-1 \
                 --initial-cluster ${PEERS} \
-                --initial-cluster-state new \
                 --data-dir /var/run/etcd/default.etcd
   connectionCredential:
     username: root

diff --git a/test/e2e/testdata/smoketest/etcd/00_etcdcluster.yaml b/test/e2e/testdata/smoketest/etcd/00_etcdcluster.yaml
@@ -23,3 +23,12 @@ spec:
       monitor: false
       replicas: 3
       serviceAccountName: kb-etcd-cluster
+      volumeClaimTemplates:
+        - name: data # ref clusterdefinition components.containers.volumeMounts.name
+          spec:
+            storageClassName: 
+            accessModes:
+              - ReadWriteOnce
+            resources:
+              requests:
+                storage: 1Gi
diff --git a/test/e2e/testdata/smoketest/etcd/03_vscale.yaml b/test/e2e/testdata/smoketest/etcd/03_vscale.yaml
@@ -9,4 +9,4 @@ spec:
     - componentName: etcd
       requests:
         cpu: "500m"
-        memory: 500Mi
+        memory: 500Mi