From bd86e26741809e98baefb3643e84a8726bf04b7a Mon Sep 17 00:00:00 2001 From: free6om Date: Tue, 31 Oct 2023 22:17:48 +0800 Subject: [PATCH] chore: etcd powered by rsm (#5272) (cherry picked from commit 8f6e37fc8e02e8ee269dbf69d867a3d96fd59768) --- deploy/etcd-cluster/templates/NOTES.txt | 22 +------ deploy/etcd-cluster/templates/ingress.yaml | 61 ------------------- .../templates/tests/test-connection.yaml | 15 ----- deploy/etcd-cluster/values.yaml | 4 +- deploy/etcd/templates/clusterdefinition.yaml | 32 ++++++---- .../smoketest/etcd/00_etcdcluster.yaml | 9 +++ .../testdata/smoketest/etcd/03_vscale.yaml | 2 +- 7 files changed, 33 insertions(+), 112 deletions(-) delete mode 100644 deploy/etcd-cluster/templates/ingress.yaml delete mode 100644 deploy/etcd-cluster/templates/tests/test-connection.yaml diff --git a/deploy/etcd-cluster/templates/NOTES.txt b/deploy/etcd-cluster/templates/NOTES.txt index 57f61e32e51..786eacaf78b 100644 --- a/deploy/etcd-cluster/templates/NOTES.txt +++ b/deploy/etcd-cluster/templates/NOTES.txt @@ -1,22 +1,2 @@ 1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "etcd-cluster.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "etcd-cluster.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "etcd-cluster.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "etcd-cluster.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} +kubectl get svc --namespace {{ .Release.Namespace }} diff --git a/deploy/etcd-cluster/templates/ingress.yaml b/deploy/etcd-cluster/templates/ingress.yaml deleted file mode 100644 index 551983970e3..00000000000 --- a/deploy/etcd-cluster/templates/ingress.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "etcd-cluster.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "etcd-cluster.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/deploy/etcd-cluster/templates/tests/test-connection.yaml b/deploy/etcd-cluster/templates/tests/test-connection.yaml deleted file mode 100644 index f6bbd83c2cd..00000000000 --- a/deploy/etcd-cluster/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "clustername" . }}-test-connection" - labels: - {{- include "etcd-cluster.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "clustername" . }}:{{ .Values.service.port }}'] - restartPolicy: Never diff --git a/deploy/etcd-cluster/values.yaml b/deploy/etcd-cluster/values.yaml index 47ff849f7e7..b1942938125 100644 --- a/deploy/etcd-cluster/values.yaml +++ b/deploy/etcd-cluster/values.yaml @@ -57,7 +57,7 @@ resources: { } persistence: ## @param shard[*].persistence.enabled Enable persistence using Persistent Volume Claims ## - enabled: false + enabled: true ## `data` volume settings ## data: @@ -71,7 +71,7 @@ persistence: storageClassName: ## @param shard[*].persistence.size Size of data volume ## - size: 10Gi + size: 1Gi ingress: ## @param ingress.enabled Enable ingress record generation for etcd diff --git a/deploy/etcd/templates/clusterdefinition.yaml b/deploy/etcd/templates/clusterdefinition.yaml index 6c3049234bc..5fa41b864b0 100644 --- a/deploy/etcd/templates/clusterdefinition.yaml +++ b/deploy/etcd/templates/clusterdefinition.yaml @@ -10,18 +10,27 @@ spec: - name: etcd characterType: etcd workloadType: Consensus - consensusSpec: - leader: - name: "leader" - accessMode: ReadWrite - followers: - - name: "follower" + rsmSpec: + roles: + - name: leader accessMode: ReadWrite - updateStrategy: BestEffortParallel - probes: + isLeader: true + canVote: true + - name: follower + accessMode: ReadWrite + isLeader: false + canVote: true roleProbe: - periodSeconds: 1 - failureThreshold: 3 + customHandler: + - image: quay.io/coreos/etcd:v3.5.6 + command: + - | + Status=$(etcdctl --endpoints=127.0.0.1:2379 endpoint status -w simple --command-timeout=300ms --dial-timeout=100m) && + IsLeader=$(echo $Status | awk -F ', ' '{print $5}') && + IsLearner=$(echo $Status | awk -F ', ' '{print $6}') && + if [ "true" = "$IsLeader" ]; then echo -n "leader"; elif [ "true" = "$IsLearner" ]; then echo -n "learner"; else echo -n "follower"; fi + roleUpdateMechanism: DirectAPIServerEventUpdate + memberUpdateStrategy: BestEffortParallel service: ports: - name: client @@ -61,13 +70,12 @@ spec: # TODO: clusterDomain 'cluster.local' requires configurable MY_PEER=$KB_POD_FQDN{{ .Values.clusterDomain }} exec etcd --name ${HOSTNAME} \ + --experimental-initial-corrupt-check=true \ --listen-peer-urls http://0.0.0.0:2380 \ --listen-client-urls http://0.0.0.0:2379 \ --advertise-client-urls http://${MY_PEER}:2379 \ --initial-advertise-peer-urls http://${MY_PEER}:2380 \ - --initial-cluster-token etcd-cluster-1 \ --initial-cluster ${PEERS} \ - --initial-cluster-state new \ --data-dir /var/run/etcd/default.etcd connectionCredential: username: root diff --git a/test/e2e/testdata/smoketest/etcd/00_etcdcluster.yaml b/test/e2e/testdata/smoketest/etcd/00_etcdcluster.yaml index 0a93c1afd69..fa5facd6e00 100644 --- a/test/e2e/testdata/smoketest/etcd/00_etcdcluster.yaml +++ b/test/e2e/testdata/smoketest/etcd/00_etcdcluster.yaml @@ -23,3 +23,12 @@ spec: monitor: false replicas: 3 serviceAccountName: kb-etcd-cluster + volumeClaimTemplates: + - name: data # ref clusterdefinition components.containers.volumeMounts.name + spec: + storageClassName: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/test/e2e/testdata/smoketest/etcd/03_vscale.yaml b/test/e2e/testdata/smoketest/etcd/03_vscale.yaml index c8151ba8fbd..f5a8fd169e8 100644 --- a/test/e2e/testdata/smoketest/etcd/03_vscale.yaml +++ b/test/e2e/testdata/smoketest/etcd/03_vscale.yaml @@ -9,4 +9,4 @@ spec: - componentName: etcd requests: cpu: "500m" - memory: 500Mi \ No newline at end of file + memory: 500Mi