feat: milvus cluster mode

addons/etcd-cluster/values.yaml

@@ -1,6 +1,6 @@
 ## @param terminationPolicy define Cluster termination policy. One of DoNotTerminate, Halt, Delete, WipeOut.
 ##
-terminationPolicy: Halt
+terminationPolicy: Delete
 
 ## Monitoring configurations
 monitor:

addons/etcd/templates/clusterdefinition.yaml

@@ -80,5 +80,5 @@ spec:
   connectionCredential:
     username: root
     password: ""
-    endpoint: "$(SVC_FQDN)"
+    endpoint: "$(SVC_FQDN):$(SVC_PORT_client)"
     port: "$(SVC_PORT_client)"

addons/milvus-cluster/Chart.yaml

@@ -7,3 +7,12 @@ type: application
 version: 0.7.0
 
 appVersion: "2.2.4"
+
+home: https://milvus.io/
+icon: https://github.com/milvus-io/artwork/blob/master/horizontal/color/milvus-horizontal-color.png
+
+dependencies:
+  - name: kblib
+    version: 0.1.0
+    repository: file://../kblib
+    alias: extra

addons/milvus-cluster/templates/_helpers.tpl

@@ -60,3 +60,39 @@ Create the name of the service account to use
 {{- define "milvus.serviceAccountName" -}}
 {{- default (printf "kb-%s" (include "clustername" .)) .Values.serviceAccount.name }}
 {{- end }}
+
+{{/*
+External meta storage service reference
+*/}}
+{{- define "milvus.serviceRef.meta" }}
+{{- if eq .Values.storage.meta.mode "serviceref" }}
+- name: milvus-meta-storage
+  namespace: {{ .Values.storage.meta.serviceRef.namespace }}
+  cluster: {{ .Values.storage.meta.serviceRef.cluster }}
+  serviceDescriptor: {{ .Values.storage.meta.serviceRef.serviceDescriptor }}
+{{- end }}
+{{- end }}
+
+{{/*
+External log storage service reference
+*/}}
+{{- define "milvus.serviceRef.log" }}
+{{- if eq .Values.storage.log.mode "serviceref" }}
+- name: milvus-log-storage
+  namespace: {{ .Values.storage.log.serviceRef.namespace }}
+  cluster: {{ .Values.storage.log.serviceRef.cluster }}
+  serviceDescriptor: {{ .Values.storage.log.serviceRef.serviceDescriptor }}
+{{- end }}
+{{- end }}
+
+{{/*
+External object storage service reference
+*/}}
+{{- define "milvus.serviceRef.object" }}
+{{- if eq .Values.storage.object.mode "serviceref" }}
+- name: milvus-object-storage
+  namespace: {{ .Values.storage.object.serviceRef.namespace }}
+  cluster: {{ .Values.storage.object.serviceRef.cluster }}
+  serviceDescriptor: {{ .Values.storage.object.serviceRef.serviceDescriptor }}
+{{- end }}
+{{- end }}

addons/milvus-cluster/templates/cluster.yaml

@@ -1,101 +1,157 @@
+{{- if eq .Values.mode "cluster" }}
 apiVersion: apps.kubeblocks.io/v1alpha1
 kind: Cluster
 metadata:
-  name: {{ include "clustername" . }}
-  labels: {{ include "milvus.labels" . | nindent 4 }}
+  namespace: {{ .Release.Namespace }}
+  name: {{ include "kblib.clusterName" . }}
+  labels: {{ include "kblib.clusterLabels" . | nindent 4 }}
 spec:
-  clusterDefinitionRef: milvus # ref clusterdefinition.name
-  clusterVersionRef: milvus-{{ default .Chart.AppVersion .Values.clusterVersionOverride }} # ref clusterversion.name
-  terminationPolicy: {{ .Values.terminationPolicy }}
-  affinity:
-    {{- with .Values.topologyKeys }}
-    topologyKeys: {{ . | toYaml | nindent 6 }}
-    {{- end }}
+  clusterDefinitionRef: milvus-{{ .Chart.Version }}
+  terminationPolicy: {{ .Values.extra.terminationPolicy }}
+  {{- include "kblib.affinity" . | indent 2 }}
   {{- with $.Values.tolerations }}
   tolerations: {{ . | toYaml | nindent 4 }}
   {{- end }}
   componentSpecs:
-    - name: milvus # user-defined
-      componentDefRef: milvus # ref clusterdefinition components.name
-      monitor: {{ .Values.monitor.enabled | default false }}
-      replicas: {{ .Values.replicaCount | default 1 }}
-      {{- with  .Values.resources }}
+#    {{- if eq .Values.storage.meta.mode "incluster" }}
+#    - name: {{ .Values.storage.meta.name }}
+#      componentDef: {{ .Values.storage.meta.inCluster.provider }}
+#      replicas: {{ .Values.storage.meta.inCluster.replicas}}
+#      {{- include "kblib.componentMonitor" . | indent 6 }}
+#      {{- with .Values.storage.meta.inCluster.resources }}
+#      resources:
+#        requests:
+#          cpu: {{ .cpu | quote }}
+#          memory: {{ print .memory "Gi" }}
+#      {{- end }}
+#      {{- with .Values.storage.meta.inCluster.storage }}
+#      volumeClaimTemplates:
+#        - name: data  # TODO: volume name and multiple volumes
+#          spec:
+#            storageClassName: {{ .storageClassName }}
+#            accessModes:
+#              - ReadWriteOnce
+#            resources:
+#              requests:
+#                storage: {{ print .size "Gi" }}
+#      {{- end }}
+#    {{- end }}
+#    {{- if eq .Values.storage.log.mode "incluster" }}
+#    - name: {{ .Values.storage.log.name }}
+#      componentDef: {{ .Values.storage.log.inCluster.provider }}
+#      replicas: {{ .Values.storage.log.inCluster.replicas}}
+#      {{- include "kblib.componentMonitor" . | indent 6 }}
+#      {{- with .Values.storage.log.inCluster.resources }}
+#      resources:
+#        requests:
+#          cpu: {{ .cpu | quote }}
+#          memory: {{ print .memory "Gi" }}
+#      {{- end }}
+#      {{- with .Values.storage.log.inCluster.storage }}
+#      volumeClaimTemplates:
+#        - name: data  # TODO: volume name and multiple volumes
+#          spec:
+#            storageClassName: {{ .storageClassName }}
+#            accessModes:
+#              - ReadWriteOnce
+#            resources:
+#              requests:
+#                storage: {{ print .size "Gi" }}
+#      {{- end }}
+#    {{- end }}
+#    {{- if eq .Values.storage.object.mode "incluster" }}
+#    - name: {{ .Values.storage.object.name }}
+#      componentDef: {{ .Values.storage.object.inCluster.provider }}
+#      replicas: {{ .Values.storage.object.inCluster.replicas}}
+#      {{- include "kblib.componentMonitor" . | indent 6 }}
+#      {{- with .Values.storage.object.inCluster.resources }}
+#      resources:
+#        requests:
+#          cpu: {{ .cpu | quote }}
+#          memory: {{ print .memory "Gi" }}
+#      {{- end }}
+#      {{- with .Values.storage.object.inCluster.storage }}
+#      volumeClaimTemplates:
+#        - name: data  # TODO: volume name and multiple volumes
+#          spec:
+#            storageClassName: {{ .storageClassName }}
+#            accessModes:
+#              - ReadWriteOnce
+#            resources:
+#              requests:
+#                storage: {{ print .size "Gi" }}
+#      {{- end }}
+#    {{- end }}
+    {{- if .Values.proxy.enabled }}
+    - name: proxy
+      componentDefRef: proxy
+      replicas: {{ .Values.proxy.replicas }}
+      {{- include "kblib.componentMonitor" . | indent 6 }}
+      {{- with .Values.proxy.resources }}
       resources:
-        {{- with .limits }}
-        limits:
-          cpu: {{ .cpu | quote }}
-          memory: {{ .memory | quote }}
-        {{- end }}
-        {{- with .requests }}
         requests:
           cpu: {{ .cpu | quote }}
-          memory: {{ .memory | quote }}
-        {{- end }}
-      {{- end }}
-      {{- if .Values.persistence.enabled }}
-      volumeClaimTemplates:
-        - name: data # ref clusterdefinition components.containers.volumeMounts.name
-          spec:
-            storageClassName: {{ .Values.persistence.data.storageClassName }}
-            accessModes:
-              - ReadWriteOnce
-            resources:
-              requests:
-                storage: {{ .Values.persistence.data.size }}
+          memory: {{ print .memory "Gi" }}
       {{- end }}
-    - name: etcd # user-defined
-      componentDefRef: etcd # ref clusterdefinition components.name
-      monitor: {{ .Values.monitor.enabled | default false }}
-      replicas: {{ .Values.replicaCount | default 1 }}
-      {{- with  .Values.resources }}
+      serviceRefs:
+        {{- include "milvus.serviceRef.meta" . | indent 8 }}
+        {{- include "milvus.serviceRef.log" . | indent 8 }}
+        {{- include "milvus.serviceRef.object" . | indent 8 }}
+    {{- end }}
+    - name: mixcoord
+      componentDefRef: mixcoord
+      replicas: {{ .Values.mixcoord.replicas}}
+      {{- include "kblib.componentMonitor" . | indent 6 }}
+      {{- with .Values.mixcoord.resources }}
       resources:
-        {{- with .limits }}
-        limits:
-          cpu: {{ .cpu | quote }}
-          memory: {{ .memory | quote }}
-        {{- end }}
-        {{- with .requests }}
         requests:
           cpu: {{ .cpu | quote }}
-          memory: {{ .memory | quote }}
-        {{- end }}
+          memory: {{ print .memory "Gi" }}
       {{- end }}
-      {{- if .Values.persistence.enabled }}
-      volumeClaimTemplates:
-        - name: data # ref clusterdefinition components.containers.volumeMounts.name
-          spec:
-            storageClassName: {{ .Values.persistence.data.storageClassName }}
-            accessModes:
-              - ReadWriteOnce
-            resources:
-              requests:
-                storage: {{ .Values.persistence.data.size }}
+      serviceRefs:
+        {{- include "milvus.serviceRef.meta" . | indent 8 }}
+        {{- include "milvus.serviceRef.log" . | indent 8 }}
+        {{- include "milvus.serviceRef.object" . | indent 8 }}
+    - name: datanode
+      componentDefRef: datanode
+      replicas: {{ .Values.datanode.replicas}}
+      {{- include "kblib.componentMonitor" . | indent 6 }}
+      {{- with .Values.datanode.resources }}
+      resources:
+        requests:
+          cpu: {{ .cpu | quote }}
+          memory: {{ print .memory "Gi" }}
       {{- end }}
-    - name: minio # user-defined
-      componentDefRef: minio # ref clusterdefinition components.name
-      monitor: {{ .Values.monitor.enabled | default false }}
-      replicas: {{ .Values.replicaCount | default 1 }}
-      {{- with  .Values.resources }}
+      serviceRefs:
+        {{- include "milvus.serviceRef.meta" . | indent 8 }}
+        {{- include "milvus.serviceRef.log" . | indent 8 }}
+        {{- include "milvus.serviceRef.object" . | indent 8 }}
+    - name: indexnode
+      componentDefRef: indexnode
+      replicas: {{ .Values.indexnode.replicas}}
+      {{- include "kblib.componentMonitor" . | indent 6 }}
+      {{- with .Values.indexnode.resources }}
       resources:
-        {{- with .limits }}
-        limits:
+        requests:
           cpu: {{ .cpu | quote }}
-          memory: {{ .memory | quote }}
-        {{- end }}
-        {{- with .requests }}
+          memory: {{ print .memory "Gi" }}
+      {{- end }}
+      serviceRefs:
+        {{- include "milvus.serviceRef.meta" . | indent 8 }}
+        {{- include "milvus.serviceRef.log" . | indent 8 }}
+        {{- include "milvus.serviceRef.object" . | indent 8 }}
+    - name: querynode
+      componentDefRef: querynode
+      replicas: {{ .Values.querynode.replicas}}
+      {{- include "kblib.componentMonitor" . | indent 6 }}
+      {{- with .Values.querynode.resources }}
+      resources:
         requests:
           cpu: {{ .cpu | quote }}
-          memory: {{ .memory | quote }}
-        {{- end }}
+          memory: {{ print .memory "Gi" }}
       {{- end }}
-      {{- if .Values.persistence.enabled }}
-      volumeClaimTemplates:
-        - name: data # ref clusterdefinition components.containers.volumeMounts.name
-          spec:
-            storageClassName: {{ .Values.persistence.data.storageClassName }}
-            accessModes:
-              - ReadWriteOnce
-            resources:
-              requests:
-                storage: {{ .Values.persistence.data.size }}
-      {{- end }}
+      serviceRefs:
+        {{- include "milvus.serviceRef.meta" . | indent 8 }}
+        {{- include "milvus.serviceRef.log" . | indent 8 }}
+        {{- include "milvus.serviceRef.object" . | indent 8 }}
+{{- end }}