chore: clickhouse support tls (#1020)

Co-authored-by: loomts <[email protected]>
apecloud · Sep 26, 2024 · 21bb8d3 · 21bb8d3
1 parent 2e06d56
commit 21bb8d3
Show file tree

Hide file tree

Showing 16 changed files with 503 additions and 204 deletions.
diff --git a/README.md b/README.md
@@ -21,7 +21,7 @@ KubeBlocks add-ons.
 | apecloud-mysql | apecloud-mysql-8.0.30<br>wescale-0.2.7 | ApeCloud MySQL is a database that is compatible with MySQL syntax and achieves high availability through the utilization of the RAFT consensus protocol. | xuriwuyun |
 | apecloud-postgresql | apecloud-postgresql-14.11.0 | ApeCloud PostgreSQL is a database that is compatible with PostgreSQL syntax and achieves high availability through the utilization of the RAFT consensus protocol. | ldming |
 | camellia-redis-proxy | camellia-redis-proxy-1.2.26 | camellia-redis-proxy is a high-performance redis proxy developed using Netty4. | Y-Rookie |
-| clickhouse | clickhouse-cluster-24.3.10 | ClickHouse is an open-source column-oriented OLAP database management system. Use it to boost your database performance while providing linear scalability and hardware efficiency. | sophon-zt |
+| clickhouse | clickhouse-24.8.3 | ClickHouse is an open-source column-oriented OLAP database management system. Use it to boost your database performance while providing linear scalability and hardware efficiency. | sophon-zt |
 | dmdb | dmdb-0.1.0 | A Helm chart for Kubernetes |  |
 | elasticsearch | elasticsearch-7.10.1<br>elasticsearch-7.7.1<br>elasticsearch-7.8.1<br>elasticsearch-8.1.3<br>elasticsearch-8.8.2 | Elasticsearch is a distributed, RESTful search engine optimized for speed and relevance on production-scale workloads. | iziang |
 | etcd | etcd-v3.5.15<br>etcd-v3.5.6 | Etcd is a strongly consistent, distributed key-value store that provides a reliable way to store data that needs to be accessed by a distributed system or cluster of machines. | free6om |

diff --git a/addons-cluster/clickhouse/Chart.yaml b/addons-cluster/clickhouse/Chart.yaml
@@ -6,7 +6,7 @@ type: application
 
 version: 1.0.0-alpha.0
 
-appVersion: 24.3.10
+appVersion: 24.8.3
 
 dependencies:
   - name: kblib

diff --git a/addons-cluster/clickhouse/templates/NOTES.txt b/addons-cluster/clickhouse/templates/NOTES.txt
@@ -15,7 +15,7 @@
   export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "clickhouse-cluster.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
   echo http://$SERVICE_IP:{{ .Values.service.port }}
 {{- else if contains "ClusterIP" .Values.service.type }}
-  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "clickhouse-cluster.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name=clickhouse-24,app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
   export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
   echo "Visit http://127.0.0.1:8080 to use your application"
   kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT

diff --git a/addons-cluster/clickhouse/templates/_helpers.tpl b/addons-cluster/clickhouse/templates/_helpers.tpl
@@ -61,6 +61,24 @@ Create the name of the service account to use
 {{- default (printf "kb-%s" (include "clustername" .)) .Values.serviceAccount.name }}
 {{- end }}
 
+{{/*
+TLS file
+*/}}
+{{- define "clickhouse-cluster.tls" -}}
+tls: {{ $.Values.tls.enabled }}
+{{- if $.Values.tls.enabled }}
+issuer:
+  name: {{ $.Values.tls.issuer }}
+  {{- if eq $.Values.tls.issuer "UserProvided" }}
+  secretRef:
+    name: {{ $.Values.tls.secretName }}
+    ca: ca.crt
+    cert: tls.crt
+    key: tls.key
+  {{- end }}
+{{- end }}
+{{- end }}
+
 {{/*
 Define clickhouse componentSpec with ComponentDefinition.
 */}}
@@ -90,6 +108,7 @@ Define clickhouse componentSpec with ComponentDefinition.
         resources:
           requests:
             storage: {{ $.Values.clickhouse.persistence.data.size }}
+{{ include "clickhouse-cluster.tls" . | indent 2 }}
 {{- end }}
 
 {{/*
@@ -120,6 +139,7 @@ Define clickhouse keeper componentSpec with ComponentDefinition.
         resources:
           requests:
             storage: {{ $.Values.keeper.persistence.data.size }}
+{{ include "clickhouse-cluster.tls" . | indent 2 }}
 {{- end }}
 
 {{/*
@@ -154,6 +174,7 @@ Define clickhouse shardingComponentSpec with ComponentDefinition.
           resources:
             requests:
               storage: {{ $.Values.clickhouse.persistence.data.size }}
+{{ include "clickhouse-cluster.tls" . | indent 4 }}
 {{- end }}
 
 {{/*
@@ -187,5 +208,6 @@ Define clickhouse componentSpec with compatible ComponentDefinition API
         resources:
           requests:
             storage: {{ $.Values.clickhouse.persistence.data.size }}
+{{ include "clickhouse-cluster.tls" . | indent 2 }}
 {{- end }}
 {{- end }}
diff --git a/addons-cluster/clickhouse/templates/cluster-tls-secrets.yaml b/addons-cluster/clickhouse/templates/cluster-tls-secrets.yaml
@@ -0,0 +1,15 @@
+{{- if and .Values.tls.enabled (eq .Values.tls.issuer "UserProvided") }}
+{{- $ca := genCA "KubeBlocks" 36500 }}
+{{- $cert := genSignedCert "clickhouse" (list "127.0.0.1" "::1") (list "localhost" "*.cluster.local") 36500 $ca }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Values.tls.secretName }}
+  namespace: {{ .Release.Namespace | quote }}
+  labels: {{- include "clickhouse-cluster.labels" . | nindent 4 }}
+type: Opaque
+stringData:
+  ca.crt: {{ $ca.Cert | quote }}
+  tls.crt: {{ $cert.Cert | quote }}
+  tls.key: {{ $cert.Key | quote }}
+{{- end }}
diff --git a/addons-cluster/clickhouse/values.yaml b/addons-cluster/clickhouse/values.yaml
@@ -283,3 +283,11 @@ extra:
 
 sharding:
   enabled: false
+
+## @param tlsEnable Enable TLS for clickhouse cluster
+tls:
+  enabled: false
+  # KubeBlocks or UserProvided
+  issuer: UserProvided
+  # if issuer is UserProvided, will automatically generate tls file by cluster-tls-secrets.yaml
+  secretName: clickhouse-cluster-tls
diff --git a/addons/clickhouse/Chart.yaml b/addons/clickhouse/Chart.yaml
@@ -8,7 +8,7 @@ type: application
 
 version: 1.0.0-alpha.0
 
-appVersion: 24.3.10
+appVersion: 24.8.3
 
 home: https://clickhouse.com/
 icon: https://bitnami.com/assets/stacks/clickhouse/img/clickhouse-stack-220x234.png

diff --git a/addons/clickhouse/configs/00_default_overrides.xml.tpl b/addons/clickhouse/configs/00_default_overrides.xml.tpl
@@ -1,10 +1,23 @@
 {{- $clusterName := $.cluster.metadata.name }}
 {{- $namespace := $.cluster.metadata.namespace }}
 <clickhouse>
-  <!-- Macros -->
+  <listen_host>0.0.0.0</listen_host>
+  {{- if $.component.tlsConfig }}
+  <https_port replace="replace" from_env="CLICKHOUSE_HTTPS_PORT"/>
+  <tcp_port_secure replace="replace" from_env="CLICKHOUSE_TCP_SECURE_PORT"/>
+  <interserver_https_port replace="replace" from_env="CLICKHOUSE_INTERSERVER_HTTPS_PORT"/>
+  <http_port remove="remove"/>
+  <tcp_port remove="remove"/>
+  <interserver_http_port remove="remove"/>
+  {{- else }}
+  <http_port replace="replace" from_env="CLICKHOUSE_HTTP_PORT"/>
+  <tcp_port replace="replace" from_env="CLICKHOUSE_TCP_PORT"/>
+  <interserver_http_port replace="replace" from_env="CLICKHOUSE_INTERSERVER_HTTP_PORT"/>
+  {{- end }}
+  <!-- Macros, self defined -->
   <macros>
-    <shard from_env="CLICKHOUSE_SHARD_ID"></shard>
-    <replica from_env="CLICKHOUSE_REPLICA_ID"></replica>
+    <shard from_env="CLICKHOUSE_SHARD_ID"/>
+    <replica from_env="CLICKHOUSE_REPLICA_ID"/>
     <layer>{{ $clusterName }}</layer>
   </macros>
   <!-- Log Level -->
@@ -15,12 +28,17 @@
   <remote_servers>
     <default>
       <shard>
-    {{- range $_, $host := splitList "," .CLICKHOUSE_POD_FQDN_LIST }}
+        {{- range $_, $host := splitList "," .CLICKHOUSE_POD_FQDN_LIST }}
         <replica>
-            <host>{{ $host }}</host>
-            <port>9000</port>
+          <host>{{ $host }}</host>
+          {{- if $.component.tlsConfig }}
+          <port replace="replace" from_env="CLICKHOUSE_TCP_SECURE_PORT"/>
+          <secure>1</secure>
+          {{- else }}
+          <port replace="replace" from_env="CLICKHOUSE_TCP_PORT"/>
+          {{- end }}
         </replica>
-    {{- end }}
+        {{- end }}
       </shard>
     </default>
   </remote_servers>
@@ -29,16 +47,76 @@
     {{- range $_, $host := splitList "," .CH_KEEPER_POD_FQDN_LIST }}
     <node>
       <host>{{ $host }}</host>
-      <port>2181</port>
+      {{- if $.component.tlsConfig }}
+      <port replace="replace" from_env="CLICKHOUSE_KEEPER_TCP_TLS_PORT"/>
+      <secure>1</secure>
+      {{- else }}
+      <port replace="replace" from_env="CLICKHOUSE_KEEPER_TCP_PORT"/>
+      {{- end }}
     </node>
     {{- end }}
   </zookeeper>
   <!-- Prometheus metrics -->
   <prometheus>
     <endpoint>/metrics</endpoint>
-    <port from_env="CLICKHOUSE_METRICS_PORT"></port>
+    <port replace="replace" from_env="CLICKHOUSE_METRICS_PORT"/>
     <metrics>true</metrics>
     <events>true</events>
     <asynchronous_metrics>true</asynchronous_metrics>
   </prometheus>
+  <!-- tls configuration -->
+  {{- if $.component.tlsConfig -}}
+  {{- $CA_FILE := getCAFile -}}
+  {{- $CERT_FILE := getCertFile -}}
+  {{- $KEY_FILE := getKeyFile }}
+  <protocols>
+    <prometheus_protocol>
+      <type>prometheus</type>
+      <description>prometheus protocol</description>
+    </prometheus_protocol>
+    <prometheus_secure>
+      <type>tls</type>
+      <impl>prometheus_protocol</impl>
+      <description>prometheus over https</description>
+      <certificateFile>{{$CERT_FILE}}</certificateFile>
+      <privateKeyFile>{{$KEY_FILE}}</privateKeyFile>
+    </prometheus_secure>
+  </protocols>
+  <openSSL>
+    <server>
+      <certificateFile>{{$CERT_FILE}}</certificateFile>
+      <privateKeyFile>{{$KEY_FILE}}</privateKeyFile>
+      <verificationMode>relaxed</verificationMode>
+      <caConfig>{{$CA_FILE}}</caConfig>
+      <cacheSessions>true</cacheSessions>
+      <disableProtocols>sslv2,sslv3</disableProtocols>
+      <preferServerCiphers>true</preferServerCiphers>
+    </server>
+    <client>
+      <loadDefaultCAFile>false</loadDefaultCAFile>
+      <certificateFile>{{$CERT_FILE}}</certificateFile>
+      <privateKeyFile>{{$KEY_FILE}}</privateKeyFile>
+      <caConfig>{{$CA_FILE}}</caConfig>
+      <cacheSessions>true</cacheSessions>
+      <disableProtocols>sslv2,sslv3</disableProtocols>
+      <preferServerCiphers>true</preferServerCiphers>
+      <verificationMode>relaxed</verificationMode>
+      <invalidCertificateHandler>
+        <name>RejectCertificateHandler</name>
+      </invalidCertificateHandler>
+    </client>
+  </openSSL>
+  <grpc>
+    <enable_ssl>1</enable_ssl>
+    <ssl_cert_file>{{$CERT_FILE}}</ssl_cert_file>
+    <ssl_key_file>{{$KEY_FILE}}</ssl_key_file>
+    <ssl_require_client_auth>false</ssl_require_client_auth>
+    <ssl_ca_cert_file>{{$CA_FILE}}</ssl_ca_cert_file>
+    <transport_compression_type>none</transport_compression_type>
+    <transport_compression_level>0</transport_compression_level>
+    <max_send_message_size>-1</max_send_message_size>
+    <max_receive_message_size>-1</max_receive_message_size>
+    <verbose_logs>false</verbose_logs>
+  </grpc>
+  {{- end }}
 </clickhouse>
diff --git a/addons/clickhouse/configs/ch-keeper_00_default_overrides.xml.tpl b/addons/clickhouse/configs/ch-keeper_00_default_overrides.xml.tpl
@@ -2,9 +2,26 @@
 {{- $namespace := $.cluster.metadata.namespace }}
 <clickhouse>
   <listen_host>0.0.0.0</listen_host>
+  {{- if $.component.tlsConfig }}
+  <https_port replace="replace" from_env="CLICKHOUSE_HTTPS_PORT"/>
+  <tcp_port_secure replace="replace" from_env="CLICKHOUSE_TCP_SECURE_PORT"/>
+  <interserver_https_port replace="replace" from_env="CLICKHOUSE_INTERSERVER_HTTPS_PORT"/>
+  <http_port remove="remove"/>
+  <tcp_port remove="remove"/>
+  <interserver_http_port remove="remove"/>
+  {{- else }}
+  <http_port replace="replace" from_env="CLICKHOUSE_HTTP_PORT"/>
+  <tcp_port replace="replace" from_env="CLICKHOUSE_TCP_PORT"/>
+  <interserver_http_port replace="replace" from_env="CLICKHOUSE_INTERSERVER_HTTP_PORT"/>
+  {{- end }}
   <keeper_server>
-      <tcp_port from_env="CLICKHOUSE_KEEPER_TCP_PORT"></tcp_port>
-      <server_id from_env="CH_KEEPER_ID"></server_id>
+      {{- if $.component.tlsConfig }}
+      <tcp_port_secure replace="replace" from_env="CLICKHOUSE_KEEPER_TCP_TLS_PORT"/>
+      <secure>1</secure>
+      {{- else }}
+      <tcp_port replace="replace" from_env="CLICKHOUSE_KEEPER_TCP_PORT"/>
+      {{- end }}
+      <server_id from_env="CH_KEEPER_ID"/>
       <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
       <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
       <coordination_settings>
@@ -13,21 +30,83 @@
           <raft_logs_level>warning</raft_logs_level>
       </coordination_settings>
       <raft_configuration>
-{{- range $id, $host := splitList "," .CH_KEEPER_POD_FQDN_LIST }}
+      {{- if $.component.tlsConfig }}
+      <secure>true</secure>
+      {{- end }}
+      {{- range $id, $host := splitList "," .CH_KEEPER_POD_FQDN_LIST }}
         <server>
           <id>{{ $id }}</id>
           <hostname>{{ $host }}</hostname>
-          <port from_env="CLICKHOUSE_KEEPER_RAFT_PORT"></port>
+          {{- if $.component.tlsConfig }}
+          <port replace="replace" from_env="CLICKHOUSE_KEEPER_RAFT_TLS_PORT"/>
+          {{- else }}
+          <port replace="replace" from_env="CLICKHOUSE_KEEPER_RAFT_PORT"/>
+          {{- end }}
         </server>
-{{- end }}
+        {{- end }}
       </raft_configuration>
   </keeper_server>
   <!-- Prometheus metrics -->
   <prometheus>
-      <endpoint>/metrics</endpoint>
-      <port from_env="CLICKHOUSE_METRICS_PORT"></port>
-      <metrics>true</metrics>
-      <events>true</events>
-      <asynchronous_metrics>true</asynchronous_metrics>
+    <endpoint>/metrics</endpoint>
+    <port replace="replace" from_env="CLICKHOUSE_METRICS_PORT"/>
+    <metrics>true</metrics>
+    <events>true</events>
+    <asynchronous_metrics>true</asynchronous_metrics>
   </prometheus>
+  <!-- tls configuration -->
+  {{- if $.component.tlsConfig -}}
+  {{- $CA_FILE := getCAFile -}}
+  {{- $CERT_FILE := getCertFile -}}
+  {{- $KEY_FILE := getKeyFile -}}
+  <protocols>
+    <prometheus_protocol>
+      <type>prometheus</type>
+      <description>prometheus protocol</description>
+    </prometheus_protocol>
+    <prometheus_secure>
+      <type>tls</type>
+      <impl>prometheus_protocol</impl>
+      <description>prometheus over https</description>
+      <certificateFile>{{$CERT_FILE}}</certificateFile>
+      <privateKeyFile>{{$KEY_FILE}}</privateKeyFile>
+    </prometheus_secure>
+  </protocols>
+  <openSSL>
+    <server>
+      <certificateFile>{{$CERT_FILE}}</certificateFile>
+      <privateKeyFile>{{$KEY_FILE}}</privateKeyFile>
+      <verificationMode>relaxed</verificationMode>
+      <caConfig>{{$CA_FILE}}</caConfig>
+      <cacheSessions>true</cacheSessions>
+      <disableProtocols>sslv2,sslv3</disableProtocols>
+      <preferServerCiphers>true</preferServerCiphers>
+    </server>
+    <client>
+      <loadDefaultCAFile>false</loadDefaultCAFile>
+      <certificateFile>{{$CERT_FILE}}</certificateFile>
+      <privateKeyFile>{{$KEY_FILE}}</privateKeyFile>
+      <caConfig>{{$CA_FILE}}</caConfig>
+      <cacheSessions>true</cacheSessions>
+      <disableProtocols>sslv2,sslv3</disableProtocols>
+      <preferServerCiphers>true</preferServerCiphers>
+      <verificationMode>relaxed</verificationMode>
+      <invalidCertificateHandler>
+        <name>RejectCertificateHandler</name>
+      </invalidCertificateHandler>
+    </client>
+  </openSSL>
+  <grpc>
+    <enable_ssl>1</enable_ssl>
+    <ssl_cert_file>{{$CERT_FILE}}</ssl_cert_file>
+    <ssl_key_file>{{$KEY_FILE}}</ssl_key_file>
+    <ssl_require_client_auth>true</ssl_require_client_auth>
+    <ssl_ca_cert_file>{{$CA_FILE}}</ssl_ca_cert_file>
+    <transport_compression_type>none</transport_compression_type>
+    <transport_compression_level>0</transport_compression_level>
+    <max_send_message_size>-1</max_send_message_size>
+    <max_receive_message_size>-1</max_receive_message_size>
+    <verbose_logs>false</verbose_logs>
+  </grpc>
+  {{- end }}
 </clickhouse>
diff --git a/addons/clickhouse/configs/client.xml.tpl b/addons/clickhouse/configs/client.xml.tpl
@@ -0,0 +1,15 @@
+<config>
+  <user>admin</user>
+  <password from_env="CLICKHOUSE_ADMIN_PASSWORD"/>
+  {{- if $.component.tlsConfig -}}
+  {{- $CA_FILE := getCAFile -}}
+  {{- $CERT_FILE := getCertFile -}}
+  {{- $KEY_FILE := getKeyFile }}
+  <secure>true</secure>
+  <openSSL>
+    <client>
+      <caConfig>{{$CA_FILE}}</caConfig>
+    </client>
+  </openSSL>
+  {{- end }}
+</config>