Merge pull request #76 from Altinity/release-0.2.1

Release 0.2.1

cmd/clickhouse-operator/app/clickhouse_operator.go

@@ -37,11 +37,13 @@ import (
 	kuberest "k8s.io/client-go/rest"
 	kubeclientcmd "k8s.io/client-go/tools/clientcmd"
 
+	chopmodels "github.com/altinity/clickhouse-operator/pkg/models"
+
 	"github.com/golang/glog"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-// Version defines current build version
+// Version defines current build versionRequest
 const Version = "0.2.0-beta"
 
 // Prometheus exporter defaults
@@ -52,6 +54,9 @@ const (
 )
 
 var (
+	// versionRequest defines versionRequest request
+	versionRequest bool
+
 	// chopConfigFile defines path to clickhouse-operator config file to be used
 	chopConfigFile string
 
@@ -66,6 +71,7 @@ var (
 )
 
 func init() {
+	flag.BoolVar(&versionRequest, "version", false, "Display versionRequest and exit")
 	flag.StringVar(&chopConfigFile, "config", "", "Path to clickhouse-operator config file.")
 	flag.StringVar(&kubeConfigFile, "kube-config", "", "Path to kubernetes config file. Only required if called outside of the cluster.")
 	flag.StringVar(&masterURL, "master", "", "The address of the Kubernetes API server. Only required if called outside of the cluster and not being specified in kube config file.")
@@ -123,13 +129,20 @@ func createClientsets(config *kuberest.Config) (*kube.Clientset, *chopclientset.
 
 // Run is an entry point of the application
 func Run() {
-	glog.V(1).Infof("Starting clickhouse-operator version '%s'\n", Version)
+	if versionRequest {
+		fmt.Printf("%s\n", Version)
+		os.Exit(0)
+	}
+
+	glog.V(1).Infof("Starting clickhouse-operator versionRequest '%s'\n", Version)
 	chopConfig, err := config.GetConfig(chopConfigFile)
 	if err != nil {
 		glog.Fatalf("Unable to build config file %v\n", err)
 		os.Exit(1)
 	}
 
+	chopmodels.SetAppVersion(Version)
+
 	// Initializing Prometheus Metrics Exporter
 	glog.V(1).Infof("Starting metrics exporter at '%s%s'\n", metricsEP, metricsPath)
 	metricsExporter := chopmetrics.CreateExporter()
@@ -166,6 +179,7 @@ func Run() {
 		kubeClient,
 		chopInformerFactory.Clickhouse().V1().ClickHouseInstallations(),
 		kubeInformerFactory.Core().V1().Services(),
+		kubeInformerFactory.Core().V1().Endpoints(),
 		kubeInformerFactory.Core().V1().ConfigMaps(),
 		kubeInformerFactory.Apps().V1().StatefulSets(),
 		kubeInformerFactory.Core().V1().Pods(),

docs/examples/02-standard-layout-01-1shard-1repl-simple-persistent-volume.yaml

@@ -17,8 +17,6 @@ spec:
     volumeClaimTemplates:
       - name: volumeclaim-template
         persistentVolumeClaim:
-          metadata:
-            name: USE_DEFAULT_NAME
           spec:
             accessModes:
               - ReadWriteOnce

docs/examples/chi-example-02-default-pv.yaml → docs/examples/02-standard-layout-02-multicluster-simple-persistent-volume.yaml

@@ -1,11 +1,11 @@
 apiVersion: "clickhouse.altinity.com/v1"
 kind: "ClickHouseInstallation"
 metadata:
-  name: "test2-replication"
+  name: "syandard-02-multicluster"
 spec:
   defaults:
     deployment:
-      volumeClaimTemplate: default
+      volumeClaimTemplate: auto-simple-vc
   configuration:
     clusters:
       - name: "sharded-replicated"
@@ -23,10 +23,8 @@ spec:
           replicasCount: 2
   templates:
     volumeClaimTemplates:
-    - name: default
+    - name: auto-simple-vc
       persistentVolumeClaim:
-        metadata:
-          name: USE_DEFAULT_NAME
         spec:
           accessModes:
           - ReadWriteOnce

docs/examples/03-settings-01.yaml

@@ -0,0 +1,37 @@
+apiVersion: "clickhouse.altinity.com/v1"
+kind: "ClickHouseInstallation"
+metadata:
+  name: "settings-01"
+spec:
+  configuration:
+    users:
+      test/password: qwerty
+      test/networks/ip:
+        - "::1"
+        - "127.0.0.1"
+      test/profile: test_profile
+      test/quota: test_quota
+      test/allow_databases/database:
+        - "dbname1"
+        - "dbname2"
+        - "dbname3"
+      admin/password_sha256_hex: 8bd66e4932b4968ec111da24d7e42d399a05cb90bf96f587c3fa191c56c401f8
+      admin/networks/ip: 0.0.0.0/0
+      admin/profile: default
+      admin/quota: default
+      readonly/profile: readonly
+      readonly/quota: default
+    profiles:
+      test_profile/max_memory_usage: "1000000000"
+      test_profile/readonly: "1"
+      readonly/readonly: "1"
+    quotas:
+      test_quota/interval/duration: "3600"
+    settings:
+      compression/case/method: zstd
+    clusters:
+      - name: "standard"
+        layout:
+          type: Standard
+          shardsCount: 1
+          replicasCount: 1

docs/quick-start.md

@@ -151,8 +151,6 @@ spec:
     volumeClaimTemplates:
       - name: volumeclaim-template
         persistentVolumeClaim:
-          metadata:
-            name: USE_DEFAULT_NAME
           spec:
             accessModes:
               - ReadWriteOnce
@@ -168,7 +166,7 @@ Let's install more complex example with:
 1. Pod template
 1. VolumeClaim template
 
-Manifest is [available in examples](./examples/02-standard-layout-02-1shard-1repl-deployment-persistent-volume.yaml)
+Manifest is [available in examples](./examples/02-standard-layout-03-1shard-1repl-deployment-persistent-volume.yaml)
 
 ```yaml
 apiVersion: "clickhouse.altinity.com/v1"
@@ -186,6 +184,7 @@ spec:
           type: Standard
           shardsCount: 1
           replicasCount: 1
+
   templates:
     podTemplates:
       - name: clickhouse-with-volume-template
@@ -202,6 +201,7 @@ spec:
             volumeMounts:
               - name: clickhouse-storage
                 mountPath: /var/lib/clickhouse
+
     volumeClaimTemplates:
       - name: clickhouse-storage-template
         persistentVolumeClaim:
@@ -216,11 +216,12 @@ spec:
 ```
 
 ## Custom Deployment with Specific ClickHouse Configuration
+More extended settings example is [available here](./examples/03-settings-01.yaml)
 ```yaml
 apiVersion: "clickhouse.altinity.com/v1"
 kind: "ClickHouseInstallation"
 metadata:
-  name: "test4"
+  name: "test-custom-settings"
 spec:
   configuration:
     users:

docs/storage.md

@@ -2,7 +2,7 @@
 
 Examples are availabe in [examples](./examples) folder:
 1. [Simple Persistent Volume](./examples/02-standard-layout-01-1shard-1repl-simple-persistent-volume.yaml)
-1. [Simple Deployment with Persistent Volume](./examples/02-standard-layout-02-1shard-1repl-deployment-persistent-volume.yaml)
+1. [Deployment with Persistent Volume](./examples/02-standard-layout-03-1shard-1repl-deployment-persistent-volume.yaml)
 1. AWS-based cluster with data replication and Persistent Volumes [minimal](./examples/04-zookeeper-replication-03-minimal-AWS-persistent-volume.yaml) and [medium](./examples/04-zookeeper-replication-04-medium-AWS-persistent-volume.yaml) Zookeeper installations
 
 ## Persistent Volumes

manifests/dev/dev-config.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+DEV_NAMESPACE="dev"
+
+#INSTALL_FROM_ALTINITY_RELEASE_DOCKERHUB="yes"
+INSTALL_FROM_ALTINITY_RELEASE_DOCKERHUB="no"
+
+#INSTALL_FROM_PERSONAL_DEVELOP_DOCKERHUB="yes"
+INSTALL_FROM_PERSONAL_DEVELOP_DOCKERHUB="no"
+
+# In case both INSTALL_* options are "no" we are going to run operator manually, not from dockerhub

manifests/dev/dev-delete.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-DEV_NAMESPACE="dev"
+. ./dev-config.sh
 
 if kubectl get namespace "${DEV_NAMESPACE}"; then
     echo "Delete ${DEV_NAMESPACE} namespace"

manifests/dev/dev-install.sh

@@ -1,17 +1,24 @@
 #!/bin/bash
 
-DEV_NAMESPACE="dev"
+. ./dev-config.sh
 
 echo "Create ${DEV_NAMESPACE} namespace"
 kubectl create namespace "${DEV_NAMESPACE}"
 
-# Full dev install in k8s
-#kubectl -n "${DEV_NAMESPACE}" apply -f ./clickhouse-operator-install.yaml
+if [[ ${INSTALL_FROM_ALTINITY_RELEASE_DOCKERHUB} == "yes" ]]; then
+    # Full dev install in k8s
+    kubectl -n "${DEV_NAMESPACE}" apply -f ./clickhouse-operator-install.yaml
 
-# Partial dev install
+    # Installation done
+    exit $?
+fi
+
+# Dev install from all components
 echo "Install operator requirements"
 kubectl -n "${DEV_NAMESPACE}" apply -f ./custom-resource-definition.yaml
 kubectl -n "${DEV_NAMESPACE}" apply -f ./rbac-service.yaml
 
-# Install operator from Docker Registry (dockerhub or whatever)
-kubectl -n "${DEV_NAMESPACE}" apply -f ./deployment-dockerhub-sunsingerus-dev.yaml
+if [[ ${INSTALL_FROM_PERSONAL_DEVELOP_DOCKERHUB} == "yes" ]]; then
+    # Install operator from Docker Registry (dockerhub or whatever)
+    kubectl -n "${DEV_NAMESPACE}" apply -f ./deployment-dockerhub-sunsingerus-dev.yaml
+fi

manifests/dev/dev-reset.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
-echo "Reset dev env"
+. ./dev-config.sh
+
+echo "Reset dev env  via ${DEV_NAMESPACE} namespace"
 ./dev-delete.sh && ./dev-install.sh
 

manifests/dev/dev-show.sh

@@ -1,22 +1,24 @@
 #!/bin/bash
 
+. ./dev-config.sh
+
 echo "=== Dev Pods ==="
-kubectl -n dev -o wide get pod
+kubectl -n "${DEV_NAMESPACE}" -o wide get pod
 
 echo "=== Dev Services ==="
-kubectl -n dev -o wide get service
+kubectl -n "${DEV_NAMESPACE}" -o wide get service
 
 echo "=== Dev Statefule Sets ==="
-kubectl -n dev -o wide get statefulset
+kubectl -n "${DEV_NAMESPACE}" -o wide get statefulset
 
 echo "=== Dev Config Maps ==="
-kubectl -n dev -o wide get configmap
+kubectl -n "${DEV_NAMESPACE}" -o wide get configmap
 
 echo "=== Dev PVC ==="
 kubectl get storageclasses
 
 echo "=== Dev PVC ==="
-kubectl -n dev -o wide get persistentvolumeclaims
+kubectl -n "${DEV_NAMESPACE}" -o wide get persistentvolumeclaims
 
 echo "=== Dev PV ==="
-kubectl -n dev -o wide get persistentvolume
+kubectl -n "${DEV_NAMESPACE}" -o wide get persistentvolume

manifests/dev/dev-watch.sh

@@ -1,3 +1,5 @@
 #!/bin/bash
 
-watch -n1 "kubectl -n dev get all,configmap"
+. ./dev-config.sh
+
+watch -n1 "kubectl -n ${DEV_NAMESPACE} get all,configmap,endpoints"

pkg/apis/clickhouse.altinity.com/v1/type_chi.go

@@ -221,3 +221,14 @@ func (chi *ClickHouseInstallation) MergeFrom(from *ClickHouseInstallation) {
 	// Copy Status for now
 	chi.Status = from.Status
 }
+
+func (chi *ClickHouseInstallation) FindCluster(name string) *ChiCluster {
+	var cluster *ChiCluster
+	chi.WalkClusters(func(c *ChiCluster) error {
+		if c.Name == name {
+			cluster = c
+		}
+		return nil
+	})
+	return cluster
+}

pkg/apis/clickhouse.altinity.com/v1/type_configuration.go

@@ -30,24 +30,22 @@ func (configuration *ChiConfiguration) MergeFrom(from *ChiConfiguration) {
 }
 
 // mapMergeFrom merges into `dst` non-empty new-key-values from `src` in case no such `key` already in `src`
-func mapMergeFrom(dst, src *map[string]string) {
+func mapMergeFrom(dst, src *map[string]interface{}) {
 	if (src == nil) || (*src == nil) {
 		return
 	}
 
 	if *dst == nil {
-		*dst = make(map[string]string)
+		*dst = make(map[string]interface{})
 	}
 
 	for key, value := range *src {
 		if _, ok := (*dst)[key]; ok {
-			// Such a key already in dst
+			// Such key already exists in dst
 			continue
 		}
 
 		// No such a key in dst
-		if len(value) > 0 {
-			(*dst)[key] = value
-		}
+		(*dst)[key] = value
 	}
 }