Use uuid of ClowdEnvironment to generate unique kafka(connect) cluste…

…r name (#392) * Use uuid of ClowdEnvironment to generate unique kafka(connect) cluster name * Shorten names of kafka tests to stay under label length limits * Fix bootstrapServers assertion
RedHatInsights · Jul 15, 2021 · 353cf66 · 353cf66
1 parent 8b9a8c4
commit 353cf66
Show file tree

Hide file tree

Showing 15 changed files with 59 additions and 41 deletions.
diff --git a/apis/cloud.redhat.com/v1alpha1/clowdenvironment_types.go b/apis/cloud.redhat.com/v1alpha1/clowdenvironment_types.go
@@ -91,7 +91,7 @@ type KafkaMode string
 
 // KafkaClusterConfig defines options related to the Kafka cluster managed/monitored by Clowder
 type KafkaClusterConfig struct {
-	// Defines the kafka cluster name (default: name of ClowdEnvironment)
+	// Defines the kafka cluster name (default: <ClowdEnvironment Name>-<UID>)
 	Name string `json:"name,omitempty"`
 
 	// The namespace the kafka cluster is expected to reside in (default: the environment's targetNamespace)
@@ -124,7 +124,7 @@ type KafkaClusterConfig struct {
 
 // KafkaConnectClusterConfig defines options related to the Kafka Connect cluster managed/monitored by Clowder
 type KafkaConnectClusterConfig struct {
-	// Defines the kafka connect cluster name (default: '<kafka cluster's name>-connect')
+	// Defines the kafka connect cluster name (default: <kafka cluster's name>)
 	Name string `json:"name,omitempty"`
 
 	// The namespace the kafka connect cluster is expected to reside in (default: the kafka cluster's namespace)

diff --git a/...fka-provisioner-ephemeral/00-install.yaml → ...t-kafka-strimzi-ephemeral/00-install.yaml b/...fka-provisioner-ephemeral/00-install.yaml → ...t-kafka-strimzi-ephemeral/00-install.yaml
@@ -1,23 +1,23 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: test-kafka-strimzi-kafka-provisioner-ephemeral
+  name: test-kafka-strimzi-ephemeral
 spec:
   finalizers:
   - kubernetes
 ---
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: test-kafka-strimzi-kafka-provisioner-ephemeral-kafka
+  name: test-kafka-strimzi-ephemeral-kafka
 spec:
   finalizers:
   - kubernetes
 ---
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: test-kafka-strimzi-kafka-provisioner-ephemeral-kafka-connect
+  name: test-kafka-strimzi-ephemeral-connect
 spec:
   finalizers:
   - kubernetes
diff --git a/...afka-provisioner-ephemeral/01-assert.yaml → ...st-kafka-strimzi-ephemeral/01-assert.yaml b/...afka-provisioner-ephemeral/01-assert.yaml → ...st-kafka-strimzi-ephemeral/01-assert.yaml
@@ -2,12 +2,12 @@
 apiVersion: kafka.strimzi.io/v1beta1
 kind: Kafka
 metadata:
-  name: my-ephemeral-cluster
-  namespace: test-kafka-strimzi-kafka-provisioner-ephemeral-kafka
+  # name is asserted in json asserts
+  namespace: test-kafka-strimzi-ephemeral-kafka
   ownerReferences:
   - apiVersion: cloud.redhat.com/v1alpha1
     kind: ClowdEnvironment
-    name: env-test-kafka-strimzi-kafka-provisioner-ephemeral
+    name: env-test-kafka-strimzi-ephemeral
 spec:
   kafka:
     version: "2.7.0"
@@ -29,14 +29,14 @@ spec:
 apiVersion: kafka.strimzi.io/v1beta1
 kind: KafkaConnect
 metadata:
-  name: my-other-connect-cluster
-  namespace: test-kafka-strimzi-kafka-provisioner-ephemeral-kafka-connect
+  # name is asserted in json asserts
+  namespace: test-kafka-strimzi-ephemeral-connect
   annotations:
     strimzi.io/use-connector-resources: 'true'
 spec:
   version: "2.7.0"
   image: "quay.io/cloudservices/xjoin-kafka-connect-strimzi:latest"
-  bootstrapServers: my-ephemeral-cluster-kafka-bootstrap.test-kafka-strimzi-kafka-provisioner-ephemeral-kafka.svc:9092
+  # bootstrapServers is asserted in json asserts
   config:
     group.id: connect-cluster
     offset.storage.topic: connect-cluster-offsets

diff --git a/...-kafka-provisioner-ephemeral/01-pods.yaml → ...test-kafka-strimzi-ephemeral/01-pods.yaml b/...-kafka-provisioner-ephemeral/01-pods.yaml → ...test-kafka-strimzi-ephemeral/01-pods.yaml
@@ -2,9 +2,9 @@
 apiVersion: cloud.redhat.com/v1alpha1
 kind: ClowdEnvironment
 metadata:
-  name: env-test-kafka-strimzi-kafka-provisioner-ephemeral
+  name: env-test-kafka-strimzi-ephemeral
 spec:
-  targetNamespace: test-kafka-strimzi-kafka-provisioner-ephemeral
+  targetNamespace: test-kafka-strimzi-ephemeral
   providers:
     web:
       port: 8000
@@ -15,11 +15,9 @@ spec:
       path: "/metrics"
     kafka:
       cluster:
-        namespace: test-kafka-strimzi-kafka-provisioner-ephemeral-kafka
-        name: my-ephemeral-cluster
+        namespace: test-kafka-strimzi-ephemeral-kafka
       connect:
-        name: my-other-connect-cluster
-        namespace: test-kafka-strimzi-kafka-provisioner-ephemeral-kafka-connect
+        namespace: test-kafka-strimzi-ephemeral-connect
       mode: operator
       pvc: false
       enableLegacyStrimzi: true

diff --git a/bundle/tests/scorecard/kuttl/test-kafka-strimzi-ephemeral/02-json-asserts.yaml b/bundle/tests/scorecard/kuttl/test-kafka-strimzi-ephemeral/02-json-asserts.yaml
@@ -0,0 +1,11 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+- script: sleep 5
+# assert that kafka cluster name matches expected
+- script: ENV_UID=$(kubectl get env env-test-kafka-strimzi-ephemeral -o jsonpath={.metadata.uid} | cut -f1 -d'-'); CLUSTER_NAME=$(kubectl get kafka -n test-kafka-strimzi-ephemeral-kafka -o jsonpath='{.items[0].metadata.name}'); [[ "$CLUSTER_NAME" == "env-test-kafka-strimzi-ephemeral-$ENV_UID" ]]
+# assert that kafka connect cluster name matches expected
+- script: ENV_UID=$(kubectl get env env-test-kafka-strimzi-ephemeral -o jsonpath={.metadata.uid} | cut -f1 -d'-'); CLUSTER_NAME=$(kubectl get kafkaconnect -n test-kafka-strimzi-ephemeral-connect -o jsonpath='{.items[0].metadata.name}'); [[ "$CLUSTER_NAME" == "env-test-kafka-strimzi-ephemeral-$ENV_UID" ]]
+# assert that kafka connect's bootstrap servers matches kafka cluster hostname
+- script: KAFKA_CLUSTER_NAME=$(kubectl get kafka -n test-kafka-strimzi-ephemeral-kafka -o jsonpath='{.items[0].metadata.name}'); CONNECT_BOOTSTRAP_SERVERS=$(kubectl get kafkaconnect -n test-kafka-strimzi-ephemeral-connect -o jsonpath='{.items[0].spec.bootstrapServers}'); [[ "$CONNECT_BOOTSTRAP_SERVERS" == "$KAFKA_CLUSTER_NAME-kafka-bootstrap.test-kafka-strimzi-ephemeral-kafka.svc:9092" ]]
diff --git a/...afka-provisioner-ephemeral/02-delete.yaml → ...st-kafka-strimzi-ephemeral/03-delete.yaml b/...afka-provisioner-ephemeral/02-delete.yaml → ...st-kafka-strimzi-ephemeral/03-delete.yaml
@@ -4,16 +4,16 @@ kind: TestStep
 delete:
 - apiVersion: cloud.redhat.com/v1alpha1
   kind: ClowdEnvironment
-  name: env-test-kafka-strimzi-kafka-provisioner-ephemeral
+  name: env-test-kafka-strimzi-ephemeral
 - apiVersion: cloud.redhat.com/v1alpha1
   kind: ClowdEnvironment
   name: env-test-kafka-strimzi-kafka-provisioner-pvc
 - apiVersion: v1
   kind: Namespace
-  name: test-kafka-strimzi-kafka-provisioner-kafka
+  name: test-kafka-strimzi-ephemeral
 - apiVersion: v1
   kind: Namespace
-  name: test-kafka-strimzi-kafka-provisioner
+  name: test-kafka-strimzi-ephemeral-kafka
 - apiVersion: v1
   kind: Namespace
-  name: test-kafka-strimzi-kafka-provisioner-kafka-connect
+  name: test-kafka-strimzi-ephemeral-connect
diff --git a/...mzi-kafka-provisioner-pvc/00-install.yaml → ...tl/test-kafka-strimzi-pvc/00-install.yaml b/...mzi-kafka-provisioner-pvc/00-install.yaml → ...tl/test-kafka-strimzi-pvc/00-install.yaml
@@ -1,15 +1,15 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: test-kafka-strimzi-kafka-provisioner-pvc
+  name: test-kafka-strimzi-pvc
 spec:
   finalizers:
   - kubernetes
 ---
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: test-kafka-strimzi-kafka-provisioner-pvc-kafka
+  name: test-kafka-strimzi-pvc-kafka
 spec:
   finalizers:
   - kubernetes
diff --git a/...imzi-kafka-provisioner-pvc/01-assert.yaml → ...ttl/test-kafka-strimzi-pvc/01-assert.yaml b/...imzi-kafka-provisioner-pvc/01-assert.yaml → ...ttl/test-kafka-strimzi-pvc/01-assert.yaml
@@ -3,11 +3,11 @@ apiVersion: kafka.strimzi.io/v1beta1
 kind: Kafka
 metadata:
   name: my-pvc-cluster
-  namespace: test-kafka-strimzi-kafka-provisioner-pvc-kafka
+  namespace: test-kafka-strimzi-pvc-kafka
   ownerReferences:
   - apiVersion: cloud.redhat.com/v1alpha1
     kind: ClowdEnvironment
-    name: env-test-kafka-strimzi-kafka-provisioner-pvc
+    name: env-test-kafka-strimzi-pvc
 spec:
   kafka:
     version: "2.7.0"
@@ -33,14 +33,14 @@ spec:
 apiVersion: kafka.strimzi.io/v1beta1
 kind: KafkaConnect
 metadata:
-  name: my-pvc-cluster-connect
-  namespace: test-kafka-strimzi-kafka-provisioner-pvc-kafka
+  name: my-pvc-cluster
+  namespace: test-kafka-strimzi-pvc-kafka
   annotations:
     strimzi.io/use-connector-resources: 'true'
 spec:
   version: "2.7.0"
   image: "quay.io/cloudservices/xjoin-kafka-connect-strimzi:latest"
-  bootstrapServers: my-pvc-cluster-kafka-bootstrap.test-kafka-strimzi-kafka-provisioner-pvc-kafka.svc:9092
+  bootstrapServers: my-pvc-cluster-kafka-bootstrap.test-kafka-strimzi-pvc-kafka.svc:9092
   config:
     group.id: connect-cluster
     offset.storage.topic: connect-cluster-offsets

diff --git a/...trimzi-kafka-provisioner-pvc/01-pods.yaml → ...kuttl/test-kafka-strimzi-pvc/01-pods.yaml b/...trimzi-kafka-provisioner-pvc/01-pods.yaml → ...kuttl/test-kafka-strimzi-pvc/01-pods.yaml
@@ -2,9 +2,9 @@
 apiVersion: cloud.redhat.com/v1alpha1
 kind: ClowdEnvironment
 metadata:
-  name: env-test-kafka-strimzi-kafka-provisioner-pvc
+  name: env-test-kafka-strimzi-pvc
 spec:
-  targetNamespace: test-kafka-strimzi-kafka-provisioner-pvc
+  targetNamespace: test-kafka-strimzi-pvc
   providers:
     web:
       port: 8000
@@ -15,7 +15,7 @@ spec:
       path: "/metrics"
     kafka:
       cluster:
-        namespace: test-kafka-strimzi-kafka-provisioner-pvc-kafka
+        namespace: test-kafka-strimzi-pvc-kafka
         name: my-pvc-cluster
         provisionCluster: true
         storageSize: 100Mi

diff --git a/...imzi-kafka-provisioner-pvc/02-delete.yaml → ...ttl/test-kafka-strimzi-pvc/02-delete.yaml b/...imzi-kafka-provisioner-pvc/02-delete.yaml → ...ttl/test-kafka-strimzi-pvc/02-delete.yaml
@@ -4,10 +4,10 @@ kind: TestStep
 delete:
 - apiVersion: cloud.redhat.com/v1alpha1
   kind: ClowdEnvironment
-  name: env-test-kafka-strimzi-kafka-provisioner-pvc
+  name: env-test-kafka-strimzi-pvc
 - apiVersion: v1
   kind: Namespace
-  name: test-kafka-strimzi-kafka-provisioner-pvc-kafka
+  name: test-kafka-strimzi-pvc-kafka
 - apiVersion: v1
   kind: Namespace
-  name: test-kafka-strimzi-kafka-provisioner-pvc
+  name: test-kafka-strimzi-pvc
diff --git a/bundle/tests/scorecard/kuttl/test-kafka-strimzi-topic-auth/01-pods.yaml b/bundle/tests/scorecard/kuttl/test-kafka-strimzi-topic-auth/01-pods.yaml
@@ -15,6 +15,7 @@ spec:
       path: "/metrics"
     kafka:
       cluster:
+        name: test-kafka-strimzi-topic-auth
         namespace: test-kafka-strimzi-topic-auth-kafka
       mode: operator
     db:

diff --git a/config/crd/bases/cloud.redhat.com_clowdenvironments.yaml b/config/crd/bases/cloud.redhat.com_clowdenvironments.yaml
@@ -197,7 +197,7 @@ spec:
                             type: object
                           name:
                             description: 'Defines the kafka cluster name (default:
-                              name of ClowdEnvironment)'
+                              <ClowdEnvironment Name>-<UID>)'
                             type: string
                           namespace:
                             description: 'The namespace the kafka cluster is expected
@@ -248,7 +248,7 @@ spec:
                             type: string
                           name:
                             description: 'Defines the kafka connect cluster name (default:
-                              ''<kafka cluster''s name>-connect'')'
+                              <kafka cluster''s name>)'
                             type: string
                           namespace:
                             description: 'The namespace the kafka connect cluster

diff --git a/controllers/cloud.redhat.com/providers/kafka/provider.go b/controllers/cloud.redhat.com/providers/kafka/provider.go
@@ -2,6 +2,7 @@ package kafka
 
 import (
 	"fmt"
+	"strings"
 
 	crd "cloud.redhat.com/clowder/v2/apis/cloud.redhat.com/v1alpha1"
 	cyndi "cloud.redhat.com/clowder/v2/apis/cyndi-operator/v1alpha1"
@@ -50,7 +51,14 @@ func getKafkaUsername(env *crd.ClowdEnvironment, app *crd.ClowdApp) string {
 
 func getKafkaName(e *crd.ClowdEnvironment) string {
 	if e.Spec.Providers.Kafka.Cluster.Name == "" {
-		return e.Name
+		// generate a unique name based on the ClowdEnvironment's UID
+
+		// convert e.UID (which is a apimachinery types.UID) to string
+		// types.UID is a string alias so this should not fail...
+		uidString := string(e.UID)
+
+		// append the initial portion of the UUID onto the kafka cluster's name
+		return fmt.Sprintf("%s-%s", e.Name, strings.Split(uidString, "-")[0])
 	}
 	return e.Spec.Providers.Kafka.Cluster.Name
 }
@@ -71,7 +79,7 @@ func getConnectNamespace(env *crd.ClowdEnvironment) string {
 
 func getConnectClusterName(env *crd.ClowdEnvironment) string {
 	if env.Spec.Providers.Kafka.Connect.Name == "" {
-		return fmt.Sprintf("%s-connect", getKafkaName(env))
+		return getKafkaName(env)
 	}
 	return env.Spec.Providers.Kafka.Connect.Name
 }

diff --git a/controllers/cloud.redhat.com/suite_test.go b/controllers/cloud.redhat.com/suite_test.go
@@ -187,7 +187,7 @@ func applyKafkaStatus(t *testing.T, ch chan int) {
 		// set a mock status on strimzi KafkaConnect cluster
 		connectCluster := strimzi.KafkaConnect{}
 		nn := types.NamespacedName{
-			Name:      "kafka-connect",
+			Name:      "kafka",
 			Namespace: "kafka",
 		}
 		err = k8sClient.Get(ctx, nn, &connectCluster)

diff --git a/docs/antora/modules/ROOT/pages/api_reference.adoc b/docs/antora/modules/ROOT/pages/api_reference.adoc
@@ -503,7 +503,7 @@ KafkaClusterConfig defines options related to the Kafka cluster managed/monitore
 [cols="25a,75a", options="header"]
 |===
 | Field | Description
-| *`name`* __string__ | Defines the kafka cluster name (default: name of ClowdEnvironment)
+| *`name`* __string__ | Defines the kafka cluster name (default: <ClowdEnvironment Name>-<UID>)
 | *`namespace`* __string__ | The namespace the kafka cluster is expected to reside in (default: the environment's targetNamespace)
 | *`replicas`* __integer__ | The requested number of replicas for kafka/zookeeper. If unset, default is '1'
 | *`storageSize`* __string__ | Persistent volume storage size. If unset, default is '1Gi' Only applies when KafkaConfig.PVC is set to 'true'
@@ -555,7 +555,7 @@ KafkaConnectClusterConfig defines options related to the Kafka Connect cluster m
 [cols="25a,75a", options="header"]
 |===
 | Field | Description
-| *`name`* __string__ | Defines the kafka connect cluster name (default: '<kafka cluster's name>-connect')
+| *`name`* __string__ | Defines the kafka connect cluster name (default: <kafka cluster's name>)
 | *`namespace`* __string__ | The namespace the kafka connect cluster is expected to reside in (default: the kafka cluster's namespace)
 | *`replicas`* __integer__ | The requested number of replicas for kafka connect. If unset, default is '1'
 | *`version`* __string__ | Version. If unset, default is '2.5.0'