feat: update Clowder to reconcile on changes to non-app secrets/confi…

…gmaps

* Add hashCache to the ClowdEnvironmentReconciliation
* Remove all Secrets/ConfigMaps from Environment at start of reconciliation cycle
* Embue HashObject with the always flag
* Change updateHashCacheForConfigMapAndSecret to try to read in the sec/config and return true if always set
* Make managed_kafka add the secret to the cache at the start of reconciliation
* Update HashCache tests
* Add kuttl tests for managed kafka secrets

cc: @psav

controllers/cloud.redhat.com/clowdenvironment_controller.go

@@ -165,6 +165,7 @@ func (r *ClowdEnvironmentReconciler) Reconcile(ctx context.Context, req ctrl.Req
 		env:       &env,
 		log:       &log,
 		oldStatus: env.Status.DeepCopy(),
+		hashCache: r.HashCache,
 	}
 
 	result, resErr := reconciliation.Reconcile()

controllers/cloud.redhat.com/clowdenvironment_reconciliation.go

@@ -7,6 +7,7 @@ import (
 
 	crd "github.com/RedHatInsights/clowder/apis/cloud.redhat.com/v1alpha1"
 	"github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/clowderconfig"
+	"github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/hashcache"
 	"github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers"
 	rc "github.com/RedHatInsights/rhc-osdk-utils/resourceCache"
 	"github.com/go-logr/logr"
@@ -57,6 +58,7 @@ type ClowdEnvironmentReconciliation struct {
 	env       *crd.ClowdEnvironment
 	log       *logr.Logger
 	oldStatus *crd.ClowdEnvironmentStatus
+	hashCache *hashcache.HashCache
 }
 
 // Returns a list of step methods that should be run during reconciliation
@@ -285,6 +287,8 @@ func (r *ClowdEnvironmentReconciliation) isTargetNamespaceMarkedForDeletion() (c
 }
 
 func (r *ClowdEnvironmentReconciliation) runProviders() (ctrl.Result, error) {
+	r.hashCache.RemoveClowdObjectFromObjects(r.env)
+
 	provider := providers.Provider{
 		Ctx:    r.ctx,
 		Client: r.client,

controllers/cloud.redhat.com/handlers.go

@@ -106,7 +106,15 @@ func (e *enqueueRequestForObjectCustom) updateHashCacheForConfigMapAndSecret(obj
 	switch obj.(type) {
 	case *core.ConfigMap, *core.Secret:
 		if obj.GetAnnotations()[clowderconfig.LoadedConfig.Settings.RestarterAnnotationName] == "true" {
-			return e.hashCache.CreateOrUpdateObject(obj)
+			return e.hashCache.CreateOrUpdateObject(obj, false)
+		} else {
+			hcOjb, err := e.hashCache.Read(obj)
+			if err != nil {
+				return false, err
+			}
+			if hcOjb.Always {
+				return e.hashCache.CreateOrUpdateObject(obj, false)
+			}
 		}
 	}
 	return false, nil

controllers/cloud.redhat.com/hashcache/hashcache.go

@@ -32,6 +32,7 @@ type HashObject struct {
 	Hash      string
 	ClowdApps map[types.NamespacedName]bool
 	ClowdEnvs map[types.NamespacedName]bool
+	Always    bool // Secret/ConfigMap should be always updated
 }
 
 type HashCache struct {
@@ -46,11 +47,12 @@ func NewHashCache() HashCache {
 	}
 }
 
-func NewHashObject(hash string) HashObject {
+func NewHashObject(hash string, always bool) HashObject {
 	return HashObject{
 		Hash:      hash,
 		ClowdApps: map[types.NamespacedName]bool{},
 		ClowdEnvs: map[types.NamespacedName]bool{},
+		Always:    always,
 	}
 }
 
@@ -101,7 +103,9 @@ func (hc *HashCache) RemoveClowdObjectFromObjects(obj client.Object) {
 	}
 }
 
-func (hc *HashCache) CreateOrUpdateObject(obj client.Object) (bool, error) {
+// CreatesOrUpdates HashObject and adding attribute alwaysUpdate.
+// This function returns a boolean indicating whether the hashCache should be updated.
+func (hc *HashCache) CreateOrUpdateObject(obj client.Object, alwaysUpdate bool) (bool, error) {
 	hc.lock.Lock()
 	defer hc.lock.Unlock()
 
@@ -129,7 +133,7 @@ func (hc *HashCache) CreateOrUpdateObject(obj client.Object) (bool, error) {
 	hashObject, ok := hc.data[id]
 
 	if !ok {
-		hashObj := NewHashObject(hash)
+		hashObj := NewHashObject(hash, alwaysUpdate)
 		hc.data[id] = &hashObj
 		return true, nil
 	}
@@ -177,11 +181,6 @@ func (hc *HashCache) GetSuperHashForClowdObject(clowdObj object.ClowdObject) str
 }
 
 func (hc *HashCache) AddClowdObjectToObject(clowdObj object.ClowdObject, obj client.Object) error {
-
-	if obj.GetAnnotations()[clowderconfig.LoadedConfig.Settings.RestarterAnnotationName] != "true" {
-		return nil
-	}
-
 	var oType string
 
 	switch obj.(type) {
@@ -198,6 +197,10 @@ func (hc *HashCache) AddClowdObjectToObject(clowdObj object.ClowdObject, obj cli
 	if !ok {
 		return ItemNotFoundError{item: fmt.Sprintf("%s/%s", id.NN.Name, id.NN.Namespace)}
 	}
+	if obj.GetAnnotations()[clowderconfig.LoadedConfig.Settings.RestarterAnnotationName] != "true" && !hc.data[id].Always {
+		return nil
+	}
+
 	hc.lock.Lock()
 	defer hc.lock.Unlock()
 

controllers/cloud.redhat.com/hashcache/hashcache_test.go

@@ -21,7 +21,7 @@ func TestHashCacheAddItemAndRetrieve(t *testing.T) {
 	}
 
 	hc := NewHashCache()
-	update, err := hc.CreateOrUpdateObject(sec)
+	update, err := hc.CreateOrUpdateObject(sec, false)
 	assert.NoError(t, err)
 	assert.True(t, update)
 	obj, err := hc.Read(sec)
@@ -39,7 +39,7 @@ func TestHashCacheDeleteItem(t *testing.T) {
 	}
 
 	hc := NewHashCache()
-	shouldUpdate, err := hc.CreateOrUpdateObject(sec)
+	shouldUpdate, err := hc.CreateOrUpdateObject(sec, false)
 	assert.True(t, shouldUpdate)
 	assert.NoError(t, err)
 	obj, err := hc.Read(sec)
@@ -63,7 +63,7 @@ func TestHashCacheUpdateItem(t *testing.T) {
 	}
 
 	hc := NewHashCache()
-	_, err := hc.CreateOrUpdateObject(sec)
+	_, err := hc.CreateOrUpdateObject(sec, false)
 	assert.NoError(t, err)
 
 	obj, err := hc.Read(sec)
@@ -75,7 +75,7 @@ func TestHashCacheUpdateItem(t *testing.T) {
 		"test2": []byte("test2"),
 	}
 
-	update, err := hc.CreateOrUpdateObject(sec)
+	update, err := hc.CreateOrUpdateObject(sec, false)
 	assert.NoError(t, err)
 	assert.True(t, update)
 	obj, err = hc.Read(sec)
@@ -120,7 +120,7 @@ func TestHashCacheAddClowdObj(t *testing.T) {
 	}
 
 	hc := NewHashCache()
-	_, err := hc.CreateOrUpdateObject(sec)
+	_, err := hc.CreateOrUpdateObject(sec, false)
 	assert.NoError(t, err)
 
 	err = hc.AddClowdObjectToObject(capp, sec)
@@ -152,7 +152,7 @@ func TestHashCacheDeleteClowdObj(t *testing.T) {
 	}
 
 	hc := NewHashCache()
-	_, err := hc.CreateOrUpdateObject(sec)
+	_, err := hc.CreateOrUpdateObject(sec, false)
 	assert.NoError(t, err)
 
 	err = hc.AddClowdObjectToObject(capp, sec)
@@ -196,15 +196,15 @@ func TestHashCacheSuperCache(t *testing.T) {
 	}
 
 	hc := NewHashCache()
-	_, err := hc.CreateOrUpdateObject(sec)
+	_, err := hc.CreateOrUpdateObject(sec, false)
 	assert.NoError(t, err)
 	err = hc.AddClowdObjectToObject(capp, sec)
 	assert.NoError(t, err)
 	obj, err := hc.Read(sec)
 	assert.NoError(t, err)
 	assert.Contains(t, obj.ClowdApps, clowdObjNamespaceName)
 
-	_, err = hc.CreateOrUpdateObject(sec2)
+	_, err = hc.CreateOrUpdateObject(sec2, false)
 	assert.NoError(t, err)
 	err = hc.AddClowdObjectToObject(capp, sec2)
 	assert.NoError(t, err)

controllers/cloud.redhat.com/providers/kafka/managed.go

@@ -154,6 +154,15 @@ func (k *managedKafkaProvider) getSecret() (*core.Secret, error) {
 		return nil, err
 	}
 
+	_, err = k.HashCache.CreateOrUpdateObject(secret, true)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = k.HashCache.AddClowdObjectToObject(k.Env, secret); err != nil {
+		return nil, err
+	}
+
 	return secret, nil
 }
 

tests/kuttl/test-clowdapp-watcher-kafka-managed-secret/00-install.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: test-clowdapp-watcher-kafka-managed-secret
+spec:
+  finalizers:
+  - kubernetes

tests/kuttl/test-clowdapp-watcher-kafka-managed-secret/01-assert.yaml

@@ -0,0 +1,50 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: puptoo
+  namespace: test-clowdapp-watcher-kafka-managed-secret
+  labels:
+    app: puptoo
+  ownerReferences:
+  - apiVersion: cloud.redhat.com/v1alpha1
+    kind: ClowdApp
+    name: puptoo
+type: Opaque
+data:
+  cdappconfig.json: eyJoYXNoQ2FjaGUiOiJlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1OTRiNzlkNjczZjdlZTI1MjNmMzI2YjdjMTBmYTQyYTU2Y2EyNWY2YzEwM2NmZWM1NGRlY2MxODYxNDRlYzE3ZSIsImthZmthIjp7ImJyb2tlcnMiOlt7ImF1dGh0eXBlIjoic2FzbCIsImNhY2VydCI6InNvbWUtcGVtIiwiaG9zdG5hbWUiOiJrYWZrYS1ob3N0LW5hbWUiLCJwb3J0IjoyNzAxNSwic2FzbCI6eyJwYXNzd29yZCI6ImthZmthLXBhc3N3b3JkIiwic2FzbE1lY2hhbmlzbSI6IlBMQUlOIiwic2VjdXJpdHlQcm90b2NvbCI6IlNBU0xfU1NMIiwidXNlcm5hbWUiOiJrYWZrYS11c2VybmFtZSJ9LCJzZWN1cml0eVByb3RvY29sIjoiU0FTTF9TU0wifV0sInRvcGljcyI6W3sibmFtZSI6InRvcGljT25lIiwicmVxdWVzdGVkTmFtZSI6InRvcGljT25lIn0seyJuYW1lIjoidG9waWNUd28iLCJyZXF1ZXN0ZWROYW1lIjoidG9waWNUd28ifV19LCJsb2dnaW5nIjp7ImNsb3Vkd2F0Y2giOnsiYWNjZXNzS2V5SWQiOiIiLCJsb2dHcm91cCI6IiIsInJlZ2lvbiI6IiIsInNlY3JldEFjY2Vzc0tleSI6IiJ9LCJ0eXBlIjoibnVsbCJ9LCJtZXRhZGF0YSI6eyJkZXBsb3ltZW50cyI6W3siaW1hZ2UiOiJxdWF5LmlvL3BzYXYvY2xvd2Rlci1oZWxsbyIsIm5hbWUiOiJwcm9jZXNzb3IifV0sImVudk5hbWUiOiJ0ZXN0LWNsb3dkYXBwLXdhdGNoZXIta2Fma2EtdXNlci1zZWNyZXQiLCJuYW1lIjoicHVwdG9vIn0sIm1ldHJpY3NQYXRoIjoiL21ldHJpY3MiLCJtZXRyaWNzUG9ydCI6OTAwMCwicHJpdmF0ZVBvcnQiOjEwMDAwLCJwdWJsaWNQb3J0Ijo4MDAwLCJ3ZWJQb3J0Ijo4MDAwfQ==
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: puptoo-processor
+  namespace: test-clowdapp-watcher-kafka-managed-secret
+spec:
+  strategy:
+    type: RollingUpdate
+  template:
+    spec:
+      serviceAccountName: puptoo-processor
+      containers:
+      - env:
+        - name: ENV_VAR_1
+          value: "env_var_1"
+        - name: ACG_CONFIG
+          value: /cdapp/cdappconfig.json
+---
+apiVersion: cloud.redhat.com/v1alpha1
+kind: ClowdEnvironment
+metadata:
+  name: test-clowdapp-watcher-kafka-managed-secret
+status:
+  apps:
+    - name: puptoo
+      deployments:
+      - name: puptoo-processor
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: test-clowdapp-watcher-kafka-managed-secret
+  labels:
+    kubernetes.io/metadata.name: test-clowdapp-watcher-kafka-managed-secret

tests/kuttl/test-clowdapp-watcher-kafka-managed-secret/01-pods.yaml

@@ -0,0 +1,71 @@
+---
+apiVersion: v1
+data:
+  hostname: a2Fma2EtaG9zdC1uYW1l # kafka-host-name
+  port: MjcwMTU= # 27015
+  username: a2Fma2EtdXNlcm5hbWU= # kafka-username
+  password: a2Fma2EtcGFzc3dvcmQ= # kafka-password
+  cacert: c29tZS1wZW0=
+kind: Secret
+metadata:
+  name: managed-secret
+  namespace: test-clowdapp-watcher-kafka-managed-secret
+type: Opaque
+---
+apiVersion: cloud.redhat.com/v1alpha1
+kind: ClowdEnvironment
+metadata:
+  name: test-clowdapp-watcher-kafka-managed-secret
+spec:
+  targetNamespace: test-clowdapp-watcher-kafka-managed-secret
+  providers:
+    web:
+      port: 8000
+      mode: operator
+    metrics:
+      port: 9000
+      mode: operator
+      path: "/metrics"
+    kafka:
+      mode: managed
+      managedSecretRef:
+        name: managed-secret
+        namespace: test-clowdapp-watcher-kafka-managed-secret
+      managedPrefix: ""
+    db:
+      mode: none
+    logging:
+      mode: none
+    objectStore:
+      mode: none
+    inMemoryDb:
+      mode: none
+  resourceDefaults:
+    limits:
+      cpu: 400m
+      memory: 1024Mi
+    requests:
+      cpu: 30m
+      memory: 512Mi
+---
+apiVersion: cloud.redhat.com/v1alpha1
+kind: ClowdApp
+metadata:
+  name: puptoo
+  namespace: test-clowdapp-watcher-kafka-managed-secret
+spec:
+  envName: test-clowdapp-watcher-kafka-managed-secret
+  deployments:
+  - name: processor
+    podSpec:
+      image: quay.io/psav/clowder-hello
+      env: 
+        - name: ENV_VAR_1
+          value: env_var_1
+  kafkaTopics:
+  - replicas: 3
+    partitions: 64
+    topicName: topicOne
+  - replicas: 5
+    partitions: 32
+    topicName: topicTwo

tests/kuttl/test-clowdapp-watcher-kafka-managed-secret/02-json-asserts.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+- script: sleep 5
+- script: kubectl get secret --namespace=test-clowdapp-watcher-kafka-managed-secret puptoo -o json > /tmp/test-clowdapp-watcher-kafka-managed-secret
+- script: jq -r '.data["cdappconfig.json"]' < /tmp/test-clowdapp-watcher-kafka-managed-secret | base64 -d > /tmp/test-clowdapp-watcher-kafka-managed-secret-json
+
+
+- script: jq -r '.kafka.brokers[0].sasl.password == "kafka-password"'  -e < /tmp/test-clowdapp-watcher-kafka-managed-secret-json

tests/kuttl/test-clowdapp-watcher-kafka-managed-secret/03-assert.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: puptoo
+  namespace: test-clowdapp-watcher-kafka-managed-secret
+  labels:
+    app: puptoo
+  ownerReferences:
+  - apiVersion: cloud.redhat.com/v1alpha1
+    kind: ClowdApp
+    name: puptoo
+type: Opaque
+data:
+  cdappconfig.json: eyJoYXNoQ2FjaGUiOiJlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1ZTNlZTVkZTI0MDM4MTFhODI5MzQxMjk0NGZkYzExN2MwMGFlMDk3ZjcxNzZlYjZmZWExNWM0YTgxZjAzYWE0NCIsImthZmthIjp7ImJyb2tlcnMiOlt7ImF1dGh0eXBlIjoic2FzbCIsImNhY2VydCI6InNvbWUtcGVtIiwiaG9zdG5hbWUiOiJrYWZrYS1ob3N0LW5hbWUiLCJwb3J0IjoyNzAxNSwic2FzbCI6eyJwYXNzd29yZCI6ImthZmthLW5ldy1wYXNzd29yZCIsInNhc2xNZWNoYW5pc20iOiJQTEFJTiIsInNlY3VyaXR5UHJvdG9jb2wiOiJTQVNMX1NTTCIsInVzZXJuYW1lIjoia2Fma2EtdXNlcm5hbWUifSwic2VjdXJpdHlQcm90b2NvbCI6IlNBU0xfU1NMIn1dLCJ0b3BpY3MiOlt7Im5hbWUiOiJ0b3BpY09uZSIsInJlcXVlc3RlZE5hbWUiOiJ0b3BpY09uZSJ9LHsibmFtZSI6InRvcGljVHdvIiwicmVxdWVzdGVkTmFtZSI6InRvcGljVHdvIn1dfSwibG9nZ2luZyI6eyJjbG91ZHdhdGNoIjp7ImFjY2Vzc0tleUlkIjoiIiwibG9nR3JvdXAiOiIiLCJyZWdpb24iOiIiLCJzZWNyZXRBY2Nlc3NLZXkiOiIifSwidHlwZSI6Im51bGwifSwibWV0YWRhdGEiOnsiZGVwbG95bWVudHMiOlt7ImltYWdlIjoicXVheS5pby9wc2F2L2Nsb3dkZXItaGVsbG8iLCJuYW1lIjoicHJvY2Vzc29yIn1dLCJlbnZOYW1lIjoidGVzdC1jbG93ZGFwcC13YXRjaGVyLWthZmthLXVzZXItc2VjcmV0IiwibmFtZSI6InB1cHRvbyJ9LCJtZXRyaWNzUGF0aCI6Ii9tZXRyaWNzIiwibWV0cmljc1BvcnQiOjkwMDAsInByaXZhdGVQb3J0IjoxMDAwMCwicHVibGljUG9ydCI6ODAwMCwid2ViUG9ydCI6ODAwMH0=

tests/kuttl/test-clowdapp-watcher-kafka-managed-secret/03-pods.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+data:
+  hostname: a2Fma2EtaG9zdC1uYW1l # kafka-host-name
+  port: MjcwMTU= # 27015
+  username: a2Fma2EtdXNlcm5hbWU= # kafka-username
+  password: a2Fma2EtbmV3LXBhc3N3b3Jk # kafka-new-password
+  cacert: c29tZS1wZW0=
+kind: Secret
+metadata:
+  name: managed-secret
+  namespace: test-clowdapp-watcher-kafka-managed-secret
+type: Opaque

tests/kuttl/test-clowdapp-watcher-kafka-managed-secret/04-json-asserts.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+- script: sleep 5
+- script: kubectl get secret --namespace=test-clowdapp-watcher-kafka-managed-secret puptoo -o json > /tmp/test-clowdapp-watcher-kafka-managed-secret2
+- script: jq -r '.data["cdappconfig.json"]' < /tmp/test-clowdapp-watcher-kafka-managed-secret2 | base64 -d > /tmp/test-clowdapp-watcher-kafka-managed-secret2-json
+
+- script: jq -r '.kafka.brokers[0].sasl.password == "kafka-new-password"'  -e < /tmp/test-clowdapp-watcher-kafka-managed-secret2-json

tests/kuttl/test-clowdapp-watcher-kafka-managed-secret/05-delete.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+delete:
+- apiVersion: v1
+  kind: Namespace
+  name: test-clowdapp-watcher-kafka-managed-secret
+- apiVersion: cloud.redhat.com/v1alpha1
+  kind: ClowdEnvironment
+  name: test-clowdapp-watcher-kafka-managed-secret