[Feat] : support ignore specify annotation key in statefulset (#666)

* Support ignore specify annotation key in statefulset Signed-off-by: drivebyer <[email protected]> * add e2e Signed-off-by: drivebyer <[email protected]> * add test path Signed-off-by: drivebyer <[email protected]> * [DEBUG] ignore test dir Signed-off-by: drivebyer <[email protected]> * remove comment Signed-off-by: drivebyer <[email protected]> * remove assert file, which already assert in setup directory Signed-off-by: drivebyer <[email protected]> * add test dirs Signed-off-by: drivebyer <[email protected]> * remove exporter Signed-off-by: drivebyer <[email protected]> * add error assert Signed-off-by: drivebyer <[email protected]> * add more assert Signed-off-by: drivebyer <[email protected]> * add more assert Signed-off-by: drivebyer <[email protected]> --------- Signed-off-by: drivebyer <[email protected]> Signed-off-by: drivebyer <[email protected]>
OT-CONTAINER-KIT · Oct 10, 2023 · f5c68da · f5c68da
1 parent aa02370
commit f5c68da
Show file tree

Hide file tree

Showing 27 changed files with 461 additions and 14 deletions.
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -16,6 +16,7 @@ jobs:
       matrix:
         testpath:
         - ./tests/e2e/v1beta2/setup
+        - ./tests/e2e/v1beta2/ignore-annots
 
     steps:
     - name: Checkout code

diff --git a/api/common_types.go b/api/common_types.go
@@ -15,6 +15,7 @@ type KubernetesConfig struct {
 	ImagePullSecrets       *[]corev1.LocalObjectReference   `json:"imagePullSecrets,omitempty"`
 	UpdateStrategy         appsv1.StatefulSetUpdateStrategy `json:"updateStrategy,omitempty"`
 	Service                *ServiceConfig                   `json:"service,omitempty"`
+	IgnoreAnnotations      []string                         `json:"ignoreAnnotations,omitempty"`
 }
 
 // ServiceConfig define the type of service to be created and its annotations

diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redis.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redis.yaml
@@ -932,6 +932,10 @@ spec:
                 description: KubernetesConfig will be the JSON struct for Basic Redis
                   Config
                 properties:
+                  ignoreAnnotations:
+                    items:
+                      type: string
+                    type: array
                   image:
                     type: string
                   imagePullPolicy:
@@ -5100,6 +5104,10 @@ spec:
                 description: KubernetesConfig will be the JSON struct for Basic Redis
                   Config
                 properties:
+                  ignoreAnnotations:
+                    items:
+                      type: string
+                    type: array
                   image:
                     type: string
                   imagePullPolicy:

diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml
@@ -138,6 +138,10 @@ spec:
                 description: KubernetesConfig will be the JSON struct for Basic Redis
                   Config
                 properties:
+                  ignoreAnnotations:
+                    items:
+                      type: string
+                    type: array
                   image:
                     type: string
                   imagePullPolicy:
@@ -5478,6 +5482,10 @@ spec:
                 description: KubernetesConfig will be the JSON struct for Basic Redis
                   Config
                 properties:
+                  ignoreAnnotations:
+                    items:
+                      type: string
+                    type: array
                   image:
                     type: string
                   imagePullPolicy:

diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml
@@ -934,6 +934,10 @@ spec:
                 description: KubernetesConfig will be the JSON struct for Basic Redis
                   Config
                 properties:
+                  ignoreAnnotations:
+                    items:
+                      type: string
+                    type: array
                   image:
                     type: string
                   imagePullPolicy:
@@ -5105,6 +5109,10 @@ spec:
                 description: KubernetesConfig will be the JSON struct for Basic Redis
                   Config
                 properties:
+                  ignoreAnnotations:
+                    items:
+                      type: string
+                    type: array
                   image:
                     type: string
                   imagePullPolicy:

diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redissentinels.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redissentinels.yaml
@@ -935,6 +935,10 @@ spec:
                 description: KubernetesConfig will be the JSON struct for Basic Redis
                   Config
                 properties:
+                  ignoreAnnotations:
+                    items:
+                      type: string
+                    type: array
                   image:
                     type: string
                   imagePullPolicy:
@@ -2794,6 +2798,10 @@ spec:
                 description: KubernetesConfig will be the JSON struct for Basic Redis
                   Config
                 properties:
+                  ignoreAnnotations:
+                    items:
+                      type: string
+                    type: array
                   image:
                     type: string
                   imagePullPolicy:

diff --git a/k8sutils/labels.go b/k8sutils/labels.go
@@ -86,22 +86,27 @@ func redisSentinelAsOwner(cr *redisv1beta2.RedisSentinel) metav1.OwnerReference
 }
 
 // generateStatefulSetsAnots generates and returns statefulsets annotations
-func generateStatefulSetsAnots(stsMeta metav1.ObjectMeta) map[string]string {
+func generateStatefulSetsAnots(stsMeta metav1.ObjectMeta, ignoreAnnots []string) map[string]string {
 	anots := map[string]string{
 		"redis.opstreelabs.in":       "true",
 		"redis.opstreelabs.instance": stsMeta.GetName(),
 	}
 	for k, v := range stsMeta.GetAnnotations() {
 		anots[k] = v
 	}
-	return filterAnnotations(anots)
+	return filterAnnotations(anots, ignoreAnnots...)
 }
 
 // filterAnnotations Remove autogenerated annotations which pose no use to downstream objects (Services,Pods,etc)
-func filterAnnotations(anots map[string]string) map[string]string {
+func filterAnnotations(anots map[string]string, ignoreAnnots ...string) map[string]string {
 	// Filter out some problematic annotations we don't want in the template.
 	delete(anots, "kubectl.kubernetes.io/last-applied-configuration")
 	delete(anots, "banzaicloud.com/last-applied")
+
+	for _, ignoreAnnot := range ignoreAnnots {
+		delete(anots, ignoreAnnot)
+	}
+
 	return anots
 }
 

diff --git a/k8sutils/labels_test.go b/k8sutils/labels_test.go
@@ -101,6 +101,7 @@ func TestAddOwnerRefToObject(t *testing.T) {
 func Test_generateStatefulSetsAnots(t *testing.T) {
 	type args struct {
 		stsMeta metav1.ObjectMeta
+		ignore  []string
 	}
 	tests := []struct {
 		name string
@@ -128,10 +129,33 @@ func Test_generateStatefulSetsAnots(t *testing.T) {
 				"redis.opstreelabs.instance":   "sts",
 			},
 		},
+		{
+			name: "generateStatefulSetsAnots_with_ignore",
+			args: args{
+				stsMeta: metav1.ObjectMeta{
+					Name:      "sts",
+					Namespace: "default",
+					Annotations: map[string]string{
+						"app.kubernetes.io/name":       "redis",
+						"operator.redis.com/redis":     "redis",
+						"operator.redis.com/redis-uid": "1234567890",
+						"a":                            "b",
+					},
+				},
+				ignore: []string{"a"},
+			},
+			want: map[string]string{
+				"app.kubernetes.io/name":       "redis",
+				"operator.redis.com/redis":     "redis",
+				"operator.redis.com/redis-uid": "1234567890",
+				"redis.opstreelabs.in":         "true",
+				"redis.opstreelabs.instance":   "sts",
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := generateStatefulSetsAnots(tt.args.stsMeta); !reflect.DeepEqual(got, tt.want) {
+			if got := generateStatefulSetsAnots(tt.args.stsMeta, tt.args.ignore); !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("generateStatefulSetsAnots() = %v, want %v", got, tt.want)
 			}
 		})

diff --git a/k8sutils/poddisruption.go b/k8sutils/poddisruption.go
@@ -20,7 +20,7 @@ func ReconcileRedisPodDisruptionBudget(cr *redisv1beta2.RedisCluster, role strin
 	logger := pdbLogger(cr.Namespace, pdbName)
 	if pdbParams != nil && pdbParams.Enabled {
 		labels := getRedisLabels(cr.ObjectMeta.Name, cluster, role, cr.ObjectMeta.GetLabels())
-		annotations := generateStatefulSetsAnots(cr.ObjectMeta)
+		annotations := generateStatefulSetsAnots(cr.ObjectMeta, cr.Spec.KubernetesConfig.IgnoreAnnotations)
 		pdbMeta := generateObjectMetaInformation(pdbName, cr.Namespace, labels, annotations)
 		pdbDef := generatePodDisruptionBudgetDef(cr, role, pdbMeta, cr.Spec.RedisLeader.PodDisruptionBudget)
 		return CreateOrUpdatePodDisruptionBudget(pdbDef)
@@ -43,7 +43,7 @@ func ReconcileSentinelPodDisruptionBudget(cr *redisv1beta2.RedisSentinel, pdbPar
 	logger := pdbLogger(cr.Namespace, pdbName)
 	if pdbParams != nil && pdbParams.Enabled {
 		labels := getRedisLabels(cr.ObjectMeta.Name, sentinel, "sentinel", cr.ObjectMeta.GetLabels())
-		annotations := generateStatefulSetsAnots(cr.ObjectMeta)
+		annotations := generateStatefulSetsAnots(cr.ObjectMeta, cr.Spec.KubernetesConfig.IgnoreAnnotations)
 		pdbMeta := generateObjectMetaInformation(pdbName, cr.Namespace, labels, annotations)
 		pdbDef := generateSentinelPodDisruptionBudgetDef(cr, "sentinel", pdbMeta, pdbParams)
 		return CreateOrUpdatePodDisruptionBudget(pdbDef)

diff --git a/k8sutils/redis-cluster.go b/k8sutils/redis-cluster.go
@@ -39,6 +39,7 @@ func generateRedisClusterParams(cr *redisv1beta2.RedisCluster, replicas int32, e
 		Tolerations:                   params.Tolerations,
 		ServiceAccountName:            cr.Spec.ServiceAccountName,
 		UpdateStrategy:                cr.Spec.KubernetesConfig.UpdateStrategy,
+		IgnoreAnnotations:             cr.Spec.KubernetesConfig.IgnoreAnnotations,
 	}
 	if cr.Spec.RedisExporter != nil {
 		res.EnableMetrics = cr.Spec.RedisExporter.Enabled
@@ -210,7 +211,7 @@ func (service RedisClusterSTS) CreateRedisClusterSetup(cr *redisv1beta2.RedisClu
 	stateFulName := cr.ObjectMeta.Name + "-" + service.RedisStateFulType
 	logger := statefulSetLogger(cr.Namespace, stateFulName)
 	labels := getRedisLabels(stateFulName, cluster, service.RedisStateFulType, cr.ObjectMeta.Labels)
-	annotations := generateStatefulSetsAnots(cr.ObjectMeta)
+	annotations := generateStatefulSetsAnots(cr.ObjectMeta, cr.Spec.KubernetesConfig.IgnoreAnnotations)
 	objectMetaInfo := generateObjectMetaInformation(stateFulName, cr.Namespace, labels, annotations)
 	err := CreateOrUpdateStateFul(
 		cr.Namespace,

diff --git a/k8sutils/redis-replication.go b/k8sutils/redis-replication.go
@@ -46,7 +46,7 @@ func CreateReplicationRedis(cr *redisv1beta2.RedisReplication) error {
 	stateFulName := cr.ObjectMeta.Name
 	logger := statefulSetLogger(cr.Namespace, cr.ObjectMeta.Name)
 	labels := getRedisLabels(cr.ObjectMeta.Name, replication, "replication", cr.ObjectMeta.Labels)
-	annotations := generateStatefulSetsAnots(cr.ObjectMeta)
+	annotations := generateStatefulSetsAnots(cr.ObjectMeta, cr.Spec.KubernetesConfig.IgnoreAnnotations)
 	objectMetaInfo := generateObjectMetaInformation(stateFulName, cr.Namespace, labels, annotations)
 	err := CreateOrUpdateStateFul(cr.Namespace,
 		objectMetaInfo,
@@ -76,6 +76,7 @@ func generateRedisReplicationParams(cr *redisv1beta2.RedisReplication) statefulS
 		Tolerations:                   cr.Spec.Tolerations,
 		TerminationGracePeriodSeconds: cr.Spec.TerminationGracePeriodSeconds,
 		UpdateStrategy:                cr.Spec.KubernetesConfig.UpdateStrategy,
+		IgnoreAnnotations:             cr.Spec.KubernetesConfig.IgnoreAnnotations,
 	}
 	if cr.Spec.KubernetesConfig.ImagePullSecrets != nil {
 		res.ImagePullSecrets = cr.Spec.KubernetesConfig.ImagePullSecrets

diff --git a/k8sutils/redis-sentinel.go b/k8sutils/redis-sentinel.go
@@ -64,7 +64,7 @@ func (service RedisSentinelSTS) CreateRedisSentinelSetup(ctx context.Context, cr
 	stateFulName := cr.ObjectMeta.Name + "-" + service.RedisStateFulType
 	logger := statefulSetLogger(cr.Namespace, stateFulName)
 	labels := getRedisLabels(stateFulName, cluster, service.RedisStateFulType, cr.ObjectMeta.Labels)
-	annotations := generateStatefulSetsAnots(cr.ObjectMeta)
+	annotations := generateStatefulSetsAnots(cr.ObjectMeta, cr.Spec.KubernetesConfig.IgnoreAnnotations)
 	objectMetaInfo := generateObjectMetaInformation(stateFulName, cr.Namespace, labels, annotations)
 	err := CreateOrUpdateStateFul(
 		cr.Namespace,
@@ -99,6 +99,7 @@ func generateRedisSentinelParams(cr *redisv1beta2.RedisSentinel, replicas int32,
 		Tolerations:                   cr.Spec.Tolerations,
 		ServiceAccountName:            cr.Spec.ServiceAccountName,
 		UpdateStrategy:                cr.Spec.KubernetesConfig.UpdateStrategy,
+		IgnoreAnnotations:             cr.Spec.KubernetesConfig.IgnoreAnnotations,
 	}
 
 	if cr.Spec.KubernetesConfig.ImagePullSecrets != nil {

diff --git a/k8sutils/redis-standalone.go b/k8sutils/redis-standalone.go
@@ -49,7 +49,7 @@ func CreateStandaloneService(cr *redisv1beta2.Redis) error {
 func CreateStandaloneRedis(cr *redisv1beta2.Redis) error {
 	logger := statefulSetLogger(cr.Namespace, cr.ObjectMeta.Name)
 	labels := getRedisLabels(cr.ObjectMeta.Name, standalone, "standalone", cr.ObjectMeta.Labels)
-	annotations := generateStatefulSetsAnots(cr.ObjectMeta)
+	annotations := generateStatefulSetsAnots(cr.ObjectMeta, cr.Spec.KubernetesConfig.IgnoreAnnotations)
 	objectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name, cr.Namespace, labels, annotations)
 	err := CreateOrUpdateStateFul(cr.Namespace,
 		objectMetaInfo,
@@ -80,6 +80,7 @@ func generateRedisStandaloneParams(cr *redisv1beta2.Redis) statefulSetParameters
 		TerminationGracePeriodSeconds: cr.Spec.TerminationGracePeriodSeconds,
 		Tolerations:                   cr.Spec.Tolerations,
 		UpdateStrategy:                cr.Spec.KubernetesConfig.UpdateStrategy,
+		IgnoreAnnotations:             cr.Spec.KubernetesConfig.IgnoreAnnotations,
 	}
 	if cr.Spec.KubernetesConfig.ImagePullSecrets != nil {
 		res.ImagePullSecrets = cr.Spec.KubernetesConfig.ImagePullSecrets

diff --git a/k8sutils/statefulset.go b/k8sutils/statefulset.go
@@ -46,6 +46,7 @@ type statefulSetParameters struct {
 	RecreateStatefulSet           bool
 	InitContainers                *[]redisv1beta2.InitContainer
 	TerminationGracePeriodSeconds *int64
+	IgnoreAnnotations             []string
 }
 
 // containerParameters will define container input params
@@ -221,7 +222,7 @@ func generateStatefulSetsDef(stsMeta metav1.ObjectMeta, params statefulSetParame
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels:      stsMeta.GetLabels(),
-					Annotations: generateStatefulSetsAnots(stsMeta),
+					Annotations: generateStatefulSetsAnots(stsMeta, params.IgnoreAnnotations),
 				},
 				Spec: corev1.PodSpec{
 					Containers: generateContainerDef(
@@ -317,7 +318,7 @@ func createPVCTemplate(volumeName string, stsMeta metav1.ObjectMeta, storageSpec
 	pvcTemplate.Name = volumeName
 	pvcTemplate.Labels = stsMeta.GetLabels()
 	// We want the same annoations as the StatefulSet here
-	pvcTemplate.Annotations = generateStatefulSetsAnots(stsMeta)
+	pvcTemplate.Annotations = generateStatefulSetsAnots(stsMeta, nil)
 	if storageSpec.Spec.AccessModes == nil {
 		pvcTemplate.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}
 	} else {

diff --git a/tests/_config/kuttl-test.yaml b/tests/_config/kuttl-test.yaml
@@ -5,6 +5,7 @@ kindConfig: "./kind-config.yaml"
 parallel: 1
 timeout: 300
 testDirs:
-- tests/e2e/v1beta2/setup
-suppress : 
+  - tests/e2e/v1beta2/setup
+  - tests/e2e/v1beta2/ignore-annots
+suppress :
   - events
diff --git a/tests/e2e/v1beta2/ignore-annots/redis-cluster/00-install.yaml b/tests/e2e/v1beta2/ignore-annots/redis-cluster/00-install.yaml
@@ -0,0 +1,10 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+  - cluster.yaml
+assert:
+  - ready-cluster-sts.yaml
+  - ready-cluster.yaml
+  - ready-pvc.yaml
+error:
+  - ignore-annots-sts.yaml
diff --git a/tests/e2e/v1beta2/ignore-annots/redis-cluster/cluster.yaml b/tests/e2e/v1beta2/ignore-annots/redis-cluster/cluster.yaml
@@ -0,0 +1,40 @@
+apiVersion: redis.redis.opstreelabs.in/v1beta2
+kind: RedisCluster
+metadata:
+  name: redis-cluster-v1beta2
+  annotations:
+    need-ignore-key: "need-ignore-value"
+spec:
+  clusterSize: 3
+  clusterVersion: v7
+  persistenceEnabled: true
+  podSecurityContext:
+    runAsUser: 1000
+    fsGroup: 1000
+  kubernetesConfig:
+    ignoreAnnotations:
+      - "need-ignore-key"
+    image: quay.io/opstree/redis:v7.0.12
+    imagePullPolicy: IfNotPresent
+    resources:
+      requests:
+        cpu: 101m
+        memory: 128Mi
+      limits:
+        cpu: 101m
+        memory: 128Mi
+  storage:
+    volumeClaimTemplate:
+      spec:
+        # storageClassName: standard
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 1Gi
+    nodeConfVolume: true
+    nodeConfVolumeClaimTemplate:
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 1Gi
diff --git a/tests/e2e/v1beta2/ignore-annots/redis-cluster/ignore-annots-sts.yaml b/tests/e2e/v1beta2/ignore-annots/redis-cluster/ignore-annots-sts.yaml
@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: redis-cluster-v1beta2-leader
+  annotations:
+    need-ignore-key: "need-ignore-value"
+spec:
+  template:
+    metadata:
+      annotations:
+        need-ignore-key: "need-ignore-value"
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: redis-cluster-v1beta2-follower
+  annotations:
+    need-ignore-key: "need-ignore-value"
+spec:
+  template:
+    metadata:
+      annotations:
+        need-ignore-key: "need-ignore-value"