diff --git a/api/status/redis-cluster_status.go b/api/status/redis-cluster_status.go index e58c62ea5..bd9cc9261 100644 --- a/api/status/redis-cluster_status.go +++ b/api/status/redis-cluster_status.go @@ -14,6 +14,6 @@ const ( RedisClusterInitializing RedisClusterState = "Initializing" RedisClusterBootstrap RedisClusterState = "Bootstrap" // RedisClusterReady means the RedisCluster is ready for use, we use redis-cli --cluster check 127.0.0.1:6379 to check the cluster status - RedisClusterReady RedisClusterState = "Ready" - // RedisClusterFailed RedisClusterState = "Failed" + RedisClusterReady RedisClusterState = "Ready" + RedisClusterFailed RedisClusterState = "Failed" ) diff --git a/pkg/controllers/rediscluster/rediscluster_controller.go b/pkg/controllers/rediscluster/rediscluster_controller.go index b2fb80503..bd351ca24 100644 --- a/pkg/controllers/rediscluster/rediscluster_controller.go +++ b/pkg/controllers/rediscluster/rediscluster_controller.go @@ -47,7 +47,7 @@ type RedisClusterReconciler struct { func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { reqLogger := r.Log.WithValues("Request.Namespace", req.Namespace, "Request.Name", req.Name) - reqLogger.Info("Reconciling opstree redis Cluster controller") + reqLogger.V(1).Info("Reconciling opstree redis Cluster controller") instance := &redisv1beta2.RedisCluster{} err := r.Client.Get(context.TODO(), req.NamespacedName, instance) @@ -188,12 +188,18 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request return intctrlutil.RequeueAfter(reqLogger, time.Second*60, "Redis cluster count is not desired", "Current.Count", nc, "Desired.Count", totalReplicas) } - reqLogger.Info("Number of Redis nodes match desired") + reqLogger.V(1).Info("Number of Redis nodes match desired") unhealthyNodeCount, err := k8sutils.UnhealthyNodesInCluster(ctx, r.K8sClient, r.Log, instance) if err != nil { reqLogger.Error(err, "failed to determine unhealthy node count in cluster") } if int(totalReplicas) > 1 && unhealthyNodeCount >= int(totalReplicas)-1 { + + err = k8sutils.UpdateRedisClusterStatus(instance, status.RedisClusterFailed, "RedisCluster has too many unhealthy nodes", leaderReplicas, followerReplicas, r.Dk8sClient) + if err != nil { + return intctrlutil.RequeueWithError(err, reqLogger, "") + } + reqLogger.Info("healthy leader count does not match desired; attempting to repair disconnected masters") if err = k8sutils.RepairDisconnectedMasters(ctx, r.K8sClient, r.Log, instance); err != nil { reqLogger.Error(err, "failed to repair disconnected masters") diff --git a/tests/e2e-chainsaw/v1beta2/redis-cluster-restart/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/redis-cluster-restart/chainsaw-test.yaml new file mode 100644 index 000000000..4e66cf969 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/redis-cluster-restart/chainsaw-test.yaml @@ -0,0 +1,51 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: redis-cluster +spec: + steps: + - try: + - apply: + file: cluster.yaml + - assert: + file: ready-cluster.yaml + + - name: Try saving a key With Password + try: + - script: + timeout: 30s + content: > + kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-0 -- + redis-cli -c set foo-0 bar-0 + check: + (contains($stdout, 'OK')): true + + - name: Restart pods at same time + try: + - script: + timeout: 30s + content: > + kubectl delete pod --namespace ${NAMESPACE} -l app=redis-cluster-v1beta2-leader + + - name: Wait cluster to be failed + try: + - assert: + file: failed-cluster.yaml + + - name: Wait cluster to be ready + try: + - assert: + file: ready-cluster.yaml + + - name: Get key + try: + - script: + timeout: 30s + content: > + kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-0 -- + redis-cli -c get foo-0 + check: + (contains($stdout, 'bar-0')): true + diff --git a/tests/e2e-chainsaw/v1beta2/redis-cluster-restart/cluster.yaml b/tests/e2e-chainsaw/v1beta2/redis-cluster-restart/cluster.yaml new file mode 100644 index 000000000..3d8d59913 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/redis-cluster-restart/cluster.yaml @@ -0,0 +1,52 @@ +--- +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: RedisCluster +metadata: + name: redis-cluster-v1beta2 +spec: + clusterSize: 3 + clusterVersion: v7 + persistenceEnabled: true + podSecurityContext: + runAsUser: 1000 + fsGroup: 1000 + redisLeader: + replicas: 3 + redisFollower: + replicas: 0 + kubernetesConfig: + image: quay.io/opstree/redis:latest + imagePullPolicy: Always + resources: + requests: + cpu: 101m + memory: 128Mi + limits: + cpu: 101m + memory: 128Mi + redisExporter: + enabled: true + image: quay.io/opstree/redis-exporter:v1.44.0 + imagePullPolicy: Always + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 100m + memory: 128Mi + storage: + volumeClaimTemplate: + spec: + # storageClassName: standard + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 1Gi + nodeConfVolume: true + nodeConfVolumeClaimTemplate: + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 1Gi diff --git a/tests/e2e-chainsaw/v1beta2/redis-cluster-restart/failed-cluster.yaml b/tests/e2e-chainsaw/v1beta2/redis-cluster-restart/failed-cluster.yaml new file mode 100644 index 000000000..a232fa7ae --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/redis-cluster-restart/failed-cluster.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: RedisCluster +metadata: + name: redis-cluster-v1beta2 +status: + state: Failed diff --git a/tests/e2e-chainsaw/v1beta2/redis-cluster-restart/ready-cluster.yaml b/tests/e2e-chainsaw/v1beta2/redis-cluster-restart/ready-cluster.yaml new file mode 100644 index 000000000..43462e53d --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/redis-cluster-restart/ready-cluster.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: RedisCluster +metadata: + name: redis-cluster-v1beta2 +status: + state: Ready + reason: RedisCluster is ready