feat: rediscluster support hostnetwork && custom port (#723)

* add cr filed, default port to 6379 Signed-off-by: drivebyer <[email protected]> * replace with custom port Signed-off-by: drivebyer <[email protected]> * service custom port Signed-off-by: drivebyer <[email protected]> * make default to 6379 when reconcile old cluster Signed-off-by: drivebyer <[email protected]> * add e2e Signed-off-by: drivebyer <[email protected]> --------- Signed-off-by: drivebyer <[email protected]>
OT-CONTAINER-KIT · Dec 13, 2023 · adca068 · adca068
1 parent 48b515b
commit adca068
Show file tree

Hide file tree

Showing 37 changed files with 1,499 additions and 51 deletions.
diff --git a/.github/workflows/e2e-chainsaw.yml b/.github/workflows/e2e-chainsaw.yml
@@ -17,6 +17,7 @@ jobs:
         testpath:
           - ./tests/e2e-chainsaw/v1beta2/teardown/
           - ./tests/e2e-chainsaw/v1beta2/setup/
+          - ./tests/e2e-chainsaw/v1beta2/hostnetwork/
 
     steps:
     - name: Checkout code
@@ -39,6 +40,9 @@ jobs:
         chmod +x kubectl
         sudo mv kubectl /usr/local/bin/
 
+    # - name: Install Redis
+    #   uses: shogo82148/actions-setup-redis@v1
+
     - name: Create k8s Kind Cluster
       uses: helm/[email protected]
       with:

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -19,6 +19,7 @@ jobs:
         - ./tests/e2e/v1beta2/teardown
         - ./tests/e2e/v1beta2/ignore-annots
         - ./tests/e2e/v1beta2/scaling
+        - ./tests/e2e/v1beta2/hostnetwork
 
     steps:
     - name: Checkout code

diff --git a/api/v1beta2/rediscluster_default.go b/api/v1beta2/rediscluster_default.go
@@ -0,0 +1,10 @@
+package v1beta2
+
+import "k8s.io/utils/pointer"
+
+// SetDefault sets default values for the RedisCluster object.
+func (r *RedisCluster) SetDefault() {
+	if r.Spec.Port == nil {
+		r.Spec.Port = pointer.Int(6379)
+	}
+}
diff --git a/api/v1beta2/rediscluster_types.go b/api/v1beta2/rediscluster_types.go
@@ -27,6 +27,9 @@ import (
 type RedisClusterSpec struct {
 	Size             *int32           `json:"clusterSize"`
 	KubernetesConfig KubernetesConfig `json:"kubernetesConfig"`
+	HostNetwork      bool             `json:"hostNetwork,omitempty"`
+	// +kubebuilder:default:=6379
+	Port *int `json:"port,omitempty"`
 	// +kubebuilder:default:=v7
 	ClusterVersion *string `json:"clusterVersion,omitempty"`
 	// +kubebuilder:default:={livenessProbe:{initialDelaySeconds: 1, timeoutSeconds: 1, periodSeconds: 10, successThreshold: 1, failureThreshold:3}, readinessProbe:{initialDelaySeconds: 1, timeoutSeconds: 1, periodSeconds: 10, successThreshold: 1, failureThreshold:3}}

diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go
diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml
@@ -5297,6 +5297,8 @@ spec:
                   - name
                   type: object
                 type: array
+              hostNetwork:
+                type: boolean
               initContainer:
                 description: InitContainer for each Redis pods
                 properties:
@@ -5792,6 +5794,9 @@ spec:
                         type: string
                     type: object
                 type: object
+              port:
+                default: 6379
+                type: integer
               priorityClassName:
                 type: string
               redisExporter:

diff --git a/controllers/rediscluster_controller.go b/controllers/rediscluster_controller.go
@@ -55,6 +55,7 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		}
 		return ctrl.Result{}, err
 	}
+	instance.SetDefault()
 
 	if _, found := instance.ObjectMeta.GetAnnotations()["rediscluster.opstreelabs.in/skip-reconcile"]; found {
 		reqLogger.Info("Found annotations rediscluster.opstreelabs.in/skip-reconcile, so skipping reconcile")

diff --git a/k8sutils/cluster-scaling.go b/k8sutils/cluster-scaling.go
@@ -2,6 +2,7 @@ package k8sutils
 
 import (
 	"context"
+	"fmt"
 	"strconv"
 	"strings"
 
@@ -32,9 +33,9 @@ func ReshardRedisCluster(client kubernetes.Interface, logger logr.Logger, cr *re
 	cmd = []string{"redis-cli", "--cluster", "reshard"}
 
 	if *cr.Spec.ClusterVersion == "v7" {
-		cmd = append(cmd, getRedisHostname(transferPOD, cr, "leader")+":6379")
+		cmd = append(cmd, getRedisHostname(transferPOD, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port))
 	} else {
-		cmd = append(cmd, getRedisServerIP(client, logger, transferPOD)+":6379")
+		cmd = append(cmd, getRedisServerIP(client, logger, transferPOD)+fmt.Sprintf(":%d", *cr.Spec.Port))
 	}
 
 	if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil {
@@ -157,9 +158,9 @@ func RebalanceRedisClusterEmptyMasters(client kubernetes.Interface, logger logr.
 	cmd = []string{"redis-cli", "--cluster", "rebalance"}
 
 	if *cr.Spec.ClusterVersion == "v7" {
-		cmd = append(cmd, getRedisHostname(pod, cr, "leader")+":6379")
+		cmd = append(cmd, getRedisHostname(pod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port))
 	} else {
-		cmd = append(cmd, getRedisServerIP(client, logger, pod)+":6379")
+		cmd = append(cmd, getRedisServerIP(client, logger, pod)+fmt.Sprintf(":%d", *cr.Spec.Port))
 	}
 
 	cmd = append(cmd, "--cluster-use-empty-masters")
@@ -209,9 +210,9 @@ func RebalanceRedisCluster(client kubernetes.Interface, logger logr.Logger, cr *
 	cmd = []string{"redis-cli", "--cluster", "rebalance"}
 
 	if *cr.Spec.ClusterVersion == "v7" {
-		cmd = append(cmd, getRedisHostname(pod, cr, "leader")+":6379")
+		cmd = append(cmd, getRedisHostname(pod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port))
 	} else {
-		cmd = append(cmd, getRedisServerIP(client, logger, pod)+":6379")
+		cmd = append(cmd, getRedisServerIP(client, logger, pod)+fmt.Sprintf(":%d", *cr.Spec.Port))
 	}
 
 	if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil {
@@ -246,11 +247,11 @@ func AddRedisNodeToCluster(ctx context.Context, client kubernetes.Interface, log
 	cmd = []string{"redis-cli", "--cluster", "add-node"}
 
 	if *cr.Spec.ClusterVersion == "v7" {
-		cmd = append(cmd, getRedisHostname(newPod, cr, "leader")+":6379")
-		cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+":6379")
+		cmd = append(cmd, getRedisHostname(newPod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port))
+		cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port))
 	} else {
-		cmd = append(cmd, getRedisServerIP(client, logger, newPod)+":6379")
-		cmd = append(cmd, getRedisServerIP(client, logger, existingPod)+":6379")
+		cmd = append(cmd, getRedisServerIP(client, logger, newPod)+fmt.Sprintf(":%d", *cr.Spec.Port))
+		cmd = append(cmd, getRedisServerIP(client, logger, existingPod)+fmt.Sprintf(":%d", *cr.Spec.Port))
 	}
 
 	if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil {
@@ -327,9 +328,9 @@ func RemoveRedisFollowerNodesFromCluster(ctx context.Context, client kubernetes.
 
 	cmd = append(cmd, "--cluster", "del-node")
 	if *cr.Spec.ClusterVersion == "v7" {
-		cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+":6379")
+		cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port))
 	} else {
-		cmd = append(cmd, getRedisServerIP(client, logger, existingPod)+":6379")
+		cmd = append(cmd, getRedisServerIP(client, logger, existingPod)+fmt.Sprintf(":%d", *cr.Spec.Port))
 	}
 
 	for _, followerNodeID := range followerNodeIDs {
@@ -358,9 +359,9 @@ func RemoveRedisNodeFromCluster(ctx context.Context, client kubernetes.Interface
 	cmd = []string{"redis-cli", "--cluster", "del-node"}
 
 	if *cr.Spec.ClusterVersion == "v7" {
-		cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+":6379")
+		cmd = append(cmd, getRedisHostname(existingPod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port))
 	} else {
-		cmd = append(cmd, getRedisServerIP(client, logger, existingPod)+":6379")
+		cmd = append(cmd, getRedisServerIP(client, logger, existingPod)+fmt.Sprintf(":%d", *cr.Spec.Port))
 	}
 
 	removePodNodeID := getRedisNodeID(ctx, client, logger, cr, removePod)
@@ -419,9 +420,9 @@ func ClusterFailover(ctx context.Context, client kubernetes.Interface, logger lo
 	cmd = []string{"redis-cli", "cluster", "failover"}
 
 	if *cr.Spec.ClusterVersion == "v7" {
-		cmd = append(cmd, getRedisHostname(pod, cr, "leader")+":6379")
+		cmd = append(cmd, getRedisHostname(pod, cr, "leader")+fmt.Sprintf(":%d", *cr.Spec.Port))
 	} else {
-		cmd = append(cmd, getRedisServerIP(client, logger, pod)+":6379")
+		cmd = append(cmd, getRedisServerIP(client, logger, pod)+fmt.Sprintf(":%d", *cr.Spec.Port))
 	}
 
 	if cr.Spec.KubernetesConfig.ExistingPasswordSecret != nil {

diff --git a/k8sutils/redis-cluster.go b/k8sutils/redis-cluster.go
@@ -39,6 +39,7 @@ func generateRedisClusterParams(cr *redisv1beta2.RedisCluster, replicas int32, e
 		ServiceAccountName:            cr.Spec.ServiceAccountName,
 		UpdateStrategy:                cr.Spec.KubernetesConfig.UpdateStrategy,
 		IgnoreAnnotations:             cr.Spec.KubernetesConfig.IgnoreAnnotations,
+		HostNetwork:                   cr.Spec.HostNetwork,
 	}
 	if cr.Spec.RedisExporter != nil {
 		res.EnableMetrics = cr.Spec.RedisExporter.Enabled
@@ -100,6 +101,7 @@ func generateRedisClusterContainerParams(cr *redisv1beta2.RedisCluster, security
 		ImagePullPolicy: cr.Spec.KubernetesConfig.ImagePullPolicy,
 		Resources:       cr.Spec.KubernetesConfig.Resources,
 		SecurityContext: securityContext,
+		Port:            cr.Spec.Port,
 	}
 	if cr.Spec.EnvVars != nil {
 		containerProp.EnvVars = cr.Spec.EnvVars
@@ -246,12 +248,12 @@ func (service RedisClusterService) CreateRedisClusterService(cr *redisv1beta2.Re
 	objectMetaInfo := generateObjectMetaInformation(serviceName, cr.Namespace, labels, annotations)
 	headlessObjectMetaInfo := generateObjectMetaInformation(serviceName+"-headless", cr.Namespace, labels, annotations)
 	additionalObjectMetaInfo := generateObjectMetaInformation(serviceName+"-additional", cr.Namespace, labels, generateServiceAnots(cr.ObjectMeta, additionalServiceAnnotations))
-	err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisClusterAsOwner(cr), false, true, "ClusterIP")
+	err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisClusterAsOwner(cr), false, true, "ClusterIP", *cr.Spec.Port)
 	if err != nil {
 		logger.Error(err, "Cannot create headless service for Redis", "Setup.Type", service.RedisServiceRole)
 		return err
 	}
-	err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisClusterAsOwner(cr), enableMetrics, false, "ClusterIP")
+	err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisClusterAsOwner(cr), enableMetrics, false, "ClusterIP", *cr.Spec.Port)
 	if err != nil {
 		logger.Error(err, "Cannot create service for Redis", "Setup.Type", service.RedisServiceRole)
 		return err
@@ -260,7 +262,7 @@ func (service RedisClusterService) CreateRedisClusterService(cr *redisv1beta2.Re
 	if cr.Spec.KubernetesConfig.Service != nil {
 		additionalServiceType = cr.Spec.KubernetesConfig.Service.ServiceType
 	}
-	err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisClusterAsOwner(cr), false, false, additionalServiceType)
+	err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisClusterAsOwner(cr), false, false, additionalServiceType, *cr.Spec.Port)
 	if err != nil {
 		logger.Error(err, "Cannot create additional service for Redis", "Setup.Type", service.RedisServiceRole)
 		return err

diff --git a/k8sutils/redis-replication.go b/k8sutils/redis-replication.go
@@ -19,12 +19,12 @@ func CreateReplicationService(cr *redisv1beta2.RedisReplication) error {
 	objectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name, cr.Namespace, labels, annotations)
 	headlessObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-headless", cr.Namespace, labels, annotations)
 	additionalObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-additional", cr.Namespace, labels, generateServiceAnots(cr.ObjectMeta, additionalServiceAnnotations))
-	err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisReplicationAsOwner(cr), false, true, "ClusterIP")
+	err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisReplicationAsOwner(cr), false, true, "ClusterIP", redisPort)
 	if err != nil {
 		logger.Error(err, "Cannot create replication headless service for Redis")
 		return err
 	}
-	err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisReplicationAsOwner(cr), enableMetrics, false, "ClusterIP")
+	err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisReplicationAsOwner(cr), enableMetrics, false, "ClusterIP", redisPort)
 	if err != nil {
 		logger.Error(err, "Cannot create replication service for Redis")
 		return err
@@ -33,7 +33,7 @@ func CreateReplicationService(cr *redisv1beta2.RedisReplication) error {
 	if cr.Spec.KubernetesConfig.Service != nil {
 		additionalServiceType = cr.Spec.KubernetesConfig.Service.ServiceType
 	}
-	err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisReplicationAsOwner(cr), false, false, additionalServiceType)
+	err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisReplicationAsOwner(cr), false, false, additionalServiceType, redisPort)
 	if err != nil {
 		logger.Error(err, "Cannot create additional service for Redis Replication")
 		return err

diff --git a/k8sutils/redis-sentinel.go b/k8sutils/redis-sentinel.go
@@ -215,12 +215,12 @@ func (service RedisSentinelService) CreateRedisSentinelService(cr *redisv1beta2.
 	headlessObjectMetaInfo := generateObjectMetaInformation(serviceName+"-headless", cr.Namespace, labels, annotations)
 	additionalObjectMetaInfo := generateObjectMetaInformation(serviceName+"-additional", cr.Namespace, labels, generateServiceAnots(cr.ObjectMeta, additionalServiceAnnotations))
 
-	err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisSentinelAsOwner(cr), false, true, "ClusterIP")
+	err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisSentinelAsOwner(cr), false, true, "ClusterIP", sentinelPort)
 	if err != nil {
 		logger.Error(err, "Cannot create headless service for Redis", "Setup.Type", service.RedisServiceRole)
 		return err
 	}
-	err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisSentinelAsOwner(cr), enableMetrics, false, "ClusterIP")
+	err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisSentinelAsOwner(cr), enableMetrics, false, "ClusterIP", sentinelPort)
 	if err != nil {
 		logger.Error(err, "Cannot create service for Redis", "Setup.Type", service.RedisServiceRole)
 		return err
@@ -230,7 +230,7 @@ func (service RedisSentinelService) CreateRedisSentinelService(cr *redisv1beta2.
 	if cr.Spec.KubernetesConfig.Service != nil {
 		additionalServiceType = cr.Spec.KubernetesConfig.Service.ServiceType
 	}
-	err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisSentinelAsOwner(cr), false, false, additionalServiceType)
+	err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisSentinelAsOwner(cr), false, false, additionalServiceType, sentinelPort)
 	if err != nil {
 		logger.Error(err, "Cannot create additional service for Redis", "Setup.Type", service.RedisServiceRole)
 		return err

diff --git a/k8sutils/redis-sentinel_test.go b/k8sutils/redis-sentinel_test.go
@@ -2,8 +2,10 @@ package k8sutils
 
 import (
 	"context"
+	"k8s.io/client-go/kubernetes"
 	"os"
 	"path/filepath"
+	"reflect"
 	"testing"
 
 	common "github.com/OT-CONTAINER-KIT/redis-operator/api"
@@ -246,3 +248,87 @@ func Test_generateRedisSentinelInitContainerParams(t *testing.T) {
 	actual := generateRedisSentinelInitContainerParams(input)
 	assert.EqualValues(t, expected, actual, "Expected %+v, got %+v", expected, actual)
 }
+
+func Test_getSentinelEnvVariable(t *testing.T) {
+	type args struct {
+		ctx    context.Context
+		client kubernetes.Interface
+		logger logr.Logger
+		cr     *redisv1beta2.RedisSentinel
+	}
+	tests := []struct {
+		name string
+		args args
+		want *[]corev1.EnvVar
+	}{
+		{
+			name: "When RedisSentinelConfig is nil",
+			args: args{
+				ctx:    context.TODO(),
+				client: nil,
+				logger: logr.Logger{},
+				cr:     &redisv1beta2.RedisSentinel{},
+			},
+			want: &[]corev1.EnvVar{},
+		},
+		{
+			name: "When RedisSentinelConfig is not nil",
+			args: args{
+				ctx:    context.TODO(),
+				client: nil,
+				logger: logr.Logger{},
+				cr: &redisv1beta2.RedisSentinel{
+					Spec: redisv1beta2.RedisSentinelSpec{
+						RedisSentinelConfig: &redisv1beta2.RedisSentinelConfig{
+							RedisSentinelConfig: common.RedisSentinelConfig{
+								MasterGroupName:       "master",
+								RedisPort:             "6379",
+								Quorum:                "2",
+								DownAfterMilliseconds: "30000",
+								ParallelSyncs:         "1",
+								FailoverTimeout:       "180000",
+							},
+						},
+					},
+				},
+			},
+			want: &[]corev1.EnvVar{
+				{
+					Name:  "MASTER_GROUP_NAME",
+					Value: "master",
+				},
+				{
+					Name:  "IP",
+					Value: "",
+				},
+				{
+					Name:  "PORT",
+					Value: "6379",
+				},
+				{
+					Name:  "QUORUM",
+					Value: "2",
+				},
+				{
+					Name:  "DOWN_AFTER_MILLISECONDS",
+					Value: "30000",
+				},
+				{
+					Name:  "PARALLEL_SYNCS",
+					Value: "1",
+				},
+				{
+					Name:  "FAILOVER_TIMEOUT",
+					Value: "180000",
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := getSentinelEnvVariable(tt.args.ctx, tt.args.client, tt.args.logger, tt.args.cr); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("getSentinelEnvVariable() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
diff --git a/k8sutils/redis-standalone.go b/k8sutils/redis-standalone.go
@@ -23,12 +23,12 @@ func CreateStandaloneService(cr *redisv1beta2.Redis) error {
 	objectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name, cr.Namespace, labels, annotations)
 	headlessObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-headless", cr.Namespace, labels, annotations)
 	additionalObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-additional", cr.Namespace, labels, generateServiceAnots(cr.ObjectMeta, additionalServiceAnnotations))
-	err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisAsOwner(cr), false, true, "ClusterIP")
+	err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisAsOwner(cr), false, true, "ClusterIP", redisPort)
 	if err != nil {
 		logger.Error(err, "Cannot create standalone headless service for Redis")
 		return err
 	}
-	err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisAsOwner(cr), enableMetrics, false, "ClusterIP")
+	err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisAsOwner(cr), enableMetrics, false, "ClusterIP", redisPort)
 	if err != nil {
 		logger.Error(err, "Cannot create standalone service for Redis")
 		return err
@@ -37,7 +37,7 @@ func CreateStandaloneService(cr *redisv1beta2.Redis) error {
 	if cr.Spec.KubernetesConfig.Service != nil {
 		additionalServiceType = cr.Spec.KubernetesConfig.Service.ServiceType
 	}
-	err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisAsOwner(cr), false, false, additionalServiceType)
+	err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisAsOwner(cr), false, false, additionalServiceType, redisPort)
 	if err != nil {
 		logger.Error(err, "Cannot create additional service for Redis")
 		return err