From aa7042b5d294b7580804328a0132931fa1f39d60 Mon Sep 17 00:00:00 2001 From: Mathieu Cesbron Date: Sat, 13 Jan 2024 19:30:16 +0100 Subject: [PATCH] Generate k8s client only once Signed-off-by: Mathieu Cesbron --- controllers/redis_controller.go | 4 +- controllers/rediscluster_controller.go | 16 +++--- controllers/redisreplication_controller.go | 6 +- controllers/redissentinel_controller.go | 6 +- k8sutils/poddisruption.go | 65 ++++++++-------------- k8sutils/redis-cluster.go | 40 ++++++------- k8sutils/redis-cluster_test.go | 6 +- k8sutils/redis-replication.go | 12 ++-- k8sutils/redis-sentinel.go | 19 ++++--- k8sutils/redis-standalone.go | 12 ++-- k8sutils/redis.go | 6 +- k8sutils/services.go | 40 +++++-------- k8sutils/statefulset.go | 55 ++++++------------ tests/readme.md | 2 +- 14 files changed, 123 insertions(+), 166 deletions(-) diff --git a/controllers/redis_controller.go b/controllers/redis_controller.go index ee7224ade..a73eb287f 100644 --- a/controllers/redis_controller.go +++ b/controllers/redis_controller.go @@ -65,11 +65,11 @@ func (r *RedisReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl return ctrl.Result{}, err } - err = k8sutils.CreateStandaloneRedis(instance) + err = k8sutils.CreateStandaloneRedis(instance, r.K8sClient) if err != nil { return ctrl.Result{}, err } - err = k8sutils.CreateStandaloneService(instance) + err = k8sutils.CreateStandaloneService(instance, r.K8sClient) if err != nil { return ctrl.Result{}, err } diff --git a/controllers/rediscluster_controller.go b/controllers/rediscluster_controller.go index d49230b51..a3a65c04e 100644 --- a/controllers/rediscluster_controller.go +++ b/controllers/rediscluster_controller.go @@ -117,22 +117,22 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request } if leaderReplicas != 0 { - err = k8sutils.CreateRedisLeaderService(instance) + err = k8sutils.CreateRedisLeaderService(instance, r.K8sClient) if err != nil { return ctrl.Result{}, err } } - err = k8sutils.CreateRedisLeader(instance) + err = k8sutils.CreateRedisLeader(instance, r.K8sClient) if err != nil { return ctrl.Result{}, err } - err = k8sutils.ReconcileRedisPodDisruptionBudget(instance, "leader", instance.Spec.RedisLeader.PodDisruptionBudget) + err = k8sutils.ReconcileRedisPodDisruptionBudget(instance, "leader", instance.Spec.RedisLeader.PodDisruptionBudget, r.K8sClient) if err != nil { return ctrl.Result{}, err } - redisLeaderInfo, err := k8sutils.GetStatefulSet(instance.Namespace, instance.ObjectMeta.Name+"-leader") + redisLeaderInfo, err := k8sutils.GetStatefulSet(instance.Namespace, instance.ObjectMeta.Name+"-leader", r.K8sClient) if err != nil { if errors.IsNotFound(err) { return ctrl.Result{RequeueAfter: time.Second * 60}, nil @@ -151,21 +151,21 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request } // if we have followers create their service. if followerReplicas != 0 { - err = k8sutils.CreateRedisFollowerService(instance) + err = k8sutils.CreateRedisFollowerService(instance, r.K8sClient) if err != nil { return ctrl.Result{}, err } } - err = k8sutils.CreateRedisFollower(instance) + err = k8sutils.CreateRedisFollower(instance, r.K8sClient) if err != nil { return ctrl.Result{}, err } - err = k8sutils.ReconcileRedisPodDisruptionBudget(instance, "follower", instance.Spec.RedisFollower.PodDisruptionBudget) + err = k8sutils.ReconcileRedisPodDisruptionBudget(instance, "follower", instance.Spec.RedisFollower.PodDisruptionBudget, r.K8sClient) if err != nil { return ctrl.Result{}, err } } - redisFollowerInfo, err := k8sutils.GetStatefulSet(instance.Namespace, instance.ObjectMeta.Name+"-follower") + redisFollowerInfo, err := k8sutils.GetStatefulSet(instance.Namespace, instance.ObjectMeta.Name+"-follower", r.K8sClient) if err != nil { if errors.IsNotFound(err) { return ctrl.Result{RequeueAfter: time.Second * 60}, nil diff --git a/controllers/redisreplication_controller.go b/controllers/redisreplication_controller.go index c424c85d5..a23cb8b3e 100644 --- a/controllers/redisreplication_controller.go +++ b/controllers/redisreplication_controller.go @@ -56,18 +56,18 @@ func (r *RedisReplicationReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, err } - err = k8sutils.CreateReplicationRedis(instance) + err = k8sutils.CreateReplicationRedis(instance, r.K8sClient) if err != nil { return ctrl.Result{}, err } - err = k8sutils.CreateReplicationService(instance) + err = k8sutils.CreateReplicationService(instance, r.K8sClient) if err != nil { return ctrl.Result{}, err } // Set Pod distruptiuon Budget Later - redisReplicationInfo, err := k8sutils.GetStatefulSet(instance.Namespace, instance.ObjectMeta.Name) + redisReplicationInfo, err := k8sutils.GetStatefulSet(instance.Namespace, instance.ObjectMeta.Name, r.K8sClient) if err != nil { return ctrl.Result{RequeueAfter: time.Second * 60}, err } diff --git a/controllers/redissentinel_controller.go b/controllers/redissentinel_controller.go index 3a88dcc74..03e686544 100644 --- a/controllers/redissentinel_controller.go +++ b/controllers/redissentinel_controller.go @@ -55,18 +55,18 @@ func (r *RedisSentinelReconciler) Reconcile(ctx context.Context, req ctrl.Reques } // Create Redis Sentinel - err = k8sutils.CreateRedisSentinel(ctx, r.K8sClient, r.Log, instance) + err = k8sutils.CreateRedisSentinel(ctx, r.K8sClient, r.Log, instance, r.K8sClient) if err != nil { return ctrl.Result{}, err } - err = k8sutils.ReconcileSentinelPodDisruptionBudget(instance, instance.Spec.PodDisruptionBudget) + err = k8sutils.ReconcileSentinelPodDisruptionBudget(instance, instance.Spec.PodDisruptionBudget, r.K8sClient) if err != nil { return ctrl.Result{}, err } // Create the Service for Redis Sentinel - err = k8sutils.CreateRedisSentinelService(instance) + err = k8sutils.CreateRedisSentinelService(instance, r.K8sClient) if err != nil { return ctrl.Result{}, err } diff --git a/k8sutils/poddisruption.go b/k8sutils/poddisruption.go index 2886f7ccb..77412673f 100644 --- a/k8sutils/poddisruption.go +++ b/k8sutils/poddisruption.go @@ -12,10 +12,11 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes" ) // CreateRedisLeaderPodDisruptionBudget check and create a PodDisruptionBudget for Leaders -func ReconcileRedisPodDisruptionBudget(cr *redisv1beta2.RedisCluster, role string, pdbParams *commonapi.RedisPodDisruptionBudget) error { +func ReconcileRedisPodDisruptionBudget(cr *redisv1beta2.RedisCluster, role string, pdbParams *commonapi.RedisPodDisruptionBudget, cl kubernetes.Interface) error { pdbName := cr.ObjectMeta.Name + "-" + role logger := pdbLogger(cr.Namespace, pdbName) if pdbParams != nil && pdbParams.Enabled { @@ -23,12 +24,12 @@ func ReconcileRedisPodDisruptionBudget(cr *redisv1beta2.RedisCluster, role strin annotations := generateStatefulSetsAnots(cr.ObjectMeta, cr.Spec.KubernetesConfig.IgnoreAnnotations) pdbMeta := generateObjectMetaInformation(pdbName, cr.Namespace, labels, annotations) pdbDef := generatePodDisruptionBudgetDef(cr, role, pdbMeta, cr.Spec.RedisLeader.PodDisruptionBudget) - return CreateOrUpdatePodDisruptionBudget(pdbDef) + return CreateOrUpdatePodDisruptionBudget(pdbDef, cl) } else { // Check if one exists, and delete it. - _, err := GetPodDisruptionBudget(cr.Namespace, pdbName) + _, err := GetPodDisruptionBudget(cr.Namespace, pdbName, cl) if err == nil { - return deletePodDisruptionBudget(cr.Namespace, pdbName) + return deletePodDisruptionBudget(cr.Namespace, pdbName, cl) } else if err != nil && errors.IsNotFound(err) { logger.V(1).Info("Reconciliation Successful, no PodDisruptionBudget Found.") // Its ok if its not found, as we're deleting anyway @@ -38,7 +39,7 @@ func ReconcileRedisPodDisruptionBudget(cr *redisv1beta2.RedisCluster, role strin } } -func ReconcileSentinelPodDisruptionBudget(cr *redisv1beta2.RedisSentinel, pdbParams *commonapi.RedisPodDisruptionBudget) error { +func ReconcileSentinelPodDisruptionBudget(cr *redisv1beta2.RedisSentinel, pdbParams *commonapi.RedisPodDisruptionBudget, cl kubernetes.Interface) error { pdbName := cr.ObjectMeta.Name + "-sentinel" logger := pdbLogger(cr.Namespace, pdbName) if pdbParams != nil && pdbParams.Enabled { @@ -46,12 +47,12 @@ func ReconcileSentinelPodDisruptionBudget(cr *redisv1beta2.RedisSentinel, pdbPar annotations := generateStatefulSetsAnots(cr.ObjectMeta, cr.Spec.KubernetesConfig.IgnoreAnnotations) pdbMeta := generateObjectMetaInformation(pdbName, cr.Namespace, labels, annotations) pdbDef := generateSentinelPodDisruptionBudgetDef(cr, "sentinel", pdbMeta, pdbParams) - return CreateOrUpdatePodDisruptionBudget(pdbDef) + return CreateOrUpdatePodDisruptionBudget(pdbDef, cl) } else { // Check if one exists, and delete it. - _, err := GetPodDisruptionBudget(cr.Namespace, pdbName) + _, err := GetPodDisruptionBudget(cr.Namespace, pdbName, cl) if err == nil { - return deletePodDisruptionBudget(cr.Namespace, pdbName) + return deletePodDisruptionBudget(cr.Namespace, pdbName, cl) } else if err != nil && errors.IsNotFound(err) { logger.V(1).Info("Reconciliation Successful, no PodDisruptionBudget Found.") // Its ok if its not found, as we're deleting anyway @@ -116,24 +117,24 @@ func generateSentinelPodDisruptionBudgetDef(cr *redisv1beta2.RedisSentinel, role } // CreateOrUpdateService method will create or update Redis service -func CreateOrUpdatePodDisruptionBudget(pdbDef *policyv1.PodDisruptionBudget) error { +func CreateOrUpdatePodDisruptionBudget(pdbDef *policyv1.PodDisruptionBudget, cl kubernetes.Interface) error { logger := pdbLogger(pdbDef.Namespace, pdbDef.Name) - storedPDB, err := GetPodDisruptionBudget(pdbDef.Namespace, pdbDef.Name) + storedPDB, err := GetPodDisruptionBudget(pdbDef.Namespace, pdbDef.Name, cl) if err != nil { if err := patch.DefaultAnnotator.SetLastAppliedAnnotation(pdbDef); err != nil { //nolint logger.Error(err, "Unable to patch redis PodDisruptionBudget with comparison object") return err } if errors.IsNotFound(err) { - return createPodDisruptionBudget(pdbDef.Namespace, pdbDef) + return createPodDisruptionBudget(pdbDef.Namespace, pdbDef, cl) } return err } - return patchPodDisruptionBudget(storedPDB, pdbDef, pdbDef.Namespace) + return patchPodDisruptionBudget(storedPDB, pdbDef, pdbDef.Namespace, cl) } // patchPodDisruptionBudget will patch Redis Kubernetes PodDisruptionBudgets -func patchPodDisruptionBudget(storedPdb *policyv1.PodDisruptionBudget, newPdb *policyv1.PodDisruptionBudget, namespace string) error { +func patchPodDisruptionBudget(storedPdb *policyv1.PodDisruptionBudget, newPdb *policyv1.PodDisruptionBudget, namespace string, cl kubernetes.Interface) error { logger := pdbLogger(namespace, storedPdb.Name) // We want to try and keep this atomic as possible. newPdb.ResourceVersion = storedPdb.ResourceVersion @@ -169,20 +170,15 @@ func patchPodDisruptionBudget(storedPdb *policyv1.PodDisruptionBudget, newPdb *p logger.Error(err, "Unable to patch redis PodDisruptionBudget with comparison object") return err } - return updatePodDisruptionBudget(namespace, newPdb) + return updatePodDisruptionBudget(namespace, newPdb, cl) } return nil } // createPodDisruptionBudget is a method to create PodDisruptionBudgets in Kubernetes -func createPodDisruptionBudget(namespace string, pdb *policyv1.PodDisruptionBudget) error { +func createPodDisruptionBudget(namespace string, pdb *policyv1.PodDisruptionBudget, cl kubernetes.Interface) error { logger := pdbLogger(namespace, pdb.Name) - client, err := GenerateK8sClient(GenerateK8sConfig) - if err != nil { - logger.Error(err, "Could not generate kubernetes client") - return err - } - _, err = client.PolicyV1().PodDisruptionBudgets(namespace).Create(context.TODO(), pdb, metav1.CreateOptions{}) + _, err := cl.PolicyV1().PodDisruptionBudgets(namespace).Create(context.TODO(), pdb, metav1.CreateOptions{}) if err != nil { logger.Error(err, "Redis PodDisruptionBudget creation failed") return err @@ -192,14 +188,9 @@ func createPodDisruptionBudget(namespace string, pdb *policyv1.PodDisruptionBudg } // updatePodDisruptionBudget is a method to update PodDisruptionBudgets in Kubernetes -func updatePodDisruptionBudget(namespace string, pdb *policyv1.PodDisruptionBudget) error { +func updatePodDisruptionBudget(namespace string, pdb *policyv1.PodDisruptionBudget, cl kubernetes.Interface) error { logger := pdbLogger(namespace, pdb.Name) - client, err := GenerateK8sClient(GenerateK8sConfig) - if err != nil { - logger.Error(err, "Could not generate kubernetes client") - return err - } - _, err = client.PolicyV1().PodDisruptionBudgets(namespace).Update(context.TODO(), pdb, metav1.UpdateOptions{}) + _, err := cl.PolicyV1().PodDisruptionBudgets(namespace).Update(context.TODO(), pdb, metav1.UpdateOptions{}) if err != nil { logger.Error(err, "Redis PodDisruptionBudget update failed") return err @@ -209,14 +200,9 @@ func updatePodDisruptionBudget(namespace string, pdb *policyv1.PodDisruptionBudg } // deletePodDisruptionBudget is a method to delete PodDisruptionBudgets in Kubernetes -func deletePodDisruptionBudget(namespace string, pdbName string) error { +func deletePodDisruptionBudget(namespace string, pdbName string, cl kubernetes.Interface) error { logger := pdbLogger(namespace, pdbName) - client, err := GenerateK8sClient(GenerateK8sConfig) - if err != nil { - logger.Error(err, "Could not generate kubernetes client") - return err - } - err = client.PolicyV1().PodDisruptionBudgets(namespace).Delete(context.TODO(), pdbName, metav1.DeleteOptions{}) + err := cl.PolicyV1().PodDisruptionBudgets(namespace).Delete(context.TODO(), pdbName, metav1.DeleteOptions{}) if err != nil { logger.Error(err, "Redis PodDisruption deletion failed") return err @@ -226,17 +212,12 @@ func deletePodDisruptionBudget(namespace string, pdbName string) error { } // GetPodDisruptionBudget is a method to get PodDisruptionBudgets in Kubernetes -func GetPodDisruptionBudget(namespace string, pdb string) (*policyv1.PodDisruptionBudget, error) { +func GetPodDisruptionBudget(namespace string, pdb string, cl kubernetes.Interface) (*policyv1.PodDisruptionBudget, error) { logger := pdbLogger(namespace, pdb) - client, err := GenerateK8sClient(GenerateK8sConfig) - if err != nil { - logger.Error(err, "Could not generate kubernetes client") - return nil, err - } getOpts := metav1.GetOptions{ TypeMeta: generateMetaInformation("PodDisruptionBudget", "policy/v1"), } - pdbInfo, err := client.PolicyV1().PodDisruptionBudgets(namespace).Get(context.TODO(), pdb, getOpts) + pdbInfo, err := cl.PolicyV1().PodDisruptionBudgets(namespace).Get(context.TODO(), pdb, getOpts) if err != nil { logger.V(1).Info("Redis PodDisruptionBudget get action failed") return nil, err diff --git a/k8sutils/redis-cluster.go b/k8sutils/redis-cluster.go index 92bd9f64c..cfa2748d2 100644 --- a/k8sutils/redis-cluster.go +++ b/k8sutils/redis-cluster.go @@ -5,6 +5,7 @@ import ( "strings" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes" commonapi "github.com/OT-CONTAINER-KIT/redis-operator/api" redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2" @@ -100,7 +101,7 @@ func generateRedisClusterInitContainerParams(cr *redisv1beta2.RedisCluster) init } // generateRedisClusterContainerParams generates Redis container information -func generateRedisClusterContainerParams(cr *redisv1beta2.RedisCluster, securityContext *corev1.SecurityContext, readinessProbeDef *commonapi.Probe, livenessProbeDef *commonapi.Probe, role string) containerParameters { +func generateRedisClusterContainerParams(cr *redisv1beta2.RedisCluster, securityContext *corev1.SecurityContext, readinessProbeDef *commonapi.Probe, livenessProbeDef *commonapi.Probe, role string, cl kubernetes.Interface) containerParameters { trueProperty := true falseProperty := false containerProp := containerParameters{ @@ -136,7 +137,7 @@ func generateRedisClusterContainerParams(cr *redisv1beta2.RedisCluster, security nps := map[string]ports{} // pod name to ports replicas := cr.Spec.GetReplicaCounts(role) for i := 0; i < int(replicas); i++ { - svc, err := getService(cr.Namespace, cr.ObjectMeta.Name+"-"+role+"-"+strconv.Itoa(i)) + svc, err := getService(cr.Namespace, cr.ObjectMeta.Name+"-"+role+"-"+strconv.Itoa(i), cl) if err != nil { log.Error(err, "Cannot get service for Redis", "Setup.Type", role) } else { @@ -205,7 +206,7 @@ func generateRedisClusterContainerParams(cr *redisv1beta2.RedisCluster, security } // CreateRedisLeader will create a leader redis setup -func CreateRedisLeader(cr *redisv1beta2.RedisCluster) error { +func CreateRedisLeader(cr *redisv1beta2.RedisCluster, cl kubernetes.Interface) error { prop := RedisClusterSTS{ RedisStateFulType: "leader", SecurityContext: cr.Spec.RedisLeader.SecurityContext, @@ -219,11 +220,11 @@ func CreateRedisLeader(cr *redisv1beta2.RedisCluster) error { if cr.Spec.RedisLeader.RedisConfig != nil { prop.ExternalConfig = cr.Spec.RedisLeader.RedisConfig.AdditionalRedisConfig } - return prop.CreateRedisClusterSetup(cr) + return prop.CreateRedisClusterSetup(cr, cl) } // CreateRedisFollower will create a follower redis setup -func CreateRedisFollower(cr *redisv1beta2.RedisCluster) error { +func CreateRedisFollower(cr *redisv1beta2.RedisCluster, cl kubernetes.Interface) error { prop := RedisClusterSTS{ RedisStateFulType: "follower", SecurityContext: cr.Spec.RedisFollower.SecurityContext, @@ -237,23 +238,23 @@ func CreateRedisFollower(cr *redisv1beta2.RedisCluster) error { if cr.Spec.RedisFollower.RedisConfig != nil { prop.ExternalConfig = cr.Spec.RedisFollower.RedisConfig.AdditionalRedisConfig } - return prop.CreateRedisClusterSetup(cr) + return prop.CreateRedisClusterSetup(cr, cl) } // CreateRedisLeaderService method will create service for Redis Leader -func CreateRedisLeaderService(cr *redisv1beta2.RedisCluster) error { +func CreateRedisLeaderService(cr *redisv1beta2.RedisCluster, cl kubernetes.Interface) error { prop := RedisClusterService{ RedisServiceRole: "leader", } - return prop.CreateRedisClusterService(cr) + return prop.CreateRedisClusterService(cr, cl) } // CreateRedisFollowerService method will create service for Redis Follower -func CreateRedisFollowerService(cr *redisv1beta2.RedisCluster) error { +func CreateRedisFollowerService(cr *redisv1beta2.RedisCluster, cl kubernetes.Interface) error { prop := RedisClusterService{ RedisServiceRole: "follower", } - return prop.CreateRedisClusterService(cr) + return prop.CreateRedisClusterService(cr, cl) } func (service RedisClusterSTS) getReplicaCount(cr *redisv1beta2.RedisCluster) int32 { @@ -261,7 +262,7 @@ func (service RedisClusterSTS) getReplicaCount(cr *redisv1beta2.RedisCluster) in } // CreateRedisClusterSetup will create Redis Setup for leader and follower -func (service RedisClusterSTS) CreateRedisClusterSetup(cr *redisv1beta2.RedisCluster) error { +func (service RedisClusterSTS) CreateRedisClusterSetup(cr *redisv1beta2.RedisCluster, cl kubernetes.Interface) error { stateFulName := cr.ObjectMeta.Name + "-" + service.RedisStateFulType logger := statefulSetLogger(cr.Namespace, stateFulName) labels := getRedisLabels(stateFulName, cluster, service.RedisStateFulType, cr.ObjectMeta.Labels) @@ -273,8 +274,9 @@ func (service RedisClusterSTS) CreateRedisClusterSetup(cr *redisv1beta2.RedisClu generateRedisClusterParams(cr, service.getReplicaCount(cr), service.ExternalConfig, service), redisClusterAsOwner(cr), generateRedisClusterInitContainerParams(cr), - generateRedisClusterContainerParams(cr, service.SecurityContext, service.ReadinessProbe, service.LivenessProbe, service.RedisStateFulType), + generateRedisClusterContainerParams(cr, service.SecurityContext, service.ReadinessProbe, service.LivenessProbe, service.RedisStateFulType, cl), cr.Spec.Sidecars, + cl, ) if err != nil { logger.Error(err, "Cannot create statefulset for Redis", "Setup.Type", service.RedisStateFulType) @@ -284,7 +286,7 @@ func (service RedisClusterSTS) CreateRedisClusterSetup(cr *redisv1beta2.RedisClu } // CreateRedisClusterService method will create service for Redis -func (service RedisClusterService) CreateRedisClusterService(cr *redisv1beta2.RedisCluster) error { +func (service RedisClusterService) CreateRedisClusterService(cr *redisv1beta2.RedisCluster, cl kubernetes.Interface) error { serviceName := cr.ObjectMeta.Name + "-" + service.RedisServiceRole logger := serviceLogger(cr.Namespace, serviceName) labels := getRedisLabels(serviceName, cluster, service.RedisServiceRole, cr.ObjectMeta.Labels) @@ -305,12 +307,12 @@ func (service RedisClusterService) CreateRedisClusterService(cr *redisv1beta2.Re objectMetaInfo := generateObjectMetaInformation(serviceName, cr.Namespace, labels, annotations) headlessObjectMetaInfo := generateObjectMetaInformation(serviceName+"-headless", cr.Namespace, labels, annotations) additionalObjectMetaInfo := generateObjectMetaInformation(serviceName+"-additional", cr.Namespace, labels, generateServiceAnots(cr.ObjectMeta, additionalServiceAnnotations, epp)) - err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisClusterAsOwner(cr), disableMetrics, true, "ClusterIP", *cr.Spec.Port) + err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisClusterAsOwner(cr), disableMetrics, true, "ClusterIP", *cr.Spec.Port, cl) if err != nil { logger.Error(err, "Cannot create headless service for Redis", "Setup.Type", service.RedisServiceRole) return err } - err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisClusterAsOwner(cr), epp, false, "ClusterIP", *cr.Spec.Port) + err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisClusterAsOwner(cr), epp, false, "ClusterIP", *cr.Spec.Port, cl) if err != nil { logger.Error(err, "Cannot create service for Redis", "Setup.Type", service.RedisServiceRole) return err @@ -321,14 +323,14 @@ func (service RedisClusterService) CreateRedisClusterService(cr *redisv1beta2.Re if additionalServiceType == "NodePort" { // If NodePort is enabled, we need to create a service for every redis pod. // Then use --cluster-announce-ip --cluster-announce-port --cluster-announce-bus-port to make cluster. - err = service.createOrUpdateClusterNodePortService(cr) + err = service.createOrUpdateClusterNodePortService(cr, cl) if err != nil { logger.Error(err, "Cannot create nodeport service for Redis", "Setup.Type", service.RedisServiceRole) return err } } } - err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisClusterAsOwner(cr), disableMetrics, false, additionalServiceType, *cr.Spec.Port) + err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisClusterAsOwner(cr), disableMetrics, false, additionalServiceType, *cr.Spec.Port, cl) if err != nil { logger.Error(err, "Cannot create additional service for Redis", "Setup.Type", service.RedisServiceRole) return err @@ -336,7 +338,7 @@ func (service RedisClusterService) CreateRedisClusterService(cr *redisv1beta2.Re return nil } -func (service RedisClusterService) createOrUpdateClusterNodePortService(cr *redisv1beta2.RedisCluster) error { +func (service RedisClusterService) createOrUpdateClusterNodePortService(cr *redisv1beta2.RedisCluster, cl kubernetes.Interface) error { replicas := cr.Spec.GetReplicaCounts(service.RedisServiceRole) for i := 0; i < int(replicas); i++ { @@ -356,7 +358,7 @@ func (service RedisClusterService) createOrUpdateClusterNodePortService(cr *redi IntVal: int32(*cr.Spec.Port + 10000), }, } - err := CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisClusterAsOwner(cr), disableMetrics, false, "NodePort", *cr.Spec.Port, busPort) + err := CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisClusterAsOwner(cr), disableMetrics, false, "NodePort", *cr.Spec.Port, cl, busPort) if err != nil { logger.Error(err, "Cannot create nodeport service for Redis", "Setup.Type", service.RedisServiceRole) return err diff --git a/k8sutils/redis-cluster_test.go b/k8sutils/redis-cluster_test.go index e2692951d..bdaac7a61 100644 --- a/k8sutils/redis-cluster_test.go +++ b/k8sutils/redis-cluster_test.go @@ -5,6 +5,8 @@ import ( "path/filepath" "testing" + "k8s.io/client-go/kubernetes/fake" + common "github.com/OT-CONTAINER-KIT/redis-operator/api" redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2" "github.com/stretchr/testify/assert" @@ -427,10 +429,10 @@ func Test_generateRedisClusterContainerParams(t *testing.T) { t.Fatalf("Failed to unmarshal file %s: %v", path, err) } - actualLeaderContainer := generateRedisClusterContainerParams(input, input.Spec.RedisLeader.SecurityContext, input.Spec.RedisLeader.ReadinessProbe, input.Spec.RedisLeader.LivenessProbe, "leader") + actualLeaderContainer := generateRedisClusterContainerParams(input, input.Spec.RedisLeader.SecurityContext, input.Spec.RedisLeader.ReadinessProbe, input.Spec.RedisLeader.LivenessProbe, "leader", fake.NewSimpleClientset()) assert.EqualValues(t, expectedLeaderContainer, actualLeaderContainer, "Expected %+v, got %+v", expectedLeaderContainer, actualLeaderContainer) - actualFollowerContainer := generateRedisClusterContainerParams(input, input.Spec.RedisFollower.SecurityContext, input.Spec.RedisFollower.ReadinessProbe, input.Spec.RedisFollower.LivenessProbe, "follower") + actualFollowerContainer := generateRedisClusterContainerParams(input, input.Spec.RedisFollower.SecurityContext, input.Spec.RedisFollower.ReadinessProbe, input.Spec.RedisFollower.LivenessProbe, "follower", fake.NewSimpleClientset()) assert.EqualValues(t, expectedFollowerContainer, actualFollowerContainer, "Expected %+v, got %+v", expectedFollowerContainer, actualFollowerContainer) } diff --git a/k8sutils/redis-replication.go b/k8sutils/redis-replication.go index a0f0c7fd9..81803b10a 100644 --- a/k8sutils/redis-replication.go +++ b/k8sutils/redis-replication.go @@ -3,11 +3,12 @@ package k8sutils import ( redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2" "github.com/OT-CONTAINER-KIT/redis-operator/pkg/util" + "k8s.io/client-go/kubernetes" "k8s.io/utils/pointer" ) // CreateReplicationService method will create replication service for Redis -func CreateReplicationService(cr *redisv1beta2.RedisReplication) error { +func CreateReplicationService(cr *redisv1beta2.RedisReplication, cl kubernetes.Interface) error { logger := serviceLogger(cr.Namespace, cr.ObjectMeta.Name) labels := getRedisLabels(cr.ObjectMeta.Name, replication, "replication", cr.ObjectMeta.Labels) var epp exporterPortProvider @@ -27,12 +28,12 @@ func CreateReplicationService(cr *redisv1beta2.RedisReplication) error { objectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name, cr.Namespace, labels, annotations) headlessObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-headless", cr.Namespace, labels, annotations) additionalObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-additional", cr.Namespace, labels, generateServiceAnots(cr.ObjectMeta, additionalServiceAnnotations, epp)) - err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisReplicationAsOwner(cr), disableMetrics, true, "ClusterIP", redisPort) + err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisReplicationAsOwner(cr), disableMetrics, true, "ClusterIP", redisPort, cl) if err != nil { logger.Error(err, "Cannot create replication headless service for Redis") return err } - err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisReplicationAsOwner(cr), epp, false, "ClusterIP", redisPort) + err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisReplicationAsOwner(cr), epp, false, "ClusterIP", redisPort, cl) if err != nil { logger.Error(err, "Cannot create replication service for Redis") return err @@ -41,7 +42,7 @@ func CreateReplicationService(cr *redisv1beta2.RedisReplication) error { if cr.Spec.KubernetesConfig.Service != nil { additionalServiceType = cr.Spec.KubernetesConfig.Service.ServiceType } - err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisReplicationAsOwner(cr), disableMetrics, false, additionalServiceType, redisPort) + err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisReplicationAsOwner(cr), disableMetrics, false, additionalServiceType, redisPort, cl) if err != nil { logger.Error(err, "Cannot create additional service for Redis Replication") return err @@ -50,7 +51,7 @@ func CreateReplicationService(cr *redisv1beta2.RedisReplication) error { } // CreateReplicationRedis will create a replication redis setup -func CreateReplicationRedis(cr *redisv1beta2.RedisReplication) error { +func CreateReplicationRedis(cr *redisv1beta2.RedisReplication, cl kubernetes.Interface) error { stateFulName := cr.ObjectMeta.Name logger := statefulSetLogger(cr.Namespace, cr.ObjectMeta.Name) labels := getRedisLabels(cr.ObjectMeta.Name, replication, "replication", cr.ObjectMeta.Labels) @@ -63,6 +64,7 @@ func CreateReplicationRedis(cr *redisv1beta2.RedisReplication) error { generateRedisReplicationInitContainerParams(cr), generateRedisReplicationContainerParams(cr), cr.Spec.Sidecars, + cl, ) if err != nil { logger.Error(err, "Cannot create replication statefulset for Redis") diff --git a/k8sutils/redis-sentinel.go b/k8sutils/redis-sentinel.go index fdb3dcb84..5ae9dbef1 100644 --- a/k8sutils/redis-sentinel.go +++ b/k8sutils/redis-sentinel.go @@ -37,7 +37,7 @@ type RedisReplicationObject struct { } // Redis Sentinel Create the Redis Sentinel Setup -func CreateRedisSentinel(ctx context.Context, client kubernetes.Interface, logger logr.Logger, cr *redisv1beta2.RedisSentinel) error { +func CreateRedisSentinel(ctx context.Context, client kubernetes.Interface, logger logr.Logger, cr *redisv1beta2.RedisSentinel, cl kubernetes.Interface) error { prop := RedisSentinelSTS{ RedisStateFulType: "sentinel", Affinity: cr.Spec.Affinity, @@ -50,21 +50,21 @@ func CreateRedisSentinel(ctx context.Context, client kubernetes.Interface, logge prop.ExternalConfig = cr.Spec.RedisSentinelConfig.AdditionalSentinelConfig } - return prop.CreateRedisSentinelSetup(ctx, client, logger, cr) + return prop.CreateRedisSentinelSetup(ctx, client, logger, cr, cl) } // Create RedisSentinel Service -func CreateRedisSentinelService(cr *redisv1beta2.RedisSentinel) error { +func CreateRedisSentinelService(cr *redisv1beta2.RedisSentinel, cl kubernetes.Interface) error { prop := RedisSentinelService{ RedisServiceRole: "sentinel", } - return prop.CreateRedisSentinelService(cr) + return prop.CreateRedisSentinelService(cr, cl) } // Create Redis Sentinel Cluster Setup -func (service RedisSentinelSTS) CreateRedisSentinelSetup(ctx context.Context, client kubernetes.Interface, logger logr.Logger, cr *redisv1beta2.RedisSentinel) error { +func (service RedisSentinelSTS) CreateRedisSentinelSetup(ctx context.Context, client kubernetes.Interface, logger logr.Logger, cr *redisv1beta2.RedisSentinel, cl kubernetes.Interface) error { stateFulName := cr.ObjectMeta.Name + "-" + service.RedisStateFulType labels := getRedisLabels(stateFulName, sentinel, service.RedisStateFulType, cr.ObjectMeta.Labels) annotations := generateStatefulSetsAnots(cr.ObjectMeta, cr.Spec.KubernetesConfig.IgnoreAnnotations) @@ -77,6 +77,7 @@ func (service RedisSentinelSTS) CreateRedisSentinelSetup(ctx context.Context, cl generateRedisSentinelInitContainerParams(cr), generateRedisSentinelContainerParams(ctx, client, logger, cr, service.ReadinessProbe, service.LivenessProbe), cr.Spec.Sidecars, + cl, ) if err != nil { @@ -197,7 +198,7 @@ func (service RedisSentinelSTS) getSentinelCount(cr *redisv1beta2.RedisSentinel) } // Create the Service for redis sentinel -func (service RedisSentinelService) CreateRedisSentinelService(cr *redisv1beta2.RedisSentinel) error { +func (service RedisSentinelService) CreateRedisSentinelService(cr *redisv1beta2.RedisSentinel, cl kubernetes.Interface) error { serviceName := cr.ObjectMeta.Name + "-" + service.RedisServiceRole logger := serviceLogger(cr.Namespace, serviceName) labels := getRedisLabels(serviceName, sentinel, service.RedisServiceRole, cr.ObjectMeta.Labels) @@ -220,12 +221,12 @@ func (service RedisSentinelService) CreateRedisSentinelService(cr *redisv1beta2. headlessObjectMetaInfo := generateObjectMetaInformation(serviceName+"-headless", cr.Namespace, labels, annotations) additionalObjectMetaInfo := generateObjectMetaInformation(serviceName+"-additional", cr.Namespace, labels, generateServiceAnots(cr.ObjectMeta, additionalServiceAnnotations, epp)) - err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisSentinelAsOwner(cr), disableMetrics, true, "ClusterIP", sentinelPort) + err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisSentinelAsOwner(cr), disableMetrics, true, "ClusterIP", sentinelPort, cl) if err != nil { logger.Error(err, "Cannot create headless service for Redis", "Setup.Type", service.RedisServiceRole) return err } - err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisSentinelAsOwner(cr), epp, false, "ClusterIP", sentinelPort) + err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisSentinelAsOwner(cr), epp, false, "ClusterIP", sentinelPort, cl) if err != nil { logger.Error(err, "Cannot create service for Redis", "Setup.Type", service.RedisServiceRole) return err @@ -235,7 +236,7 @@ func (service RedisSentinelService) CreateRedisSentinelService(cr *redisv1beta2. if cr.Spec.KubernetesConfig.Service != nil { additionalServiceType = cr.Spec.KubernetesConfig.Service.ServiceType } - err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisSentinelAsOwner(cr), disableMetrics, false, additionalServiceType, sentinelPort) + err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisSentinelAsOwner(cr), disableMetrics, false, additionalServiceType, sentinelPort, cl) if err != nil { logger.Error(err, "Cannot create additional service for Redis", "Setup.Type", service.RedisServiceRole) return err diff --git a/k8sutils/redis-standalone.go b/k8sutils/redis-standalone.go index 4addf77be..c47e6a934 100644 --- a/k8sutils/redis-standalone.go +++ b/k8sutils/redis-standalone.go @@ -3,6 +3,7 @@ package k8sutils import ( redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2" "github.com/OT-CONTAINER-KIT/redis-operator/pkg/util" + "k8s.io/client-go/kubernetes" "k8s.io/utils/pointer" ) @@ -11,7 +12,7 @@ import ( //) // CreateStandaloneService method will create standalone service for Redis -func CreateStandaloneService(cr *redisv1beta2.Redis) error { +func CreateStandaloneService(cr *redisv1beta2.Redis, cl kubernetes.Interface) error { logger := serviceLogger(cr.Namespace, cr.ObjectMeta.Name) labels := getRedisLabels(cr.ObjectMeta.Name, standalone, "standalone", cr.ObjectMeta.Labels) var epp exporterPortProvider @@ -31,12 +32,12 @@ func CreateStandaloneService(cr *redisv1beta2.Redis) error { objectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name, cr.Namespace, labels, annotations) headlessObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-headless", cr.Namespace, labels, annotations) additionalObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-additional", cr.Namespace, labels, generateServiceAnots(cr.ObjectMeta, additionalServiceAnnotations, epp)) - err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisAsOwner(cr), disableMetrics, true, "ClusterIP", redisPort) + err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisAsOwner(cr), disableMetrics, true, "ClusterIP", redisPort, cl) if err != nil { logger.Error(err, "Cannot create standalone headless service for Redis") return err } - err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisAsOwner(cr), epp, false, "ClusterIP", redisPort) + err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisAsOwner(cr), epp, false, "ClusterIP", redisPort, cl) if err != nil { logger.Error(err, "Cannot create standalone service for Redis") return err @@ -45,7 +46,7 @@ func CreateStandaloneService(cr *redisv1beta2.Redis) error { if cr.Spec.KubernetesConfig.Service != nil { additionalServiceType = cr.Spec.KubernetesConfig.Service.ServiceType } - err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisAsOwner(cr), disableMetrics, false, additionalServiceType, redisPort) + err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisAsOwner(cr), disableMetrics, false, additionalServiceType, redisPort, cl) if err != nil { logger.Error(err, "Cannot create additional service for Redis") return err @@ -54,7 +55,7 @@ func CreateStandaloneService(cr *redisv1beta2.Redis) error { } // CreateStandaloneRedis will create a standalone redis setup -func CreateStandaloneRedis(cr *redisv1beta2.Redis) error { +func CreateStandaloneRedis(cr *redisv1beta2.Redis, cl kubernetes.Interface) error { logger := statefulSetLogger(cr.Namespace, cr.ObjectMeta.Name) labels := getRedisLabels(cr.ObjectMeta.Name, standalone, "standalone", cr.ObjectMeta.Labels) annotations := generateStatefulSetsAnots(cr.ObjectMeta, cr.Spec.KubernetesConfig.IgnoreAnnotations) @@ -66,6 +67,7 @@ func CreateStandaloneRedis(cr *redisv1beta2.Redis) error { generateRedisStandaloneInitContainerParams(cr), generateRedisStandaloneContainerParams(cr), cr.Spec.Sidecars, + cl, ) if err != nil { logger.Error(err, "Cannot create standalone statefulset for Redis") diff --git a/k8sutils/redis.go b/k8sutils/redis.go index fa1986bd6..221917c13 100644 --- a/k8sutils/redis.go +++ b/k8sutils/redis.go @@ -476,8 +476,8 @@ func configureRedisReplicationClient(client kubernetes.Interface, logger logr.Lo } // Get Redis nodes by it's role i.e. master, slave and sentinel -func GetRedisNodesByRole(ctx context.Context, client kubernetes.Interface, logger logr.Logger, cr *redisv1beta2.RedisReplication, redisRole string) []string { - statefulset, err := GetStatefulSet(cr.Namespace, cr.Name) +func GetRedisNodesByRole(ctx context.Context, cl kubernetes.Interface, logger logr.Logger, cr *redisv1beta2.RedisReplication, redisRole string) []string { + statefulset, err := GetStatefulSet(cr.Namespace, cr.Name, cl) if err != nil { logger.Error(err, "Failed to Get the Statefulset of the", "custom resource", cr.Name, "in namespace", cr.Namespace) } @@ -488,7 +488,7 @@ func GetRedisNodesByRole(ctx context.Context, client kubernetes.Interface, logge for i := 0; i < int(replicas); i++ { podName := statefulset.Name + "-" + strconv.Itoa(i) - podRole := checkRedisServerRole(ctx, client, logger, cr, podName) + podRole := checkRedisServerRole(ctx, cl, logger, cr, podName) if podRole == redisRole { pods = append(pods, podName) } diff --git a/k8sutils/services.go b/k8sutils/services.go index c00e2c0a5..353e68f58 100644 --- a/k8sutils/services.go +++ b/k8sutils/services.go @@ -9,6 +9,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes" ) const ( @@ -95,14 +96,9 @@ func generateServiceType(k8sServiceType string) corev1.ServiceType { } // createService is a method to create service is Kubernetes -func createService(namespace string, service *corev1.Service) error { +func createService(namespace string, service *corev1.Service, cl kubernetes.Interface) error { logger := serviceLogger(namespace, service.Name) - client, err := GenerateK8sClient(GenerateK8sConfig) - if err != nil { - logger.Error(err, "Could not generate kubernetes client") - return err - } - _, err = client.CoreV1().Services(namespace).Create(context.TODO(), service, metav1.CreateOptions{}) + _, err := cl.CoreV1().Services(namespace).Create(context.TODO(), service, metav1.CreateOptions{}) if err != nil { logger.Error(err, "Redis service creation is failed") return err @@ -112,14 +108,9 @@ func createService(namespace string, service *corev1.Service) error { } // updateService is a method to update service is Kubernetes -func updateService(namespace string, service *corev1.Service) error { +func updateService(namespace string, service *corev1.Service, cl kubernetes.Interface) error { logger := serviceLogger(namespace, service.Name) - client, err := GenerateK8sClient(GenerateK8sConfig) - if err != nil { - logger.Error(err, "Could not generate kubernetes client") - return err - } - _, err = client.CoreV1().Services(namespace).Update(context.TODO(), service, metav1.UpdateOptions{}) + _, err := cl.CoreV1().Services(namespace).Update(context.TODO(), service, metav1.UpdateOptions{}) if err != nil { logger.Error(err, "Redis service update failed") return err @@ -129,17 +120,12 @@ func updateService(namespace string, service *corev1.Service) error { } // getService is a method to get service is Kubernetes -func getService(namespace string, service string) (*corev1.Service, error) { +func getService(namespace string, service string, cl kubernetes.Interface) (*corev1.Service, error) { logger := serviceLogger(namespace, service) - client, err := GenerateK8sClient(GenerateK8sConfig) - if err != nil { - logger.Error(err, "Could not generate kubernetes client") - return nil, err - } getOpts := metav1.GetOptions{ TypeMeta: generateMetaInformation("Service", "v1"), } - serviceInfo, err := client.CoreV1().Services(namespace).Get(context.TODO(), service, getOpts) + serviceInfo, err := cl.CoreV1().Services(namespace).Get(context.TODO(), service, getOpts) if err != nil { logger.V(1).Info("Redis service get action is failed") return nil, err @@ -154,24 +140,24 @@ func serviceLogger(namespace string, name string) logr.Logger { } // CreateOrUpdateService method will create or update Redis service -func CreateOrUpdateService(namespace string, serviceMeta metav1.ObjectMeta, ownerDef metav1.OwnerReference, epp exporterPortProvider, headless bool, serviceType string, port int, extra ...corev1.ServicePort) error { +func CreateOrUpdateService(namespace string, serviceMeta metav1.ObjectMeta, ownerDef metav1.OwnerReference, epp exporterPortProvider, headless bool, serviceType string, port int, cl kubernetes.Interface, extra ...corev1.ServicePort) error { logger := serviceLogger(namespace, serviceMeta.Name) serviceDef := generateServiceDef(serviceMeta, epp, ownerDef, headless, serviceType, port, extra...) - storedService, err := getService(namespace, serviceMeta.Name) + storedService, err := getService(namespace, serviceMeta.Name, cl) if err != nil { if errors.IsNotFound(err) { if err := patch.DefaultAnnotator.SetLastAppliedAnnotation(serviceDef); err != nil { //nolint logger.Error(err, "Unable to patch redis service with compare annotations") } - return createService(namespace, serviceDef) + return createService(namespace, serviceDef, cl) } return err } - return patchService(storedService, serviceDef, namespace) + return patchService(storedService, serviceDef, namespace, cl) } // patchService will patch Redis Kubernetes service -func patchService(storedService *corev1.Service, newService *corev1.Service, namespace string) error { +func patchService(storedService *corev1.Service, newService *corev1.Service, namespace string, cl kubernetes.Interface) error { logger := serviceLogger(namespace, storedService.Name) // We want to try and keep this atomic as possible. newService.ResourceVersion = storedService.ResourceVersion @@ -204,7 +190,7 @@ func patchService(storedService *corev1.Service, newService *corev1.Service, nam return err } logger.V(1).Info("Syncing Redis service with defined properties") - return updateService(namespace, newService) + return updateService(namespace, newService, cl) } logger.V(1).Info("Redis service is already in-sync") return nil diff --git a/k8sutils/statefulset.go b/k8sutils/statefulset.go index f1caa2891..67f394a6e 100644 --- a/k8sutils/statefulset.go +++ b/k8sutils/statefulset.go @@ -3,13 +3,14 @@ package k8sutils import ( "context" "fmt" - "k8s.io/utils/env" "path" "sort" "strconv" "strings" "github.com/OT-CONTAINER-KIT/redis-operator/pkg/util" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/env" "k8s.io/utils/pointer" commonapi "github.com/OT-CONTAINER-KIT/redis-operator/api" @@ -95,9 +96,9 @@ type initContainerParameters struct { } // CreateOrUpdateStateFul method will create or update Redis service -func CreateOrUpdateStateFul(namespace string, stsMeta metav1.ObjectMeta, params statefulSetParameters, ownerDef metav1.OwnerReference, initcontainerParams initContainerParameters, containerParams containerParameters, sidecars *[]redisv1beta2.Sidecar) error { +func CreateOrUpdateStateFul(namespace string, stsMeta metav1.ObjectMeta, params statefulSetParameters, ownerDef metav1.OwnerReference, initcontainerParams initContainerParameters, containerParams containerParameters, sidecars *[]redisv1beta2.Sidecar, cl kubernetes.Interface) error { logger := statefulSetLogger(namespace, stsMeta.Name) - storedStateful, err := GetStatefulSet(namespace, stsMeta.Name) + storedStateful, err := GetStatefulSet(namespace, stsMeta.Name, cl) statefulSetDef := generateStatefulSetsDef(stsMeta, params, ownerDef, initcontainerParams, containerParams, getSidecars(sidecars)) if err != nil { if err := patch.DefaultAnnotator.SetLastAppliedAnnotation(statefulSetDef); err != nil { //nolint @@ -105,21 +106,16 @@ func CreateOrUpdateStateFul(namespace string, stsMeta metav1.ObjectMeta, params return err } if apierrors.IsNotFound(err) { - return createStatefulSet(namespace, statefulSetDef) + return createStatefulSet(namespace, statefulSetDef, cl) } return err } - return patchStatefulSet(storedStateful, statefulSetDef, namespace, params.RecreateStatefulSet) + return patchStatefulSet(storedStateful, statefulSetDef, namespace, params.RecreateStatefulSet, cl) } // patchStateFulSet will patch Redis Kubernetes StateFulSet -func patchStatefulSet(storedStateful *appsv1.StatefulSet, newStateful *appsv1.StatefulSet, namespace string, recreateStateFulSet bool) error { +func patchStatefulSet(storedStateful *appsv1.StatefulSet, newStateful *appsv1.StatefulSet, namespace string, recreateStateFulSet bool, cl kubernetes.Interface) error { logger := statefulSetLogger(namespace, storedStateful.Name) - client, err := GenerateK8sClient(GenerateK8sConfig) - if err != nil { - logger.Error(err, "Could not generate kubernetes client") - return err - } // We want to try and keep this atomic as possible. newStateful.ResourceVersion = storedStateful.ResourceVersion newStateful.CreationTimestamp = storedStateful.CreationTimestamp @@ -167,7 +163,7 @@ func patchStatefulSet(storedStateful *appsv1.StatefulSet, newStateful *appsv1.St }, ), } - pvcs, err := client.CoreV1().PersistentVolumeClaims(storedStateful.Namespace).List(context.Background(), listOpt) + pvcs, err := cl.CoreV1().PersistentVolumeClaims(storedStateful.Namespace).List(context.Background(), listOpt) if err != nil { return err } @@ -178,7 +174,7 @@ func patchStatefulSet(storedStateful *appsv1.StatefulSet, newStateful *appsv1.St if realCapacity != stateCapacity { realUpdate = true pvc.Spec.Resources.Requests = newStateful.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests - _, err = client.CoreV1().PersistentVolumeClaims(storedStateful.Namespace).Update(context.Background(), &pvc, metav1.UpdateOptions{}) + _, err = cl.CoreV1().PersistentVolumeClaims(storedStateful.Namespace).Update(context.Background(), &pvc, metav1.UpdateOptions{}) if err != nil { if !updateFailed { updateFailed = true @@ -213,7 +209,7 @@ func patchStatefulSet(storedStateful *appsv1.StatefulSet, newStateful *appsv1.St logger.Error(err, "Unable to patch redis statefulset with comparison object") return err } - return updateStatefulSet(namespace, newStateful, recreateStateFulSet) + return updateStatefulSet(namespace, newStateful, recreateStateFulSet, cl) } logger.V(1).Info("Reconciliation Complete, no Changes required.") return nil @@ -690,14 +686,9 @@ func getEnvironmentVariables(role string, enabledPassword *bool, secretName *str } // createStatefulSet is a method to create statefulset in Kubernetes -func createStatefulSet(namespace string, stateful *appsv1.StatefulSet) error { - logger := statefulSetLogger(namespace, stateful.Name) - client, err := GenerateK8sClient(GenerateK8sConfig) - if err != nil { - logger.Error(err, "Could not generate kubernetes client") - return err - } - _, err = client.AppsV1().StatefulSets(namespace).Create(context.TODO(), stateful, metav1.CreateOptions{}) +func createStatefulSet(namespace string, stateful *appsv1.StatefulSet, cl kubernetes.Interface) error { + logger := statefulSetLogger(stateful.Namespace, stateful.Name) + _, err := cl.AppsV1().StatefulSets(namespace).Create(context.TODO(), stateful, metav1.CreateOptions{}) if err != nil { logger.Error(err, "Redis stateful creation failed") return err @@ -707,14 +698,9 @@ func createStatefulSet(namespace string, stateful *appsv1.StatefulSet) error { } // updateStatefulSet is a method to update statefulset in Kubernetes -func updateStatefulSet(namespace string, stateful *appsv1.StatefulSet, recreateStateFulSet bool) error { +func updateStatefulSet(namespace string, stateful *appsv1.StatefulSet, recreateStateFulSet bool, cl kubernetes.Interface) error { logger := statefulSetLogger(namespace, stateful.Name) - client, err := GenerateK8sClient(GenerateK8sConfig) - if err != nil { - logger.Error(err, "Could not generate kubernetes client") - return err - } - _, err = client.AppsV1().StatefulSets(namespace).Update(context.TODO(), stateful, metav1.UpdateOptions{}) + _, err := cl.AppsV1().StatefulSets(namespace).Update(context.TODO(), stateful, metav1.UpdateOptions{}) if recreateStateFulSet { sErr, ok := err.(*apierrors.StatusError) if ok && sErr.ErrStatus.Code == 422 && sErr.ErrStatus.Reason == metav1.StatusReasonInvalid { @@ -724,7 +710,7 @@ func updateStatefulSet(namespace string, stateful *appsv1.StatefulSet, recreateS } logger.V(1).Info("recreating StatefulSet because the update operation wasn't possible", "reason", strings.Join(failMsg, ", ")) propagationPolicy := metav1.DeletePropagationForeground - if err := client.AppsV1().StatefulSets(namespace).Delete(context.TODO(), stateful.GetName(), metav1.DeleteOptions{PropagationPolicy: &propagationPolicy}); err != nil { //nolint + if err := cl.AppsV1().StatefulSets(namespace).Delete(context.TODO(), stateful.GetName(), metav1.DeleteOptions{PropagationPolicy: &propagationPolicy}); err != nil { //nolint return errors.Wrap(err, "failed to delete StatefulSet to avoid forbidden action") } } @@ -738,17 +724,12 @@ func updateStatefulSet(namespace string, stateful *appsv1.StatefulSet, recreateS } // GetStateFulSet is a method to get statefulset in Kubernetes -func GetStatefulSet(namespace string, stateful string) (*appsv1.StatefulSet, error) { +func GetStatefulSet(namespace string, stateful string, cl kubernetes.Interface) (*appsv1.StatefulSet, error) { logger := statefulSetLogger(namespace, stateful) - client, err := GenerateK8sClient(GenerateK8sConfig) - if err != nil { - logger.Error(err, "Could not generate kubernetes client") - return nil, err - } getOpts := metav1.GetOptions{ TypeMeta: generateMetaInformation("StatefulSet", "apps/v1"), } - statefulInfo, err := client.AppsV1().StatefulSets(namespace).Get(context.TODO(), stateful, getOpts) + statefulInfo, err := cl.AppsV1().StatefulSets(namespace).Get(context.TODO(), stateful, getOpts) if err != nil { logger.V(1).Info("Redis statefulset get action failed") return nil, err diff --git a/tests/readme.md b/tests/readme.md index f4057b698..c4c91cd86 100644 --- a/tests/readme.md +++ b/tests/readme.md @@ -35,7 +35,7 @@ Please refer to the repository's README for detailed instructions on installing Execute the kuttl test using the following command: -To run all default tests ( _config/kuttl-test.yaml is the default config file ) +To run all default tests ( \_config/kuttl-test.yaml is the default config file ) ```bash kubectl kuttl test --config tests/_config/kuttl-test.yaml