From 567454d8e0c313e6e14fe77662e38ccd515ad92b Mon Sep 17 00:00:00 2001 From: Shubham Gupta <69793468+shubham-cmyk@users.noreply.github.com> Date: Fri, 10 Nov 2023 20:45:57 +0530 Subject: [PATCH] test: Make Code Testable (#697) * testing-client Signed-off-by: Shubham Gupta * client_test.go Signed-off-by: Shubham Gupta * fix Signed-off-by: Shubham Gupta * push Signed-off-by: Shubham Gupta * fix Client Signed-off-by: Shubham Gupta * fix Dockerfile Signed-off-by: Shubham Gupta * try fixing lint Signed-off-by: Shubham Gupta * fix ci Signed-off-by: Shubham Gupta * fix CI Signed-off-by: Shubham Gupta * fix CI Signed-off-by: Shubham Gupta * ignoree Signed-off-by: Shubham Gupta * change location Signed-off-by: Shubham Gupta --------- Signed-off-by: Shubham Gupta Signed-off-by: Matt Robinson --- .github/workflows/operator-ci.yaml | 16 +++- .golangci.yml | 12 +-- Dockerfile | 1 + controllers/redis_controller.go | 10 ++- k8sutils/client.go | 34 ++++---- k8sutils/client_test.go | 78 ++++++++++++++++++ k8sutils/finalizer.go | 37 ++++++--- k8sutils/finalizers_test.go | 122 +++++++++++++++++++++++++++++ k8sutils/poddisruption.go | 28 ++++++- k8sutils/redis-sentinel.go | 7 +- k8sutils/redis.go | 23 +++++- k8sutils/secrets.go | 19 ++++- k8sutils/services.go | 21 ++++- k8sutils/statefulset.go | 33 ++++++-- k8sutils/status.go | 2 +- main.go | 21 ++++- mocks/log/logger.go | 5 ++ 17 files changed, 399 insertions(+), 70 deletions(-) create mode 100644 k8sutils/client_test.go create mode 100644 k8sutils/finalizers_test.go create mode 100644 mocks/log/logger.go diff --git a/.github/workflows/operator-ci.yaml b/.github/workflows/operator-ci.yaml index 205339b76..f9322b585 100644 --- a/.github/workflows/operator-ci.yaml +++ b/.github/workflows/operator-ci.yaml @@ -64,7 +64,7 @@ jobs: verbose: true code_quality_golang_ci_lint: - needs: [gofmt, govet, gotest] + needs: [gofmt, govet] runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -72,14 +72,24 @@ jobs: uses: actions/setup-go@v4 with: go-version: ${{ env.GOLANG_VERSION }} + + - name: Download Go modules + run: go mod download + + - name: Check disk space + run: df -h + + - name: List Go module cache + run: ls -la $(go env GOPATH)/pkg/mod + - name: Run GolangCI-Lint uses: golangci/golangci-lint-action@v3 with: version: v1.54.0 - args: --timeout=5m0s ./... + args: --timeout=5m0s ./... --verbose --out-format=github-actions container_quality_dockerfile_lint: - needs: [gofmt, govet, gotest] + needs: [gofmt, govet] runs-on: ubuntu-latest steps: - name: Checkout code diff --git a/.golangci.yml b/.golangci.yml index 2a38a4fa8..48fc1e0d6 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -6,11 +6,13 @@ linters-settings: gofmt: simplify: true -# linters: -# enable-all: true -# disable: -# - errcheck -# - tagliatelle +linters: + disable: + - errcheck + - tagliatelle + - ineffassign + - staticcheck + # Exclude the files that are causing the errors issues: exclude-rules: diff --git a/Dockerfile b/Dockerfile index 2bf6f87f0..f953b9f25 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,6 +22,7 @@ COPY main.go main.go COPY api/ api/ COPY controllers/ controllers/ COPY k8sutils/ k8sutils/ +COPY mocks/ mocks/ # Build ARG LDFLAGS="-s -w" diff --git a/controllers/redis_controller.go b/controllers/redis_controller.go index 9b07469f8..ee7224ade 100644 --- a/controllers/redis_controller.go +++ b/controllers/redis_controller.go @@ -25,6 +25,8 @@ import ( "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -32,8 +34,10 @@ import ( // RedisReconciler reconciles a Redis object type RedisReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme + K8sClient kubernetes.Interface + Dk8sClinet dynamic.Interface + Log logr.Logger + Scheme *runtime.Scheme } // Reconcile is part of the main kubernetes reconciliation loop which aims @@ -53,7 +57,7 @@ func (r *RedisReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl reqLogger.Info("Found annotations redis.opstreelabs.in/skip-reconcile, so skipping reconcile") return ctrl.Result{RequeueAfter: time.Second * 10}, nil } - if err = k8sutils.HandleRedisFinalizer(instance, r.Client); err != nil { + if err = k8sutils.HandleRedisFinalizer(r.Client, r.K8sClient, r.Log, instance); err != nil { return ctrl.Result{}, err } diff --git a/k8sutils/client.go b/k8sutils/client.go index 54078ab7e..a12031004 100644 --- a/k8sutils/client.go +++ b/k8sutils/client.go @@ -8,34 +8,28 @@ import ( "k8s.io/client-go/tools/clientcmd" ) -// generateK8sClient create client for kubernetes -func generateK8sClient() *kubernetes.Clientset { - config, err := generateK8sConfig() - if err != nil { - panic(err.Error()) - } - clientset, err := kubernetes.NewForConfig(config) +type K8sConfigProvider = func() (*rest.Config, error) + +// GenerateK8sClient create client for kubernetes +func GenerateK8sClient(configProvider K8sConfigProvider) (kubernetes.Interface, error) { + config, err := configProvider() if err != nil { - panic(err.Error()) + return nil, err } - return clientset + return kubernetes.NewForConfig(config) } -// generateK8sClient create Dynamic client for kubernetes -func generateK8sDynamicClient() dynamic.Interface { - config, err := generateK8sConfig() - if err != nil { - panic(err.Error()) - } - dynamicClientset, err := dynamic.NewForConfig(config) +// GenerateK8sClient create Dynamic client for kubernetes +func GenerateK8sDynamicClient(configProvider K8sConfigProvider) (dynamic.Interface, error) { + config, err := configProvider() if err != nil { - panic(err.Error()) + return nil, err } - return dynamicClientset + return dynamic.NewForConfig(config) } -// generateK8sConfig will load the kube config file -func generateK8sConfig() (*rest.Config, error) { +// GenerateK8sConfig will load the kube config file +func GenerateK8sConfig() (*rest.Config, error) { loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() // if you want to change the loading rules (which files in which order), you can do so here configOverrides := &clientcmd.ConfigOverrides{} diff --git a/k8sutils/client_test.go b/k8sutils/client_test.go new file mode 100644 index 000000000..13618dc70 --- /dev/null +++ b/k8sutils/client_test.go @@ -0,0 +1,78 @@ +package k8sutils + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/client-go/rest" +) + +func mockK8sConfigProvider() (*rest.Config, error) { + return &rest.Config{}, nil +} +func mockInvalidK8sConfigProvider() (*rest.Config, error) { + return nil, errors.New("invalid configuration") +} + +func TestGenerateK8sClient(t *testing.T) { + tests := []struct { + name string + configProvider func() (*rest.Config, error) + wantErr bool + }{ + { + name: "valid config", + configProvider: mockK8sConfigProvider, + wantErr: false, + }, + { + name: "invalid config", + configProvider: mockInvalidK8sConfigProvider, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client, err := GenerateK8sClient(tt.configProvider) + if tt.wantErr { + assert.Error(t, err, "GenerateK8sClient() should return an error for invalid config") + } else { + assert.NoError(t, err, "GenerateK8sClient() should not return an error for valid config") + assert.NotNil(t, client, "expected a non-nil Kubernetes client") + } + }) + } +} + +func TestGenerateK8sDynamicClient(t *testing.T) { + tests := []struct { + name string + configProvider func() (*rest.Config, error) + wantErr bool + }{ + { + name: "valid config", + configProvider: mockK8sConfigProvider, + wantErr: false, + }, + { + name: "invalid config", + configProvider: mockInvalidK8sConfigProvider, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client, err := GenerateK8sDynamicClient(tt.configProvider) + if tt.wantErr { + assert.Error(t, err, "GenerateK8sDynamicClient() should return an error for invalid config") + } else { + assert.NoError(t, err, "GenerateK8sDynamicClient() should not return an error for valid config") + assert.NotNil(t, client, "expected a non-nil Kubernetes client") + } + }) + } +} diff --git a/k8sutils/finalizer.go b/k8sutils/finalizer.go index e0e6c74bd..cb68d0c1b 100644 --- a/k8sutils/finalizer.go +++ b/k8sutils/finalizer.go @@ -2,12 +2,15 @@ package k8sutils import ( "context" + "fmt" "strconv" redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2" + mockLog "github.com/OT-CONTAINER-KIT/redis-operator/mocks/log" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -26,16 +29,15 @@ func finalizerLogger(namespace string, name string) logr.Logger { } // HandleRedisFinalizer finalize resource if instance is marked to be deleted -func HandleRedisFinalizer(cr *redisv1beta2.Redis, cl client.Client) error { - logger := finalizerLogger(cr.Namespace, RedisFinalizer) +func HandleRedisFinalizer(ctrlclient client.Client, k8sClient kubernetes.Interface, logger logr.Logger, cr *redisv1beta2.Redis) error { if cr.GetDeletionTimestamp() != nil { if controllerutil.ContainsFinalizer(cr, RedisFinalizer) { - if err := finalizeRedisPVC(cr); err != nil { + if err := finalizeRedisPVC(k8sClient, logger, cr); err != nil { return err } controllerutil.RemoveFinalizer(cr, RedisFinalizer) - if err := cl.Update(context.TODO(), cr); err != nil { - logger.Error(err, "Could not remove finalizer "+RedisFinalizer) + if err := ctrlclient.Update(context.TODO(), cr); err != nil { + logger.Error(err, "Could not remove finalizer", "finalizer", RedisFinalizer) return err } } @@ -134,12 +136,11 @@ func AddRedisSentinelFinalizer(cr *redisv1beta2.RedisSentinel, cl client.Client) } // finalizeRedisPVC delete PVC -func finalizeRedisPVC(cr *redisv1beta2.Redis) error { - logger := finalizerLogger(cr.Namespace, RedisFinalizer) - PVCName := cr.Name + "-" + cr.Name + "-0" - err := generateK8sClient().CoreV1().PersistentVolumeClaims(cr.Namespace).Delete(context.TODO(), PVCName, metav1.DeleteOptions{}) +func finalizeRedisPVC(client kubernetes.Interface, logger mockLog.LoggerInterface, cr *redisv1beta2.Redis) error { + PVCName := fmt.Sprintf("%s-%s-0", cr.Name, cr.Name) + err := client.CoreV1().PersistentVolumeClaims(cr.Namespace).Delete(context.TODO(), PVCName, metav1.DeleteOptions{}) if err != nil && !errors.IsNotFound(err) { - logger.Error(err, "Could not delete Persistent Volume Claim "+PVCName) + logger.Error(err, "Could not delete Persistent Volume Claim", "PVCName", PVCName) return err } return nil @@ -148,10 +149,15 @@ func finalizeRedisPVC(cr *redisv1beta2.Redis) error { // finalizeRedisClusterPVC delete PVCs func finalizeRedisClusterPVC(cr *redisv1beta2.RedisCluster) error { logger := finalizerLogger(cr.Namespace, RedisClusterFinalizer) + client, err := GenerateK8sClient(GenerateK8sConfig) + if err != nil { + logger.Error(err, "Could not generate kubernetes client") + return err + } for _, role := range []string{"leader", "follower"} { for i := 0; i < int(cr.Spec.GetReplicaCounts(role)); i++ { PVCName := cr.Name + "-" + cr.Name + "-" + role + "-" + strconv.Itoa(i) - err := generateK8sClient().CoreV1().PersistentVolumeClaims(cr.Namespace).Delete(context.TODO(), PVCName, metav1.DeleteOptions{}) + err := client.CoreV1().PersistentVolumeClaims(cr.Namespace).Delete(context.TODO(), PVCName, metav1.DeleteOptions{}) if err != nil && !errors.IsNotFound(err) { logger.Error(err, "Could not delete Persistent Volume Claim "+PVCName) return err @@ -160,7 +166,7 @@ func finalizeRedisClusterPVC(cr *redisv1beta2.RedisCluster) error { if cr.Spec.Storage.NodeConfVolume { for i := 0; i < int(cr.Spec.GetReplicaCounts(role)); i++ { PVCName := "node-conf" + cr.Name + "-" + role + "-" + strconv.Itoa(i) - err := generateK8sClient().CoreV1().PersistentVolumeClaims(cr.Namespace).Delete(context.TODO(), PVCName, metav1.DeleteOptions{}) + err := client.CoreV1().PersistentVolumeClaims(cr.Namespace).Delete(context.TODO(), PVCName, metav1.DeleteOptions{}) if err != nil && !errors.IsNotFound(err) { logger.Error(err, "Could not delete Persistent Volume Claim "+PVCName) return err @@ -175,9 +181,14 @@ func finalizeRedisClusterPVC(cr *redisv1beta2.RedisCluster) error { // finalizeRedisReplicationPVC delete PVCs func finalizeRedisReplicationPVC(cr *redisv1beta2.RedisReplication) error { logger := finalizerLogger(cr.Namespace, RedisReplicationFinalizer) + client, err := GenerateK8sClient(GenerateK8sConfig) + if err != nil { + logger.Error(err, "Could not generate kubernetes client") + return err + } for i := 0; i < int(cr.Spec.GetReplicationCounts("replication")); i++ { PVCName := cr.Name + "-" + cr.Name + "-" + strconv.Itoa(i) - err := generateK8sClient().CoreV1().PersistentVolumeClaims(cr.Namespace).Delete(context.TODO(), PVCName, metav1.DeleteOptions{}) + err := client.CoreV1().PersistentVolumeClaims(cr.Namespace).Delete(context.TODO(), PVCName, metav1.DeleteOptions{}) if err != nil && !errors.IsNotFound(err) { logger.Error(err, "Could not delete Persistent Volume Claim "+PVCName) return err diff --git a/k8sutils/finalizers_test.go b/k8sutils/finalizers_test.go new file mode 100644 index 000000000..4ab301919 --- /dev/null +++ b/k8sutils/finalizers_test.go @@ -0,0 +1,122 @@ +package k8sutils + +import ( + "context" + "fmt" + "testing" + + "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2" + "github.com/go-logr/logr/testr" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sClientFake "k8s.io/client-go/kubernetes/fake" +) + +// func TestHandleRedisFinalizer(t *testing.T) { +// cr := &v1beta2.Redis{ +// TypeMeta: metav1.TypeMeta{ +// Kind: "Redis", +// APIVersion: "redis.opstreelabs.in/v1beta2", +// }, +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-redis", +// Namespace: "default", +// DeletionTimestamp: &metav1.Time{Time: time.Now()}, +// Finalizers: []string{RedisFinalizer}, +// }, +// } + +// // Create a fake controller-runtime client +// scheme := runtime.NewScheme() +// mockAddToScheme := v1beta2.SchemeBuilder.Register(&v1beta2.Redis{}, &v1beta2.RedisList{}).AddToScheme(scheme) +// utilruntime.Must(mockAddToScheme) + +// ctrlFakeclient := ctrlClientFake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cr.DeepCopyObject()).Build() +// k8sFakeClient := k8sClientFake.NewSimpleClientset(cr.DeepCopyObject()) + +// logger := testr.New(t) +// // Run the HandleRedisFinalizer function +// err := HandleRedisFinalizer(ctrlFakeclient, k8sFakeClient, logger, cr) +// assert.NoError(t, err) + +// // Check if the PVC was deleted +// PVCName := fmt.Sprintf("%s-%s-0", cr.Name, cr.Name) +// _, err = k8sFakeClient.CoreV1().PersistentVolumeClaims(cr.Namespace).Get(context.TODO(), PVCName, metav1.GetOptions{}) +// assert.True(t, k8serrors.IsNotFound(err)) + +// // Check if the finalizer was removed +// updatedCR := &v1beta2.Redis{} +// err = ctrlFakeclient.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: "test-redis"}, updatedCR) +// assert.NoError(t, err) +// assert.NotContains(t, updatedCR.GetFinalizers(), RedisFinalizer) + +// // Ensure the logger's Error method was not called +// //logger.AssertNotCalled(t, "Error", mock.Anything, mock.Anything, mock.Anything) +// } + +func TestFinalizeRedisPVC(t *testing.T) { + tests := []struct { + name string + existingPVC *corev1.PersistentVolumeClaim + expectError bool + errorExpected error + }{ + { + name: "PVC exists and is deleted successfully", + existingPVC: &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-redis-test-redis-0", + Namespace: "default", + }, + }, + expectError: false, + errorExpected: nil, + }, + { + name: "PVC does not exist and no error should be returned", + existingPVC: &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nonexistent", + Namespace: "default", + }, + }, + expectError: false, + errorExpected: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + logger := testr.New(t) + cr := &v1beta2.Redis{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-redis", + Namespace: "default", + }, + } + var k8sClient *k8sClientFake.Clientset + if tc.existingPVC != nil { + k8sClient = k8sClientFake.NewSimpleClientset(tc.existingPVC.DeepCopyObject()) + } else { + k8sClient = k8sClientFake.NewSimpleClientset() + } + + err := finalizeRedisPVC(k8sClient, logger, cr) + if tc.expectError { + assert.Error(t, err) + assert.Equal(t, tc.errorExpected, err) + } else { + assert.NoError(t, err) + } + + // Verify that the PVC is not found in case of success or non-existent PVC + if !tc.expectError { + pvcName := fmt.Sprintf("%s-%s-0", cr.Name, cr.Name) + _, err = k8sClient.CoreV1().PersistentVolumeClaims(cr.Namespace).Get(context.TODO(), pvcName, metav1.GetOptions{}) + assert.True(t, k8serrors.IsNotFound(err)) + } + }) + } +} diff --git a/k8sutils/poddisruption.go b/k8sutils/poddisruption.go index 00d1b7620..2886f7ccb 100644 --- a/k8sutils/poddisruption.go +++ b/k8sutils/poddisruption.go @@ -177,7 +177,12 @@ func patchPodDisruptionBudget(storedPdb *policyv1.PodDisruptionBudget, newPdb *p // createPodDisruptionBudget is a method to create PodDisruptionBudgets in Kubernetes func createPodDisruptionBudget(namespace string, pdb *policyv1.PodDisruptionBudget) error { logger := pdbLogger(namespace, pdb.Name) - _, err := generateK8sClient().PolicyV1().PodDisruptionBudgets(namespace).Create(context.TODO(), pdb, metav1.CreateOptions{}) + client, err := GenerateK8sClient(GenerateK8sConfig) + if err != nil { + logger.Error(err, "Could not generate kubernetes client") + return err + } + _, err = client.PolicyV1().PodDisruptionBudgets(namespace).Create(context.TODO(), pdb, metav1.CreateOptions{}) if err != nil { logger.Error(err, "Redis PodDisruptionBudget creation failed") return err @@ -189,7 +194,12 @@ func createPodDisruptionBudget(namespace string, pdb *policyv1.PodDisruptionBudg // updatePodDisruptionBudget is a method to update PodDisruptionBudgets in Kubernetes func updatePodDisruptionBudget(namespace string, pdb *policyv1.PodDisruptionBudget) error { logger := pdbLogger(namespace, pdb.Name) - _, err := generateK8sClient().PolicyV1().PodDisruptionBudgets(namespace).Update(context.TODO(), pdb, metav1.UpdateOptions{}) + client, err := GenerateK8sClient(GenerateK8sConfig) + if err != nil { + logger.Error(err, "Could not generate kubernetes client") + return err + } + _, err = client.PolicyV1().PodDisruptionBudgets(namespace).Update(context.TODO(), pdb, metav1.UpdateOptions{}) if err != nil { logger.Error(err, "Redis PodDisruptionBudget update failed") return err @@ -201,7 +211,12 @@ func updatePodDisruptionBudget(namespace string, pdb *policyv1.PodDisruptionBudg // deletePodDisruptionBudget is a method to delete PodDisruptionBudgets in Kubernetes func deletePodDisruptionBudget(namespace string, pdbName string) error { logger := pdbLogger(namespace, pdbName) - err := generateK8sClient().PolicyV1().PodDisruptionBudgets(namespace).Delete(context.TODO(), pdbName, metav1.DeleteOptions{}) + client, err := GenerateK8sClient(GenerateK8sConfig) + if err != nil { + logger.Error(err, "Could not generate kubernetes client") + return err + } + err = client.PolicyV1().PodDisruptionBudgets(namespace).Delete(context.TODO(), pdbName, metav1.DeleteOptions{}) if err != nil { logger.Error(err, "Redis PodDisruption deletion failed") return err @@ -213,10 +228,15 @@ func deletePodDisruptionBudget(namespace string, pdbName string) error { // GetPodDisruptionBudget is a method to get PodDisruptionBudgets in Kubernetes func GetPodDisruptionBudget(namespace string, pdb string) (*policyv1.PodDisruptionBudget, error) { logger := pdbLogger(namespace, pdb) + client, err := GenerateK8sClient(GenerateK8sConfig) + if err != nil { + logger.Error(err, "Could not generate kubernetes client") + return nil, err + } getOpts := metav1.GetOptions{ TypeMeta: generateMetaInformation("PodDisruptionBudget", "policy/v1"), } - pdbInfo, err := generateK8sClient().PolicyV1().PodDisruptionBudgets(namespace).Get(context.TODO(), pdb, getOpts) + pdbInfo, err := client.PolicyV1().PodDisruptionBudgets(namespace).Get(context.TODO(), pdb, getOpts) if err != nil { logger.V(1).Info("Redis PodDisruptionBudget get action failed") return nil, err diff --git a/k8sutils/redis-sentinel.go b/k8sutils/redis-sentinel.go index da39bd284..23d9ce509 100644 --- a/k8sutils/redis-sentinel.go +++ b/k8sutils/redis-sentinel.go @@ -283,6 +283,11 @@ func getSentinelEnvVariable(ctx context.Context, cr *redisv1beta2.RedisSentinel) func getRedisReplicationMasterIP(ctx context.Context, cr *redisv1beta2.RedisSentinel) string { logger := generateRedisManagerLogger(cr.Namespace, cr.ObjectMeta.Name) + dClient, err := GenerateK8sDynamicClient(GenerateK8sConfig) + if err != nil { + logger.Error(err, "Failed to generate dynamic client") + return "" + } replicationName := cr.Spec.RedisSentinelConfig.RedisReplicationName replicationNamespace := cr.Namespace @@ -291,7 +296,7 @@ func getRedisReplicationMasterIP(ctx context.Context, cr *redisv1beta2.RedisSent var realMasterPod string // Get Request on Dynamic Client - customObject, err := generateK8sDynamicClient().Resource(schema.GroupVersionResource{ + customObject, err := dClient.Resource(schema.GroupVersionResource{ Group: "redis.redis.opstreelabs.in", Version: "v1beta2", Resource: "redisreplications", diff --git a/k8sutils/redis.go b/k8sutils/redis.go index 6973363f0..dda5aabef 100644 --- a/k8sutils/redis.go +++ b/k8sutils/redis.go @@ -27,7 +27,12 @@ type RedisDetails struct { // getRedisServerIP will return the IP of redis service func getRedisServerIP(redisInfo RedisDetails) string { logger := generateRedisManagerLogger(redisInfo.Namespace, redisInfo.PodName) - redisPod, err := generateK8sClient().CoreV1().Pods(redisInfo.Namespace).Get(context.TODO(), redisInfo.PodName, metav1.GetOptions{}) + client, err := GenerateK8sClient(GenerateK8sConfig) + if err != nil { + logger.Error(err, "Error in getting k8s client") + return "" + } + redisPod, err := client.CoreV1().Pods(redisInfo.Namespace).Get(context.TODO(), redisInfo.PodName, metav1.GetOptions{}) if err != nil { logger.Error(err, "Error in getting redis pod IP") } @@ -358,7 +363,12 @@ func executeCommand(cr *redisv1beta2.RedisCluster, cmd []string, podName string) execErr bytes.Buffer ) logger := generateRedisManagerLogger(cr.Namespace, cr.ObjectMeta.Name) - config, err := generateK8sConfig() + client, err := GenerateK8sClient(GenerateK8sConfig) + if err != nil { + logger.Error(err, "Could not generate kubernetes client") + return + } + config, err := GenerateK8sConfig() if err != nil { logger.Error(err, "Could not find pod to execute") return @@ -369,7 +379,7 @@ func executeCommand(cr *redisv1beta2.RedisCluster, cmd []string, podName string) return } - req := generateK8sClient().CoreV1().RESTClient().Post().Resource("pods").Name(podName).Namespace(cr.Namespace).SubResource("exec") + req := client.CoreV1().RESTClient().Post().Resource("pods").Name(podName).Namespace(cr.Namespace).SubResource("exec") req.VersionedParams(&corev1.PodExecOptions{ Container: pod.Spec.Containers[targetContainer].Name, Command: cmd, @@ -397,7 +407,12 @@ func executeCommand(cr *redisv1beta2.RedisCluster, cmd []string, podName string) // getContainerID will return the id of container from pod func getContainerID(cr *redisv1beta2.RedisCluster, podName string) (int, *corev1.Pod) { logger := generateRedisManagerLogger(cr.Namespace, cr.ObjectMeta.Name) - pod, err := generateK8sClient().CoreV1().Pods(cr.Namespace).Get(context.TODO(), podName, metav1.GetOptions{}) + client, err := GenerateK8sClient(GenerateK8sConfig) + if err != nil { + logger.Error(err, "Could not generate kubernetes client") + return -1, nil + } + pod, err := client.CoreV1().Pods(cr.Namespace).Get(context.TODO(), podName, metav1.GetOptions{}) if err != nil { logger.Error(err, "Could not get pod info") } diff --git a/k8sutils/secrets.go b/k8sutils/secrets.go index 0cb7a935b..4636b54b0 100644 --- a/k8sutils/secrets.go +++ b/k8sutils/secrets.go @@ -17,7 +17,12 @@ var log = logf.Log.WithName("controller_redis") // getRedisPassword method will return the redis password func getRedisPassword(namespace, name, secretKey string) (string, error) { logger := secretLogger(namespace, name) - secretName, err := generateK8sClient().CoreV1().Secrets(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + client, err := GenerateK8sClient(GenerateK8sConfig) + if err != nil { + logger.Error(err, "Could not generate kubernetes client") + return "", err + } + secretName, err := client.CoreV1().Secrets(namespace).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { logger.Error(err, "Failed in getting existing secret for redis") return "", err @@ -36,9 +41,13 @@ func secretLogger(namespace string, name string) logr.Logger { } func getRedisTLSConfig(cr *redisv1beta2.RedisCluster, redisInfo RedisDetails) *tls.Config { + client, err := GenerateK8sClient(GenerateK8sConfig) + if err != nil { + return nil + } if cr.Spec.TLS != nil { reqLogger := log.WithValues("Request.Namespace", cr.Namespace, "Request.Name", cr.ObjectMeta.Name) - secretName, err := generateK8sClient().CoreV1().Secrets(cr.Namespace).Get(context.TODO(), cr.Spec.TLS.Secret.SecretName, metav1.GetOptions{}) + secretName, err := client.CoreV1().Secrets(cr.Namespace).Get(context.TODO(), cr.Spec.TLS.Secret.SecretName, metav1.GetOptions{}) if err != nil { reqLogger.Error(err, "Failed in getting TLS secret for redis") } @@ -84,9 +93,13 @@ func getRedisTLSConfig(cr *redisv1beta2.RedisCluster, redisInfo RedisDetails) *t } func getRedisReplicationTLSConfig(cr *redisv1beta2.RedisReplication, redisInfo RedisDetails) *tls.Config { + client, err := GenerateK8sClient(GenerateK8sConfig) + if err != nil { + return nil + } if cr.Spec.TLS != nil { reqLogger := log.WithValues("Request.Namespace", cr.Namespace, "Request.Name", cr.ObjectMeta.Name) - secretName, err := generateK8sClient().CoreV1().Secrets(cr.Namespace).Get(context.TODO(), cr.Spec.TLS.Secret.SecretName, metav1.GetOptions{}) + secretName, err := client.CoreV1().Secrets(cr.Namespace).Get(context.TODO(), cr.Spec.TLS.Secret.SecretName, metav1.GetOptions{}) if err != nil { reqLogger.Error(err, "Failed in getting TLS secret for redis") } diff --git a/k8sutils/services.go b/k8sutils/services.go index c93dab87d..0bcf56944 100644 --- a/k8sutils/services.go +++ b/k8sutils/services.go @@ -89,7 +89,12 @@ func generateServiceType(k8sServiceType string) corev1.ServiceType { // createService is a method to create service is Kubernetes func createService(namespace string, service *corev1.Service) error { logger := serviceLogger(namespace, service.Name) - _, err := generateK8sClient().CoreV1().Services(namespace).Create(context.TODO(), service, metav1.CreateOptions{}) + client, err := GenerateK8sClient(GenerateK8sConfig) + if err != nil { + logger.Error(err, "Could not generate kubernetes client") + return err + } + _, err = client.CoreV1().Services(namespace).Create(context.TODO(), service, metav1.CreateOptions{}) if err != nil { logger.Error(err, "Redis service creation is failed") return err @@ -101,7 +106,12 @@ func createService(namespace string, service *corev1.Service) error { // updateService is a method to update service is Kubernetes func updateService(namespace string, service *corev1.Service) error { logger := serviceLogger(namespace, service.Name) - _, err := generateK8sClient().CoreV1().Services(namespace).Update(context.TODO(), service, metav1.UpdateOptions{}) + client, err := GenerateK8sClient(GenerateK8sConfig) + if err != nil { + logger.Error(err, "Could not generate kubernetes client") + return err + } + _, err = client.CoreV1().Services(namespace).Update(context.TODO(), service, metav1.UpdateOptions{}) if err != nil { logger.Error(err, "Redis service update failed") return err @@ -113,10 +123,15 @@ func updateService(namespace string, service *corev1.Service) error { // getService is a method to get service is Kubernetes func getService(namespace string, service string) (*corev1.Service, error) { logger := serviceLogger(namespace, service) + client, err := GenerateK8sClient(GenerateK8sConfig) + if err != nil { + logger.Error(err, "Could not generate kubernetes client") + return nil, err + } getOpts := metav1.GetOptions{ TypeMeta: generateMetaInformation("Service", "v1"), } - serviceInfo, err := generateK8sClient().CoreV1().Services(namespace).Get(context.TODO(), service, getOpts) + serviceInfo, err := client.CoreV1().Services(namespace).Get(context.TODO(), service, getOpts) if err != nil { logger.V(1).Info("Redis service get action is failed") return nil, err diff --git a/k8sutils/statefulset.go b/k8sutils/statefulset.go index 0b34aa800..61162ca8b 100644 --- a/k8sutils/statefulset.go +++ b/k8sutils/statefulset.go @@ -108,7 +108,11 @@ func CreateOrUpdateStateFul(namespace string, stsMeta metav1.ObjectMeta, params // patchStateFulSet will patch Redis Kubernetes StateFulSet func patchStatefulSet(storedStateful *appsv1.StatefulSet, newStateful *appsv1.StatefulSet, namespace string, recreateStateFulSet bool) error { logger := statefulSetLogger(namespace, storedStateful.Name) - + client, err := GenerateK8sClient(GenerateK8sConfig) + if err != nil { + logger.Error(err, "Could not generate kubernetes client") + return err + } // We want to try and keep this atomic as possible. newStateful.ResourceVersion = storedStateful.ResourceVersion newStateful.CreationTimestamp = storedStateful.CreationTimestamp @@ -156,7 +160,7 @@ func patchStatefulSet(storedStateful *appsv1.StatefulSet, newStateful *appsv1.St }, ), } - pvcs, err := generateK8sClient().CoreV1().PersistentVolumeClaims(storedStateful.Namespace).List(context.Background(), listOpt) + pvcs, err := client.CoreV1().PersistentVolumeClaims(storedStateful.Namespace).List(context.Background(), listOpt) if err != nil { return err } @@ -167,7 +171,7 @@ func patchStatefulSet(storedStateful *appsv1.StatefulSet, newStateful *appsv1.St if realCapacity != stateCapacity { realUpdate = true pvc.Spec.Resources.Requests = newStateful.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests - _, err = generateK8sClient().CoreV1().PersistentVolumeClaims(storedStateful.Namespace).Update(context.Background(), &pvc, metav1.UpdateOptions{}) + _, err = client.CoreV1().PersistentVolumeClaims(storedStateful.Namespace).Update(context.Background(), &pvc, metav1.UpdateOptions{}) if err != nil { if !updateFailed { updateFailed = true @@ -631,7 +635,12 @@ func getEnvironmentVariables(role string, enabledPassword *bool, secretName *str // createStatefulSet is a method to create statefulset in Kubernetes func createStatefulSet(namespace string, stateful *appsv1.StatefulSet) error { logger := statefulSetLogger(namespace, stateful.Name) - _, err := generateK8sClient().AppsV1().StatefulSets(namespace).Create(context.TODO(), stateful, metav1.CreateOptions{}) + client, err := GenerateK8sClient(GenerateK8sConfig) + if err != nil { + logger.Error(err, "Could not generate kubernetes client") + return err + } + _, err = client.AppsV1().StatefulSets(namespace).Create(context.TODO(), stateful, metav1.CreateOptions{}) if err != nil { logger.Error(err, "Redis stateful creation failed") return err @@ -643,7 +652,12 @@ func createStatefulSet(namespace string, stateful *appsv1.StatefulSet) error { // updateStatefulSet is a method to update statefulset in Kubernetes func updateStatefulSet(namespace string, stateful *appsv1.StatefulSet, recreateStateFulSet bool) error { logger := statefulSetLogger(namespace, stateful.Name) - _, err := generateK8sClient().AppsV1().StatefulSets(namespace).Update(context.TODO(), stateful, metav1.UpdateOptions{}) + client, err := GenerateK8sClient(GenerateK8sConfig) + if err != nil { + logger.Error(err, "Could not generate kubernetes client") + return err + } + _, err = client.AppsV1().StatefulSets(namespace).Update(context.TODO(), stateful, metav1.UpdateOptions{}) if recreateStateFulSet { sErr, ok := err.(*apierrors.StatusError) if ok && sErr.ErrStatus.Code == 422 && sErr.ErrStatus.Reason == metav1.StatusReasonInvalid { @@ -653,7 +667,7 @@ func updateStatefulSet(namespace string, stateful *appsv1.StatefulSet, recreateS } logger.V(1).Info("recreating StatefulSet because the update operation wasn't possible", "reason", strings.Join(failMsg, ", ")) propagationPolicy := metav1.DeletePropagationForeground - if err := generateK8sClient().AppsV1().StatefulSets(namespace).Delete(context.TODO(), stateful.GetName(), metav1.DeleteOptions{PropagationPolicy: &propagationPolicy}); err != nil { //nolint + if err := client.AppsV1().StatefulSets(namespace).Delete(context.TODO(), stateful.GetName(), metav1.DeleteOptions{PropagationPolicy: &propagationPolicy}); err != nil { //nolint return errors.Wrap(err, "failed to delete StatefulSet to avoid forbidden action") } } @@ -669,10 +683,15 @@ func updateStatefulSet(namespace string, stateful *appsv1.StatefulSet, recreateS // GetStateFulSet is a method to get statefulset in Kubernetes func GetStatefulSet(namespace string, stateful string) (*appsv1.StatefulSet, error) { logger := statefulSetLogger(namespace, stateful) + client, err := GenerateK8sClient(GenerateK8sConfig) + if err != nil { + logger.Error(err, "Could not generate kubernetes client") + return nil, err + } getOpts := metav1.GetOptions{ TypeMeta: generateMetaInformation("StatefulSet", "apps/v1"), } - statefulInfo, err := generateK8sClient().AppsV1().StatefulSets(namespace).Get(context.TODO(), stateful, getOpts) + statefulInfo, err := client.AppsV1().StatefulSets(namespace).Get(context.TODO(), stateful, getOpts) if err != nil { logger.V(1).Info("Redis statefulset get action failed") return nil, err diff --git a/k8sutils/status.go b/k8sutils/status.go index 1232b2384..3cc32353d 100644 --- a/k8sutils/status.go +++ b/k8sutils/status.go @@ -26,7 +26,7 @@ func UpdateRedisClusterStatus(cr *redisv1beta2.RedisCluster, status status.Redis cr.Status.ReadyLeaderReplicas = readyLeaderReplicas cr.Status.ReadyFollowerReplicas = readyFollowerReplicas - client := generateK8sDynamicClient() + client, err := GenerateK8sDynamicClient(GenerateK8sConfig) gvr := schema.GroupVersionResource{ Group: "redis.redis.opstreelabs.in", Version: "v1beta2", diff --git a/main.go b/main.go index 70d82f333..ec277322a 100644 --- a/main.go +++ b/main.go @@ -36,6 +36,7 @@ import ( redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2" "github.com/OT-CONTAINER-KIT/redis-operator/controllers" + "github.com/OT-CONTAINER-KIT/redis-operator/k8sutils" redisv1beta1 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta1" // +kubebuilder:scaffold:imports @@ -100,10 +101,24 @@ func main() { os.Exit(1) } + k8sclient, err := k8sutils.GenerateK8sClient(k8sutils.GenerateK8sConfig) + if err != nil { + setupLog.Error(err, "unable to create k8s client") + os.Exit(1) + } + + dk8sClinet, err := k8sutils.GenerateK8sDynamicClient(k8sutils.GenerateK8sConfig) + if err != nil { + setupLog.Error(err, "unable to create k8s dynamic client") + os.Exit(1) + } + if err = (&controllers.RedisReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("Redis"), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + K8sClient: k8sclient, + Dk8sClinet: dk8sClinet, + Log: ctrl.Log.WithName("controllers").WithName("Redis"), + Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Redis") os.Exit(1) diff --git a/mocks/log/logger.go b/mocks/log/logger.go new file mode 100644 index 000000000..78f1e666a --- /dev/null +++ b/mocks/log/logger.go @@ -0,0 +1,5 @@ +package log + +type LoggerInterface interface { + Error(error, string, ...interface{}) +}