From 9b7f58acd43fff430a5084eb36d3ea10f6fc2bab Mon Sep 17 00:00:00 2001 From: Max Shaposhnyk Date: Thu, 11 Apr 2024 17:25:50 +0300 Subject: [PATCH] Add availability probe for image controller (#101) Add availability probe for image controller Signed-off-by: Max Shaposhnyk --- controllers/component_image_controller.go | 13 +-- .../component_image_controller_test.go | 90 +++++++++--------- controllers/imagerepository_controller.go | 85 ++++------------- .../imagerepository_controller_test.go | 94 +++++++++---------- controllers/suite_test.go | 8 +- main.go | 17 +++- pkg/metrics/metrics.go | 94 +++++++++++++++++++ pkg/metrics/metrics_test.go | 41 ++++++++ pkg/metrics/quay.go | 47 ++++++++++ .../quay/test_quay_client.go | 58 ++++++------ 10 files changed, 342 insertions(+), 205 deletions(-) create mode 100644 pkg/metrics/metrics.go create mode 100644 pkg/metrics/metrics_test.go create mode 100644 pkg/metrics/quay.go rename controllers/suite_util_quay_client_test.go => pkg/quay/test_quay_client.go (61%) diff --git a/controllers/component_image_controller.go b/controllers/component_image_controller.go index a8f9f2c..cb7da6a 100644 --- a/controllers/component_image_controller.go +++ b/controllers/component_image_controller.go @@ -21,6 +21,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "github.com/redhat-appstudio/image-controller/pkg/metrics" "strings" "time" @@ -79,10 +80,6 @@ type ComponentReconciler struct { // SetupWithManager sets up the controller with the Manager. func (r *ComponentReconciler) SetupWithManager(mgr ctrl.Manager) error { - if err := initMetrics(); err != nil { - return err - } - return ctrl.NewControllerManagedBy(mgr). For(&appstudioredhatcomv1alpha1.Component{}). Complete(r) @@ -117,7 +114,7 @@ func (r *ComponentReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( if !component.ObjectMeta.DeletionTimestamp.IsZero() { // remove component from metrics map - delete(repositoryTimesForMetrics, componentIdForMetrics) + delete(metrics.RepositoryTimesForMetrics, componentIdForMetrics) if controllerutil.ContainsFinalizer(component, ImageRepositoryComponentFinalizer) { pushRobotAccountName, pullRobotAccountName := generateRobotAccountsNames(component) @@ -320,9 +317,9 @@ func (r *ComponentReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( }) } - imageRepositoryProvisionTimeMetric.Observe(time.Since(repositoryTimesForMetrics[componentIdForMetrics]).Seconds()) + metrics.ImageRepositoryProvisionTimeMetric.Observe(time.Since(metrics.RepositoryTimesForMetrics[componentIdForMetrics]).Seconds()) // remove component from metrics map - delete(repositoryTimesForMetrics, componentIdForMetrics) + delete(metrics.RepositoryTimesForMetrics, componentIdForMetrics) return ctrl.Result{}, nil } @@ -338,7 +335,7 @@ func (r *ComponentReconciler) reportError(ctx context.Context, component *appstu componentIdForMetrics := getComponentIdForMetrics(component) // remove component from metrics map, permanent error - delete(repositoryTimesForMetrics, componentIdForMetrics) + delete(metrics.RepositoryTimesForMetrics, componentIdForMetrics) return r.Client.Update(ctx, component) } diff --git a/controllers/component_image_controller_test.go b/controllers/component_image_controller_test.go index 752857b..e69b120 100644 --- a/controllers/component_image_controller_test.go +++ b/controllers/component_image_controller_test.go @@ -56,7 +56,7 @@ var _ = Describe("Component image controller", func() { It("should prepare environment", func() { createNamespace(defaultNamespace) - ResetTestQuayClient() + quay.ResetTestQuayClient() pushToken = "push-token1234" pullToken = "pull-token1234" @@ -64,24 +64,24 @@ var _ = Describe("Component image controller", func() { expectedPullRobotAccountName = expectedPushRobotAccountName + "-pull" expectedRemoteSecretName = resourceKey.Name + "-pull" expectedRepoName = fmt.Sprintf("%s/%s/%s", defaultNamespace, defaultComponentApplication, defaultComponentName) - expectedImage = fmt.Sprintf("quay.io/%s/%s", testQuayOrg, expectedRepoName) + expectedImage = fmt.Sprintf("quay.io/%s/%s", quay.TestQuayOrg, expectedRepoName) }) It("should do image repository provision", func() { isCreateRepositoryInvoked := false - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { + quay.CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { isCreateRepositoryInvoked = true Expect(repository.Repository).To(Equal(expectedRepoName)) - Expect(repository.Namespace).To(Equal(testQuayOrg)) + Expect(repository.Namespace).To(Equal(quay.TestQuayOrg)) Expect(repository.Visibility).To(Equal("private")) Expect(repository.Description).ToNot(BeEmpty()) return &quay.Repository{Name: expectedRepoName}, nil } isCreatePushRobotAccountInvoked := false isCreatePullRobotAccountInvoked := false - CreateRobotAccountFunc = func(organization, robotName string) (*quay.RobotAccount, error) { + quay.CreateRobotAccountFunc = func(organization, robotName string) (*quay.RobotAccount, error) { defer GinkgoRecover() - Expect(organization).To(Equal(testQuayOrg)) + Expect(organization).To(Equal(quay.TestQuayOrg)) switch robotName { case expectedPushRobotAccountName: isCreatePushRobotAccountInvoked = true @@ -101,9 +101,9 @@ var _ = Describe("Component image controller", func() { } isAddPushPermissionsToRobotAccountInvoked := false isAddPullPermissionsToRobotAccountInvoked := false - AddPermissionsForRepositoryToRobotAccountFunc = func(organization, imageRepository, robotAccountName string, isWrite bool) error { + quay.AddPermissionsForRepositoryToRobotAccountFunc = func(organization, imageRepository, robotAccountName string, isWrite bool) error { defer GinkgoRecover() - Expect(organization).To(Equal(testQuayOrg)) + Expect(organization).To(Equal(quay.TestQuayOrg)) Expect(imageRepository).To(Equal(expectedRepoName)) if isWrite { isAddPushPermissionsToRobotAccountInvoked = true @@ -195,14 +195,14 @@ var _ = Describe("Component image controller", func() { It("should be able to switch image visibility", func() { isChangeRepositoryVisibilityInvoked := false - ChangeRepositoryVisibilityFunc = func(organization, imageRepository, visibility string) error { + quay.ChangeRepositoryVisibilityFunc = func(organization, imageRepository, visibility string) error { isChangeRepositoryVisibilityInvoked = true - Expect(organization).To(Equal(testQuayOrg)) + Expect(organization).To(Equal(quay.TestQuayOrg)) Expect(imageRepository).To(Equal(expectedRepoName)) Expect(visibility).To(Equal("public")) return nil } - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { + quay.CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { defer GinkgoRecover() Fail("Should not invoke repository creation on clean up") return nil, nil @@ -225,12 +225,12 @@ var _ = Describe("Component image controller", func() { }) It("should do nothing if the same as current visibility requested", func() { - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { + quay.CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { defer GinkgoRecover() Fail("Image repository creation should not be invoked") return nil, nil } - ChangeRepositoryVisibilityFunc = func(organization, imageRepository, visibility string) error { + quay.ChangeRepositoryVisibilityFunc = func(organization, imageRepository, visibility string) error { defer GinkgoRecover() Fail("Image repository visibility changing should not be invoked") return nil @@ -251,17 +251,17 @@ var _ = Describe("Component image controller", func() { }) It("should delete robot account and image repository on component deletion", func() { - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { + quay.CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { defer GinkgoRecover() Fail("Should not invoke repository creation on clean up") return nil, nil } - CreateRobotAccountFunc = func(organization, robotName string) (*quay.RobotAccount, error) { + quay.CreateRobotAccountFunc = func(organization, robotName string) (*quay.RobotAccount, error) { defer GinkgoRecover() Fail("Should not invoke robot account creation on clean up") return nil, nil } - AddPermissionsForRepositoryToRobotAccountFunc = func(organization, imageRepository, robotAccountName string, isWrite bool) error { + quay.AddPermissionsForRepositoryToRobotAccountFunc = func(organization, imageRepository, robotAccountName string, isWrite bool) error { defer GinkgoRecover() Fail("Should not invoke permission adding on clean up") return nil @@ -269,9 +269,9 @@ var _ = Describe("Component image controller", func() { isDeletePushRobotAccountInvoked := false isDeletePullRobotAccountInvoked := false - DeleteRobotAccountFunc = func(organization, robotAccountName string) (bool, error) { + quay.DeleteRobotAccountFunc = func(organization, robotAccountName string) (bool, error) { defer GinkgoRecover() - Expect(organization).To(Equal(testQuayOrg)) + Expect(organization).To(Equal(quay.TestQuayOrg)) switch robotAccountName { case expectedPushRobotAccountName: isDeletePushRobotAccountInvoked = true @@ -284,9 +284,9 @@ var _ = Describe("Component image controller", func() { return false, nil } isDeleteRepositoryInvoked := false - DeleteRepositoryFunc = func(organization, imageRepository string) (bool, error) { + quay.DeleteRepositoryFunc = func(organization, imageRepository string) (bool, error) { isDeleteRepositoryInvoked = true - Expect(organization).To(Equal(testQuayOrg)) + Expect(organization).To(Equal(quay.TestQuayOrg)) Expect(imageRepository).To(Equal(expectedRepoName)) return true, nil } @@ -304,15 +304,15 @@ var _ = Describe("Component image controller", func() { It("should prepare environment", func() { createNamespace(defaultNamespace) - ResetTestQuayClient() + quay.ResetTestQuayClient() deleteComponent(resourceKey) - expectedImage = fmt.Sprintf("quay.io/%s/%s", testQuayOrg, expectedRepoName) + expectedImage = fmt.Sprintf("quay.io/%s/%s", quay.TestQuayOrg, expectedRepoName) }) It("should do nothing if generate annotation is not set", func() { - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { + quay.CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { defer GinkgoRecover() Fail("Image repository creation should not be invoked") return nil, nil @@ -326,7 +326,7 @@ var _ = Describe("Component image controller", func() { }) It("should do nothing and set error if generate annotation is invalid JSON", func() { - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { + quay.CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { defer GinkgoRecover() Fail("Image repository creation should not be invoked") return nil, nil @@ -349,7 +349,7 @@ var _ = Describe("Component image controller", func() { }) It("should do nothing and set error if generate annotation has invalid visibility value", func() { - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { + quay.CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { defer GinkgoRecover() Fail("Image repository creation should not be invoked") return nil, nil @@ -372,11 +372,11 @@ var _ = Describe("Component image controller", func() { }) It("should set error if quay organization plan doesn't allow private repositories", func() { - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { + quay.CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { Expect(repository.Visibility).To(Equal("private")) return nil, fmt.Errorf("payment required") } - ChangeRepositoryVisibilityFunc = func(organization, imageRepository, visibility string) error { + quay.ChangeRepositoryVisibilityFunc = func(organization, imageRepository, visibility string) error { defer GinkgoRecover() Fail("Image repository visibility change should not be invoked") return nil @@ -400,7 +400,7 @@ var _ = Describe("Component image controller", func() { It("should add message and stop if it's not possible to switch image repository visibility", func() { isChangeRepositoryVisibilityInvoked := false - ChangeRepositoryVisibilityFunc = func(organization, imageRepository, visibility string) error { + quay.ChangeRepositoryVisibilityFunc = func(organization, imageRepository, visibility string) error { if isChangeRepositoryVisibilityInvoked { defer GinkgoRecover() Fail("Image repository visibility change should not be invoked second time") @@ -408,7 +408,7 @@ var _ = Describe("Component image controller", func() { isChangeRepositoryVisibilityInvoked = true return fmt.Errorf("payment required") } - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { + quay.CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { defer GinkgoRecover() Fail("Should not invoke repository creation") return nil, nil @@ -439,7 +439,7 @@ var _ = Describe("Component image controller", func() { It("should stop and report error if image repository creation fails", func() { isCreateRepositoryInvoked := false - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { + quay.CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { isCreateRepositoryInvoked = true return nil, fmt.Errorf("fail to marshal data") } @@ -455,7 +455,7 @@ var _ = Describe("Component image controller", func() { }) It("should do nothing and set error for changing visibility if image is invalid in image annotation", func() { - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { + quay.CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { defer GinkgoRecover() Fail("Image repository creation should not be invoked") return nil, nil @@ -483,7 +483,7 @@ var _ = Describe("Component image controller", func() { createComponent(componentConfig{ComponentKey: testComponentKey}) isChangeRepositoryVisibilityInvoked := false - ChangeRepositoryVisibilityFunc = func(string, string, string) error { + quay.ChangeRepositoryVisibilityFunc = func(string, string, string) error { isChangeRepositoryVisibilityInvoked = true return fmt.Errorf("failed to change repository visibility") } @@ -511,7 +511,7 @@ var _ = Describe("Component image controller", func() { }) It("should do nothing and set error if image annotation is invalid JSON", func() { - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { + quay.CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { defer GinkgoRecover() Fail("Image repository creation should not be invoked") return nil, nil @@ -532,15 +532,15 @@ var _ = Describe("Component image controller", func() { It("should not block component deletion if clean up fails", func() { waitImageRepositoryFinalizerOnComponent(resourceKey) - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { + quay.CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { defer GinkgoRecover() Fail("Should not invoke repository creation") return nil, nil } - DeleteRepositoryFunc = func(organization, imageRepository string) (bool, error) { + quay.DeleteRepositoryFunc = func(organization, imageRepository string) (bool, error) { return false, fmt.Errorf("failed to delete repository") } - DeleteRobotAccountFunc = func(organization, robotName string) (bool, error) { + quay.DeleteRobotAccountFunc = func(organization, robotName string) (bool, error) { return false, fmt.Errorf("failed to delete robot account") } @@ -553,7 +553,7 @@ var _ = Describe("Component image controller", func() { _ = BeforeEach(func() { createNamespace(defaultNamespace) - ResetTestQuayClient() + quay.ResetTestQuayClient() deleteComponent(resourceKey) deleteSecret(uploadSecretKey) @@ -563,7 +563,7 @@ var _ = Describe("Component image controller", func() { expectedPushRobotAccountName = fmt.Sprintf("%s%s%s", defaultNamespace, defaultComponentApplication, defaultComponentName) expectedPullRobotAccountName = expectedPushRobotAccountName + "-pull" expectedRepoName = fmt.Sprintf("%s/%s/%s", defaultNamespace, defaultComponentApplication, defaultComponentName) - expectedImage = fmt.Sprintf("quay.io/%s/%s", testQuayOrg, expectedRepoName) + expectedImage = fmt.Sprintf("quay.io/%s/%s", quay.TestQuayOrg, expectedRepoName) }) _ = AfterEach(func() { @@ -574,7 +574,7 @@ var _ = Describe("Component image controller", func() { It("should accept deprecated true value for repository options", func() { isCreateRepositoryInvoked := false - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { + quay.CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { isCreateRepositoryInvoked = true return &quay.Repository{Name: "repo-name"}, nil } @@ -609,19 +609,19 @@ var _ = Describe("Component image controller", func() { Expect(k8sErrors.IsNotFound(k8sClient.Get(ctx, remoteSecretKey, &remotesecretv1beta1.RemoteSecret{}))) isCreateRepositoryInvoked := false - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { + quay.CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { isCreateRepositoryInvoked = true Expect(repository.Repository).To(Equal(expectedRepoName)) - Expect(repository.Namespace).To(Equal(testQuayOrg)) + Expect(repository.Namespace).To(Equal(quay.TestQuayOrg)) Expect(repository.Visibility).To(Equal("public")) Expect(repository.Description).ToNot(BeEmpty()) return &quay.Repository{Name: expectedRepoName}, nil } isCreatePushRobotAccountInvoked := false isCreatePullRobotAccountInvoked := false - CreateRobotAccountFunc = func(organization, robotName string) (*quay.RobotAccount, error) { + quay.CreateRobotAccountFunc = func(organization, robotName string) (*quay.RobotAccount, error) { defer GinkgoRecover() - Expect(organization).To(Equal(testQuayOrg)) + Expect(organization).To(Equal(quay.TestQuayOrg)) switch robotName { case expectedPushRobotAccountName: isCreatePushRobotAccountInvoked = true @@ -641,9 +641,9 @@ var _ = Describe("Component image controller", func() { } isAddPushPermissionsToRobotAccountInvoked := false isAddPullPermissionsToRobotAccountInvoked := false - AddPermissionsForRepositoryToRobotAccountFunc = func(organization, imageRepository, robotAccountName string, isWrite bool) error { + quay.AddPermissionsForRepositoryToRobotAccountFunc = func(organization, imageRepository, robotAccountName string, isWrite bool) error { defer GinkgoRecover() - Expect(organization).To(Equal(testQuayOrg)) + Expect(organization).To(Equal(quay.TestQuayOrg)) Expect(imageRepository).To(Equal(expectedRepoName)) if isWrite { isAddPushPermissionsToRobotAccountInvoked = true diff --git a/controllers/imagerepository_controller.go b/controllers/imagerepository_controller.go index a1a8315..904ae61 100644 --- a/controllers/imagerepository_controller.go +++ b/controllers/imagerepository_controller.go @@ -21,9 +21,16 @@ import ( "crypto/rand" "encoding/hex" "fmt" + "github.com/redhat-appstudio/image-controller/pkg/metrics" "strings" "time" + "github.com/go-logr/logr" + appstudioredhatcomv1alpha1 "github.com/redhat-appstudio/application-api/api/v1alpha1" + imagerepositoryv1alpha1 "github.com/redhat-appstudio/image-controller/api/v1alpha1" + l "github.com/redhat-appstudio/image-controller/pkg/logs" + "github.com/redhat-appstudio/image-controller/pkg/quay" + remotesecretv1beta1 "github.com/redhat-appstudio/remote-secret/api/v1beta1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,15 +40,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ctrllog "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/metrics" - - "github.com/go-logr/logr" - "github.com/prometheus/client_golang/prometheus" - appstudioredhatcomv1alpha1 "github.com/redhat-appstudio/application-api/api/v1alpha1" - imagerepositoryv1alpha1 "github.com/redhat-appstudio/image-controller/api/v1alpha1" - l "github.com/redhat-appstudio/image-controller/pkg/logs" - "github.com/redhat-appstudio/image-controller/pkg/quay" - remotesecretv1beta1 "github.com/redhat-appstudio/remote-secret/api/v1beta1" ) const ( @@ -51,15 +49,6 @@ const ( buildPipelineServiceAccountName = "appstudio-pipeline" updateComponentAnnotationName = "image-controller.appstudio.redhat.com/update-component-image" - - metricsNamespace = "redhat_appstudio" - metricsSubsystem = "imagecontroller" -) - -var ( - imageRepositoryProvisionTimeMetric prometheus.Histogram - imageRepositoryProvisionFailureTimeMetric prometheus.Histogram - repositoryTimesForMetrics = map[string]time.Time{} ) // ImageRepositoryReconciler reconciles a ImageRepository object @@ -72,59 +61,17 @@ type ImageRepositoryReconciler struct { QuayOrganization string } -func initMetrics() error { - buckets := getProvisionTimeMetricsBuckets() - - // don't register it if it was already registered by another controller - if imageRepositoryProvisionTimeMetric != nil { - return nil - } - - imageRepositoryProvisionTimeMetric = prometheus.NewHistogram(prometheus.HistogramOpts{ - Namespace: metricsNamespace, - Subsystem: metricsSubsystem, - Buckets: buckets, - Name: "image_repository_provision_time", - Help: "The time in seconds spent from the moment of Image repository provision request to Image repository is ready to use.", - }) - - imageRepositoryProvisionFailureTimeMetric = prometheus.NewHistogram(prometheus.HistogramOpts{ - Namespace: metricsNamespace, - Subsystem: metricsSubsystem, - Buckets: buckets, - Name: "image_repository_provision_failure_time", - Help: "The time in seconds spent from the moment of Image repository provision request to Image repository failure.", - }) - - if err := metrics.Registry.Register(imageRepositoryProvisionTimeMetric); err != nil { - return fmt.Errorf("failed to register the image_repository_provision_time metric: %w", err) - } - if err := metrics.Registry.Register(imageRepositoryProvisionFailureTimeMetric); err != nil { - return fmt.Errorf("failed to register the image_repository_provision_failure_time metric: %w", err) - } - - return nil -} - -func getProvisionTimeMetricsBuckets() []float64 { - return []float64{5, 10, 15, 20, 30, 60, 120, 300} -} - // SetupWithManager sets up the controller with the Manager. func (r *ImageRepositoryReconciler) SetupWithManager(mgr ctrl.Manager) error { - if err := initMetrics(); err != nil { - return err - } - return ctrl.NewControllerManagedBy(mgr). For(&imagerepositoryv1alpha1.ImageRepository{}). Complete(r) } func setMetricsTime(idForMetrics string, reconcileStartTime time.Time) { - _, timeRecorded := repositoryTimesForMetrics[idForMetrics] + _, timeRecorded := metrics.RepositoryTimesForMetrics[idForMetrics] if !timeRecorded { - repositoryTimesForMetrics[idForMetrics] = reconcileStartTime + metrics.RepositoryTimesForMetrics[idForMetrics] = reconcileStartTime } } @@ -156,7 +103,7 @@ func (r *ImageRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Requ if !imageRepository.DeletionTimestamp.IsZero() { // remove component from metrics map - delete(repositoryTimesForMetrics, repositoryIdForMetrics) + delete(metrics.RepositoryTimesForMetrics, repositoryIdForMetrics) // Reread quay token r.QuayClient = r.BuildQuayClient(log) @@ -176,12 +123,12 @@ func (r *ImageRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Requ } if imageRepository.Status.State == imagerepositoryv1alpha1.ImageRepositoryStateFailed { - provisionTime, timeRecorded := repositoryTimesForMetrics[repositoryIdForMetrics] + provisionTime, timeRecorded := metrics.RepositoryTimesForMetrics[repositoryIdForMetrics] if timeRecorded { - imageRepositoryProvisionFailureTimeMetric.Observe(time.Since(provisionTime).Seconds()) + metrics.ImageRepositoryProvisionFailureTimeMetric.Observe(time.Since(provisionTime).Seconds()) // remove component from metrics map - delete(repositoryTimesForMetrics, repositoryIdForMetrics) + delete(metrics.RepositoryTimesForMetrics, repositoryIdForMetrics) } return ctrl.Result{}, nil @@ -274,12 +221,12 @@ func (r *ImageRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Requ // we are adding to map only for new provision, not for some partial actions, // so report time only if time was recorded - provisionTime, timeRecorded := repositoryTimesForMetrics[repositoryIdForMetrics] + provisionTime, timeRecorded := metrics.RepositoryTimesForMetrics[repositoryIdForMetrics] if timeRecorded { - imageRepositoryProvisionTimeMetric.Observe(time.Since(provisionTime).Seconds()) + metrics.ImageRepositoryProvisionTimeMetric.Observe(time.Since(provisionTime).Seconds()) } // remove component from metrics map - delete(repositoryTimesForMetrics, repositoryIdForMetrics) + delete(metrics.RepositoryTimesForMetrics, repositoryIdForMetrics) return ctrl.Result{}, nil } diff --git a/controllers/imagerepository_controller_test.go b/controllers/imagerepository_controller_test.go index bb5411f..6471d24 100644 --- a/controllers/imagerepository_controller_test.go +++ b/controllers/imagerepository_controller_test.go @@ -51,7 +51,7 @@ var _ = Describe("Image repository controller", func() { Context("Image repository provision", func() { BeforeEach(func() { - ResetTestQuayClientToFails() + quay.ResetTestQuayClientToFails() deleteUploadSecrets(defaultNamespace) }) @@ -60,34 +60,34 @@ var _ = Describe("Image repository controller", func() { pushToken = "push-token1234" expectedImageName = fmt.Sprintf("%s/%s", defaultNamespace, defaultImageRepositoryName) - expectedImage = fmt.Sprintf("quay.io/%s/%s", testQuayOrg, expectedImageName) + expectedImage = fmt.Sprintf("quay.io/%s/%s", quay.TestQuayOrg, expectedImageName) expectedRobotAccountPrefix = strings.ReplaceAll(strings.ReplaceAll(expectedImageName, "-", "_"), "/", "_") }) It("should provision image repository", func() { isCreateRepositoryInvoked := false - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { + quay.CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { defer GinkgoRecover() isCreateRepositoryInvoked = true Expect(repository.Repository).To(Equal(expectedImageName)) - Expect(repository.Namespace).To(Equal(testQuayOrg)) + Expect(repository.Namespace).To(Equal(quay.TestQuayOrg)) Expect(repository.Visibility).To(Equal("public")) Expect(repository.Description).ToNot(BeEmpty()) return &quay.Repository{Name: expectedImageName}, nil } isCreateRobotAccountInvoked := false - CreateRobotAccountFunc = func(organization, robotName string) (*quay.RobotAccount, error) { + quay.CreateRobotAccountFunc = func(organization, robotName string) (*quay.RobotAccount, error) { defer GinkgoRecover() isCreateRobotAccountInvoked = true - Expect(organization).To(Equal(testQuayOrg)) + Expect(organization).To(Equal(quay.TestQuayOrg)) Expect(strings.HasPrefix(robotName, expectedRobotAccountPrefix)).To(BeTrue()) return &quay.RobotAccount{Name: robotName, Token: pushToken}, nil } isAddPushPermissionsToRobotAccountInvoked := false - AddPermissionsForRepositoryToRobotAccountFunc = func(organization, imageRepository, robotAccountName string, isWrite bool) error { + quay.AddPermissionsForRepositoryToRobotAccountFunc = func(organization, imageRepository, robotAccountName string, isWrite bool) error { defer GinkgoRecover() isAddPushPermissionsToRobotAccountInvoked = true - Expect(organization).To(Equal(testQuayOrg)) + Expect(organization).To(Equal(quay.TestQuayOrg)) Expect(imageRepository).To(Equal(expectedImageName)) Expect(isWrite).To(BeTrue()) Expect(strings.HasPrefix(robotAccountName, expectedRobotAccountPrefix)).To(BeTrue()) @@ -154,10 +154,10 @@ var _ = Describe("Image repository controller", func() { time.Sleep(time.Second) isRegenerateRobotAccountTokenInvoked := false - RegenerateRobotAccountTokenFunc = func(organization, robotName string) (*quay.RobotAccount, error) { + quay.RegenerateRobotAccountTokenFunc = func(organization, robotName string) (*quay.RobotAccount, error) { defer GinkgoRecover() isRegenerateRobotAccountTokenInvoked = true - Expect(organization).To(Equal(testQuayOrg)) + Expect(organization).To(Equal(quay.TestQuayOrg)) Expect(strings.HasPrefix(robotName, expectedRobotAccountPrefix)).To(BeTrue()) return &quay.RobotAccount{Name: robotName, Token: newToken}, nil } @@ -196,10 +196,10 @@ var _ = Describe("Image repository controller", func() { It("should update image visibility", func() { isChangeRepositoryVisibilityInvoked := false - ChangeRepositoryVisibilityFunc = func(organization, imageRepository, visibility string) error { + quay.ChangeRepositoryVisibilityFunc = func(organization, imageRepository, visibility string) error { defer GinkgoRecover() isChangeRepositoryVisibilityInvoked = true - Expect(organization).To(Equal(testQuayOrg)) + Expect(organization).To(Equal(quay.TestQuayOrg)) Expect(imageRepository).To(Equal(expectedImageName)) Expect(visibility).To(Equal(string(imagerepositoryv1alpha1.ImageVisibilityPrivate))) return nil @@ -231,18 +231,18 @@ var _ = Describe("Image repository controller", func() { It("should cleanup repository", func() { isDeleteRobotAccountInvoked := false - DeleteRobotAccountFunc = func(organization, robotAccountName string) (bool, error) { + quay.DeleteRobotAccountFunc = func(organization, robotAccountName string) (bool, error) { defer GinkgoRecover() isDeleteRobotAccountInvoked = true - Expect(organization).To(Equal(testQuayOrg)) + Expect(organization).To(Equal(quay.TestQuayOrg)) Expect(strings.HasPrefix(robotAccountName, expectedRobotAccountPrefix)).To(BeTrue()) return true, nil } isDeleteRepositoryInvoked := false - DeleteRepositoryFunc = func(organization, imageRepository string) (bool, error) { + quay.DeleteRepositoryFunc = func(organization, imageRepository string) (bool, error) { defer GinkgoRecover() isDeleteRepositoryInvoked = true - Expect(organization).To(Equal(testQuayOrg)) + Expect(organization).To(Equal(quay.TestQuayOrg)) Expect(imageRepository).To(Equal(expectedImageName)) return true, nil } @@ -258,7 +258,7 @@ var _ = Describe("Image repository controller", func() { componentKey := types.NamespacedName{Name: defaultComponentName, Namespace: defaultNamespace} BeforeEach(func() { - ResetTestQuayClientToFails() + quay.ResetTestQuayClientToFails() deleteUploadSecrets(defaultNamespace) createComponent(componentConfig{}) }) @@ -271,27 +271,27 @@ var _ = Describe("Image repository controller", func() { pushToken = "push-token1234" pullToken = "pull-token1234" expectedImageName = fmt.Sprintf("%s/%s/%s", defaultNamespace, defaultComponentApplication, defaultComponentName) - expectedImage = fmt.Sprintf("quay.io/%s/%s", testQuayOrg, expectedImageName) + expectedImage = fmt.Sprintf("quay.io/%s/%s", quay.TestQuayOrg, expectedImageName) expectedRobotAccountPrefix = strings.ReplaceAll(strings.ReplaceAll(expectedImageName, "-", "_"), "/", "_") }) assertProvisionRepository := func(updateComponentAnnotation bool) { isCreateRepositoryInvoked := false - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { + quay.CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { defer GinkgoRecover() isCreateRepositoryInvoked = true Expect(repository.Repository).To(Equal(expectedImageName)) - Expect(repository.Namespace).To(Equal(testQuayOrg)) + Expect(repository.Namespace).To(Equal(quay.TestQuayOrg)) Expect(repository.Visibility).To(Equal("public")) Expect(repository.Description).ToNot(BeEmpty()) return &quay.Repository{Name: expectedImageName}, nil } isCreatePushRobotAccountInvoked := false isCreatePullRobotAccountInvoked := false - CreateRobotAccountFunc = func(organization, robotName string) (*quay.RobotAccount, error) { + quay.CreateRobotAccountFunc = func(organization, robotName string) (*quay.RobotAccount, error) { defer GinkgoRecover() - Expect(organization).To(Equal(testQuayOrg)) + Expect(organization).To(Equal(quay.TestQuayOrg)) Expect(strings.HasPrefix(robotName, expectedRobotAccountPrefix)).To(BeTrue()) if strings.HasSuffix(robotName, "_pull") { isCreatePullRobotAccountInvoked = true @@ -302,9 +302,9 @@ var _ = Describe("Image repository controller", func() { } isAddPushPermissionsToRobotAccountInvoked := false isAddPullPermissionsToRobotAccountInvoked := false - AddPermissionsForRepositoryToRobotAccountFunc = func(organization, imageRepository, robotAccountName string, isWrite bool) error { + quay.AddPermissionsForRepositoryToRobotAccountFunc = func(organization, imageRepository, robotAccountName string, isWrite bool) error { defer GinkgoRecover() - Expect(organization).To(Equal(testQuayOrg)) + Expect(organization).To(Equal(quay.TestQuayOrg)) Expect(imageRepository).To(Equal(expectedImageName)) Expect(strings.HasPrefix(robotAccountName, expectedRobotAccountPrefix)).To(BeTrue()) if strings.HasSuffix(robotAccountName, "_pull") { @@ -430,10 +430,10 @@ var _ = Describe("Image repository controller", func() { It("should provision image repository for component, without update component annotation", func() { assertProvisionRepository(false) - DeleteRobotAccountFunc = func(organization, robotAccountName string) (bool, error) { + quay.DeleteRobotAccountFunc = func(organization, robotAccountName string) (bool, error) { return true, nil } - DeleteRepositoryFunc = func(organization, imageRepository string) (bool, error) { + quay.DeleteRepositoryFunc = func(organization, imageRepository string) (bool, error) { return true, nil } @@ -458,9 +458,9 @@ var _ = Describe("Image repository controller", func() { isRegenerateRobotAccountTokenForPushInvoked := false isRegenerateRobotAccountTokenForPullInvoked := false - RegenerateRobotAccountTokenFunc = func(organization, robotName string) (*quay.RobotAccount, error) { + quay.RegenerateRobotAccountTokenFunc = func(organization, robotName string) (*quay.RobotAccount, error) { defer GinkgoRecover() - Expect(organization).To(Equal(testQuayOrg)) + Expect(organization).To(Equal(quay.TestQuayOrg)) Expect(strings.HasPrefix(robotName, expectedRobotAccountPrefix)).To(BeTrue()) if strings.HasSuffix(robotName, "_pull") { isRegenerateRobotAccountTokenForPullInvoked = true @@ -521,9 +521,9 @@ var _ = Describe("Image repository controller", func() { It("should cleanup component repository", func() { isDeleteRobotAccountForPushInvoked := false isDeleteRobotAccountForPullInvoked := false - DeleteRobotAccountFunc = func(organization, robotAccountName string) (bool, error) { + quay.DeleteRobotAccountFunc = func(organization, robotAccountName string) (bool, error) { defer GinkgoRecover() - Expect(organization).To(Equal(testQuayOrg)) + Expect(organization).To(Equal(quay.TestQuayOrg)) Expect(strings.HasPrefix(robotAccountName, expectedRobotAccountPrefix)).To(BeTrue()) if strings.HasSuffix(robotAccountName, "_pull") { isDeleteRobotAccountForPushInvoked = true @@ -533,10 +533,10 @@ var _ = Describe("Image repository controller", func() { return true, nil } isDeleteRepositoryInvoked := false - DeleteRepositoryFunc = func(organization, imageRepository string) (bool, error) { + quay.DeleteRepositoryFunc = func(organization, imageRepository string) (bool, error) { defer GinkgoRecover() isDeleteRepositoryInvoked = true - Expect(organization).To(Equal(testQuayOrg)) + Expect(organization).To(Equal(quay.TestQuayOrg)) Expect(imageRepository).To(Equal(expectedImageName)) return true, nil } @@ -552,7 +552,7 @@ var _ = Describe("Image repository controller", func() { Context("Other image repository scenarios", func() { BeforeEach(func() { - ResetTestQuayClient() + quay.ResetTestQuayClient() deleteImageRepository(resourceKey) deleteUploadSecrets(defaultNamespace) }) @@ -562,11 +562,11 @@ var _ = Describe("Image repository controller", func() { expectedImageName = defaultNamespace + "/" + customImageName isCreateRepositoryInvoked := false - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { + quay.CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { defer GinkgoRecover() isCreateRepositoryInvoked = true Expect(repository.Repository).To(Equal(expectedImageName)) - Expect(repository.Namespace).To(Equal(testQuayOrg)) + Expect(repository.Namespace).To(Equal(quay.TestQuayOrg)) Expect(repository.Visibility).To(Equal("public")) Expect(repository.Description).ToNot(BeEmpty()) return &quay.Repository{Name: expectedImageName}, nil @@ -587,7 +587,7 @@ var _ = Describe("Image repository controller", func() { Context("Image repository error scenarios", func() { BeforeEach(func() { - ResetTestQuayClient() + quay.ResetTestQuayClient() deleteImageRepository(resourceKey) deleteUploadSecrets(defaultNamespace) }) @@ -595,17 +595,17 @@ var _ = Describe("Image repository controller", func() { It("should prepare environment", func() { pushToken = "push-token1234" expectedImageName = fmt.Sprintf("%s/%s", defaultNamespace, defaultImageRepositoryName) - expectedImage = fmt.Sprintf("quay.io/%s/%s", testQuayOrg, expectedImageName) + expectedImage = fmt.Sprintf("quay.io/%s/%s", quay.TestQuayOrg, expectedImageName) expectedRobotAccountPrefix = strings.ReplaceAll(strings.ReplaceAll(expectedImageName, "-", "_"), "/", "_") }) It("should permanently fail if private image repository requested on creation but quota exceeded", func() { isCreateRepositoryInvoked := false - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { + quay.CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { defer GinkgoRecover() isCreateRepositoryInvoked = true Expect(repository.Repository).To(Equal(expectedImageName)) - Expect(repository.Namespace).To(Equal(testQuayOrg)) + Expect(repository.Namespace).To(Equal(quay.TestQuayOrg)) Expect(repository.Visibility).To(Equal("private")) Expect(repository.Description).ToNot(BeEmpty()) return nil, fmt.Errorf("payment required") @@ -628,12 +628,12 @@ var _ = Describe("Image repository controller", func() { }) It("should add error message and revert visibility in spec if private visibility requested after provision but quota exceeded", func() { - ResetTestQuayClient() + quay.ResetTestQuayClient() - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { + quay.CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { return &quay.Repository{Name: expectedImageName}, nil } - CreateRobotAccountFunc = func(organization, robotName string) (*quay.RobotAccount, error) { + quay.CreateRobotAccountFunc = func(organization, robotName string) (*quay.RobotAccount, error) { return &quay.RobotAccount{Name: robotName, Token: pushToken}, nil } createImageRepository(imageRepositoryConfig{}) @@ -641,13 +641,13 @@ var _ = Describe("Image repository controller", func() { waitImageRepositoryFinalizerOnImageRepository(resourceKey) - ResetTestQuayClientToFails() + quay.ResetTestQuayClientToFails() isChangeRepositoryVisibilityInvoked := false - ChangeRepositoryVisibilityFunc = func(organization, imageRepository, visibility string) error { + quay.ChangeRepositoryVisibilityFunc = func(organization, imageRepository, visibility string) error { defer GinkgoRecover() isChangeRepositoryVisibilityInvoked = true - Expect(organization).To(Equal(testQuayOrg)) + Expect(organization).To(Equal(quay.TestQuayOrg)) Expect(imageRepository).To(Equal(expectedImageName)) Expect(visibility).To(Equal(string(imagerepositoryv1alpha1.ImageVisibilityPrivate))) return fmt.Errorf("payment required") @@ -665,12 +665,12 @@ var _ = Describe("Image repository controller", func() { imageRepository.Status.Message != "" }, timeout, interval).Should(BeTrue()) - ResetTestQuayClient() + quay.ResetTestQuayClient() deleteImageRepository(resourceKey) }) It("should fail if invalid image repository linked by annotation to unexisting component", func() { - ResetTestQuayClientToFails() + quay.ResetTestQuayClientToFails() createImageRepository(imageRepositoryConfig{ ImageName: fmt.Sprintf("%s/%s", defaultComponentApplication, defaultComponentName), diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 0aac754..57defeb 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -112,16 +112,16 @@ var _ = BeforeSuite(func() { err = (&ImageRepositoryReconciler{ Client: k8sManager.GetClient(), Scheme: k8sManager.GetScheme(), - BuildQuayClient: func(l logr.Logger) quay.QuayService { return testQuayClient }, - QuayOrganization: testQuayOrg, + BuildQuayClient: func(l logr.Logger) quay.QuayService { return quay.TestQuayClient{} }, + QuayOrganization: quay.TestQuayOrg, }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) err = (&ComponentReconciler{ Client: k8sManager.GetClient(), Scheme: k8sManager.GetScheme(), - BuildQuayClient: func(logr.Logger) quay.QuayService { return testQuayClient }, - QuayOrganization: testQuayOrg, + BuildQuayClient: func(logr.Logger) quay.QuayService { return quay.TestQuayClient{} }, + QuayOrganization: quay.TestQuayOrg, }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) diff --git a/main.go b/main.go index 6f64077..3caf778 100644 --- a/main.go +++ b/main.go @@ -19,8 +19,10 @@ package main import ( "flag" "fmt" + "github.com/redhat-appstudio/image-controller/pkg/metrics" "net/http" "os" + cmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" "strings" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) @@ -169,8 +171,21 @@ func main() { os.Exit(1) } + ctx := ctrl.SetupSignalHandler() + quayProbe, err := metrics.NewQuayAvailabilityProbe(ctx, buildQuayClientFunc, quayOrganization) + if err != nil { + setupLog.Error(err, "unable to register quay availability probe") + os.Exit(1) + } + imageControllerMetrics := metrics.NewImageControllerMetrics([]metrics.AvailabilityProbe{quayProbe}) + if err := imageControllerMetrics.InitMetrics(cmetrics.Registry); err != nil { + setupLog.Error(err, "unable to initialize metrics") + os.Exit(1) + } + imageControllerMetrics.StartMetrics(ctx) + setupLog.Info("starting manager") - if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + if err := mgr.Start(ctx); err != nil { setupLog.Error(err, "problem running manager") os.Exit(1) } diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 0000000..99c3522 --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,94 @@ +package metrics + +import ( + "context" + "fmt" + "github.com/prometheus/client_golang/prometheus" + ctrllog "sigs.k8s.io/controller-runtime/pkg/log" + "time" +) + +const ( + MetricsNamespace = "redhat_appstudio" + MetricsSubsystem = "imagecontroller" +) + +var ( + HistogramBuckets = []float64{5, 10, 15, 20, 30, 60, 120, 300} + ImageRepositoryProvisionTimeMetric = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: MetricsNamespace, + Subsystem: MetricsSubsystem, + Buckets: HistogramBuckets, + Name: "image_repository_provision_time", + Help: "The time in seconds spent from the moment of Image repository provision request to Image repository is ready to use.", + }) + + ImageRepositoryProvisionFailureTimeMetric = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: MetricsNamespace, + Subsystem: MetricsSubsystem, + Buckets: HistogramBuckets, + Name: "image_repository_provision_failure_time", + Help: "The time in seconds spent from the moment of Image repository provision request to Image repository failure.", + }) + + RepositoryTimesForMetrics = map[string]time.Time{} +) + +func (m *ImageControllerMetrics) InitMetrics(registerer prometheus.Registerer) error { + // controller metrics + registerer.MustRegister(ImageRepositoryProvisionTimeMetric, ImageRepositoryProvisionFailureTimeMetric) + // availability metrics + for _, probe := range m.probes { + if err := registerer.Register(probe.AvailabilityGauge()); err != nil { + return fmt.Errorf("failed to register the availability metric: %w", err) + } + } + return nil +} + +// ImageControllerMetrics represents a collection of metrics to be registered on a +// Prometheus metrics registry for a image controller service. +type ImageControllerMetrics struct { + probes []AvailabilityProbe +} + +func NewImageControllerMetrics(probes []AvailabilityProbe) *ImageControllerMetrics { + return &ImageControllerMetrics{probes: probes} +} + +func (m *ImageControllerMetrics) StartMetrics(ctx context.Context) { + ticker := time.NewTicker(time.Minute) + log := ctrllog.FromContext(ctx) + log.Info("Starting image controller metrics") + go func() { + for { + select { + case <-ctx.Done(): // Shutdown if context is canceled + log.Info("Shutting down metrics") + ticker.Stop() + return + case <-ticker.C: + m.checkProbes(ctx) + } + } + }() +} + +func (m *ImageControllerMetrics) checkProbes(ctx context.Context) { + for _, probe := range m.probes { + pingErr := probe.CheckAvailability(ctx) + if pingErr != nil { + log := ctrllog.FromContext(ctx) + log.Error(pingErr, "Error checking availability probe", "probe", probe) + probe.AvailabilityGauge().Set(0) + } else { + probe.AvailabilityGauge().Set(1) + } + } +} + +// AvailabilityProbe represents a probe that checks the availability of a certain aspects of the service +type AvailabilityProbe interface { + CheckAvailability(ctx context.Context) error + AvailabilityGauge() prometheus.Gauge +} diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go new file mode 100644 index 0000000..ee9c2f5 --- /dev/null +++ b/pkg/metrics/metrics_test.go @@ -0,0 +1,41 @@ +package metrics + +import ( + "context" + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/redhat-appstudio/image-controller/pkg/quay" + "testing" +) + +func TestRegisterMetrics(t *testing.T) { + t.Run("Should register and record availability metric", func(t *testing.T) { + probe, err := NewQuayAvailabilityProbe(context.Background(), getTestClient, quay.TestQuayOrg) + if err != nil { + t.Errorf("Fail to register probe: %v", err) + } + buildMetrics := NewImageControllerMetrics([]AvailabilityProbe{probe}) + registry := prometheus.NewPedanticRegistry() + err = buildMetrics.InitMetrics(registry) + if err != nil { + t.Errorf("Fail to register metrics: %v", err) + } + + buildMetrics.checkProbes(context.Background()) + + count, err := testutil.GatherAndCount(registry, "redhat_appstudio_imagecontroller_global_quay_app_available") + if err != nil { + t.Errorf("Fail to gather metrics: %v", err) + } + + if count != 1 { + t.Errorf("Fail to record metric. Expected 1 got : %v", count) + } + }) +} + +func getTestClient(logger logr.Logger) quay.QuayService { + quay.ResetTestQuayClient() + return &quay.TestQuayClient{} +} diff --git a/pkg/metrics/quay.go b/pkg/metrics/quay.go new file mode 100644 index 0000000..10f3b7a --- /dev/null +++ b/pkg/metrics/quay.go @@ -0,0 +1,47 @@ +package metrics + +import ( + "context" + "fmt" + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + "github.com/redhat-appstudio/image-controller/pkg/quay" + ctrllog "sigs.k8s.io/controller-runtime/pkg/log" +) + +type QuayAvailabilityProbe struct { + BuildQuayClient func(logr.Logger) quay.QuayService + QuayOrganization string + gauge prometheus.Gauge +} + +const testRobotAccountName = "robot_konflux_api_healthcheck" + +func NewQuayAvailabilityProbe(ctx context.Context, clientBuilder func(logr.Logger) quay.QuayService, quayOrganization string) (*QuayAvailabilityProbe, error) { + client := clientBuilder(ctrllog.FromContext(ctx)) + _, err := client.CreateRobotAccount(quayOrganization, testRobotAccountName) + if err != nil { + return nil, fmt.Errorf("could not create test robot account: %w", err) + } + return &QuayAvailabilityProbe{ + BuildQuayClient: clientBuilder, + QuayOrganization: quayOrganization, + gauge: prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: MetricsNamespace, + Subsystem: MetricsSubsystem, + Name: "global_quay_app_available", + Help: "The availability of the Quay App", + }), + }, nil +} + +func (q *QuayAvailabilityProbe) CheckAvailability(ctx context.Context) error { + client := q.BuildQuayClient(ctrllog.FromContext(ctx)) + _, err := client.GetRobotAccount(q.QuayOrganization, testRobotAccountName) + return err +} + +func (q *QuayAvailabilityProbe) AvailabilityGauge() prometheus.Gauge { + return q.gauge +} diff --git a/controllers/suite_util_quay_client_test.go b/pkg/quay/test_quay_client.go similarity index 61% rename from controllers/suite_util_quay_client_test.go rename to pkg/quay/test_quay_client.go index 3853bfe..eb2de6e 100644 --- a/controllers/suite_util_quay_client_test.go +++ b/pkg/quay/test_quay_client.go @@ -14,49 +14,45 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controllers +package quay import ( . "github.com/onsi/ginkgo/v2" - - "github.com/redhat-appstudio/image-controller/pkg/quay" ) const ( - testQuayOrg = "user-workloads" + TestQuayOrg = "user-workloads" ) // TestQuayClient is a QuayClient for testing the controller type TestQuayClient struct{} -var _ quay.QuayService = (*TestQuayClient)(nil) +var _ QuayService = (*TestQuayClient)(nil) var ( - testQuayClient = &TestQuayClient{} - - CreateRepositoryFunc func(repository quay.RepositoryRequest) (*quay.Repository, error) + CreateRepositoryFunc func(repository RepositoryRequest) (*Repository, error) DeleteRepositoryFunc func(organization, imageRepository string) (bool, error) ChangeRepositoryVisibilityFunc func(organization, imageRepository string, visibility string) error - GetRobotAccountFunc func(organization string, robotName string) (*quay.RobotAccount, error) - CreateRobotAccountFunc func(organization string, robotName string) (*quay.RobotAccount, error) + GetRobotAccountFunc func(organization string, robotName string) (*RobotAccount, error) + CreateRobotAccountFunc func(organization string, robotName string) (*RobotAccount, error) DeleteRobotAccountFunc func(organization string, robotName string) (bool, error) AddPermissionsForRepositoryToRobotAccountFunc func(organization, imageRepository, robotAccountName string, isWrite bool) error - RegenerateRobotAccountTokenFunc func(organization string, robotName string) (*quay.RobotAccount, error) + RegenerateRobotAccountTokenFunc func(organization string, robotName string) (*RobotAccount, error) ) func ResetTestQuayClient() { - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { return &quay.Repository{}, nil } + CreateRepositoryFunc = func(repository RepositoryRequest) (*Repository, error) { return &Repository{}, nil } DeleteRepositoryFunc = func(organization, imageRepository string) (bool, error) { return true, nil } ChangeRepositoryVisibilityFunc = func(organization, imageRepository string, visibility string) error { return nil } - GetRobotAccountFunc = func(organization, robotName string) (*quay.RobotAccount, error) { return &quay.RobotAccount{}, nil } - CreateRobotAccountFunc = func(organization, robotName string) (*quay.RobotAccount, error) { return &quay.RobotAccount{}, nil } + GetRobotAccountFunc = func(organization, robotName string) (*RobotAccount, error) { return &RobotAccount{}, nil } + CreateRobotAccountFunc = func(organization, robotName string) (*RobotAccount, error) { return &RobotAccount{}, nil } DeleteRobotAccountFunc = func(organization, robotName string) (bool, error) { return true, nil } AddPermissionsForRepositoryToRobotAccountFunc = func(organization, imageRepository, robotAccountName string, isWrite bool) error { return nil } - RegenerateRobotAccountTokenFunc = func(organization, robotName string) (*quay.RobotAccount, error) { return &quay.RobotAccount{}, nil } + RegenerateRobotAccountTokenFunc = func(organization, robotName string) (*RobotAccount, error) { return &RobotAccount{}, nil } } func ResetTestQuayClientToFails() { - CreateRepositoryFunc = func(repository quay.RepositoryRequest) (*quay.Repository, error) { + CreateRepositoryFunc = func(repository RepositoryRequest) (*Repository, error) { defer GinkgoRecover() Fail("CreateRepositoryFunc invoked") return nil, nil @@ -71,12 +67,12 @@ func ResetTestQuayClientToFails() { Fail("ChangeRepositoryVisibility invoked") return nil } - GetRobotAccountFunc = func(organization, robotName string) (*quay.RobotAccount, error) { + GetRobotAccountFunc = func(organization, robotName string) (*RobotAccount, error) { defer GinkgoRecover() Fail("GetRobotAccount invoked") return nil, nil } - CreateRobotAccountFunc = func(organization, robotName string) (*quay.RobotAccount, error) { + CreateRobotAccountFunc = func(organization, robotName string) (*RobotAccount, error) { defer GinkgoRecover() Fail("CreateRobotAccount invoked") return nil, nil @@ -91,46 +87,46 @@ func ResetTestQuayClientToFails() { Fail("AddPermissionsForRepositoryToRobotAccount invoked") return nil } - RegenerateRobotAccountTokenFunc = func(organization, robotName string) (*quay.RobotAccount, error) { + RegenerateRobotAccountTokenFunc = func(organization, robotName string) (*RobotAccount, error) { defer GinkgoRecover() Fail("RegenerateRobotAccountToken invoked") return nil, nil } } -func (c *TestQuayClient) CreateRepository(repositoryRequest quay.RepositoryRequest) (*quay.Repository, error) { +func (c TestQuayClient) CreateRepository(repositoryRequest RepositoryRequest) (*Repository, error) { return CreateRepositoryFunc(repositoryRequest) } -func (c *TestQuayClient) DeleteRepository(organization, imageRepository string) (bool, error) { +func (c TestQuayClient) DeleteRepository(organization, imageRepository string) (bool, error) { return DeleteRepositoryFunc(organization, imageRepository) } -func (*TestQuayClient) ChangeRepositoryVisibility(organization, imageRepository string, visibility string) error { +func (TestQuayClient) ChangeRepositoryVisibility(organization, imageRepository string, visibility string) error { return ChangeRepositoryVisibilityFunc(organization, imageRepository, visibility) } -func (c *TestQuayClient) GetRobotAccount(organization string, robotName string) (*quay.RobotAccount, error) { +func (c TestQuayClient) GetRobotAccount(organization string, robotName string) (*RobotAccount, error) { return GetRobotAccountFunc(organization, robotName) } -func (c *TestQuayClient) CreateRobotAccount(organization string, robotName string) (*quay.RobotAccount, error) { +func (c TestQuayClient) CreateRobotAccount(organization string, robotName string) (*RobotAccount, error) { return CreateRobotAccountFunc(organization, robotName) } -func (c *TestQuayClient) DeleteRobotAccount(organization string, robotName string) (bool, error) { +func (c TestQuayClient) DeleteRobotAccount(organization string, robotName string) (bool, error) { return DeleteRobotAccountFunc(organization, robotName) } -func (c *TestQuayClient) AddPermissionsForRepositoryToRobotAccount(organization, imageRepository, robotAccountName string, isWrite bool) error { +func (c TestQuayClient) AddPermissionsForRepositoryToRobotAccount(organization, imageRepository, robotAccountName string, isWrite bool) error { return AddPermissionsForRepositoryToRobotAccountFunc(organization, imageRepository, robotAccountName, isWrite) } -func (c *TestQuayClient) RegenerateRobotAccountToken(organization string, robotName string) (*quay.RobotAccount, error) { +func (c TestQuayClient) RegenerateRobotAccountToken(organization string, robotName string) (*RobotAccount, error) { return RegenerateRobotAccountTokenFunc(organization, robotName) } -func (c *TestQuayClient) GetAllRepositories(organization string) ([]quay.Repository, error) { +func (c TestQuayClient) GetAllRepositories(organization string) ([]Repository, error) { return nil, nil } -func (c *TestQuayClient) GetAllRobotAccounts(organization string) ([]quay.RobotAccount, error) { +func (c TestQuayClient) GetAllRobotAccounts(organization string) ([]RobotAccount, error) { return nil, nil } -func (*TestQuayClient) DeleteTag(organization string, repository string, tag string) (bool, error) { +func (TestQuayClient) DeleteTag(organization string, repository string, tag string) (bool, error) { return true, nil } -func (*TestQuayClient) GetTagsFromPage(organization string, repository string, page int) ([]quay.Tag, bool, error) { +func (TestQuayClient) GetTagsFromPage(organization string, repository string, page int) ([]Tag, bool, error) { return nil, false, nil }