Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add availability probe for image controller #101

Merged
merged 8 commits into from
Apr 11, 2024
Merged

Add availability probe for image controller #101

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
54df6b7
Ad availability probe for image controller
mshaposhnik Apr 3, 2024
d75bc4b
Fixups
mshaposhnik Apr 3, 2024
4b110a6
Change probe method
mshaposhnik Apr 4, 2024
60b1a6a
Minor fix
mshaposhnik Apr 5, 2024
8206677
Merge branch 'main' into STONEBLD-1931
mshaposhnik Apr 8, 2024
c26bb36
Fixups
mshaposhnik Apr 10, 2024
5090cfc
Review fixups
mshaposhnik Apr 10, 2024
8375566
Merge branch 'main' into STONEBLD-1931
mshaposhnik Apr 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 5 additions & 8 deletions controllers/component_image_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"encoding/base64"
"encoding/json"
"fmt"
"github.com/redhat-appstudio/image-controller/pkg/metrics"
"strings"
"time"

Expand Down Expand Up @@ -79,10 +80,6 @@ type ComponentReconciler struct {

// SetupWithManager sets up the controller with the Manager.
func (r *ComponentReconciler) SetupWithManager(mgr ctrl.Manager) error {
if err := initMetrics(); err != nil {
return err
}

return ctrl.NewControllerManagedBy(mgr).
For(&appstudioredhatcomv1alpha1.Component{}).
Complete(r)
Expand Down Expand Up @@ -117,7 +114,7 @@ func (r *ComponentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (

if !component.ObjectMeta.DeletionTimestamp.IsZero() {
// remove component from metrics map
delete(repositoryTimesForMetrics, componentIdForMetrics)
delete(metrics.RepositoryTimesForMetrics, componentIdForMetrics)

if controllerutil.ContainsFinalizer(component, ImageRepositoryComponentFinalizer) {
pushRobotAccountName, pullRobotAccountName := generateRobotAccountsNames(component)
Expand Down Expand Up @@ -320,9 +317,9 @@ func (r *ComponentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
})
}

imageRepositoryProvisionTimeMetric.Observe(time.Since(repositoryTimesForMetrics[componentIdForMetrics]).Seconds())
metrics.ImageRepositoryProvisionTimeMetric.Observe(time.Since(metrics.RepositoryTimesForMetrics[componentIdForMetrics]).Seconds())
// remove component from metrics map
delete(repositoryTimesForMetrics, componentIdForMetrics)
delete(metrics.RepositoryTimesForMetrics, componentIdForMetrics)

return ctrl.Result{}, nil
}
Expand All @@ -338,7 +335,7 @@ func (r *ComponentReconciler) reportError(ctx context.Context, component *appstu

componentIdForMetrics := getComponentIdForMetrics(component)
// remove component from metrics map, permanent error
delete(repositoryTimesForMetrics, componentIdForMetrics)
delete(metrics.RepositoryTimesForMetrics, componentIdForMetrics)

return r.Client.Update(ctx, component)
}
Expand Down
90 changes: 45 additions & 45 deletions controllers/component_image_controller_test.go
View file
Open in desktop

Large diffs are not rendered by default.

85 changes: 16 additions & 69 deletions controllers/imagerepository_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,16 @@ import (
"crypto/rand"
"encoding/hex"
"fmt"
"github.com/redhat-appstudio/image-controller/pkg/metrics"
"strings"
"time"

"github.com/go-logr/logr"
appstudioredhatcomv1alpha1 "github.com/redhat-appstudio/application-api/api/v1alpha1"
imagerepositoryv1alpha1 "github.com/redhat-appstudio/image-controller/api/v1alpha1"
l "github.com/redhat-appstudio/image-controller/pkg/logs"
"github.com/redhat-appstudio/image-controller/pkg/quay"
remotesecretv1beta1 "github.com/redhat-appstudio/remote-secret/api/v1beta1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -33,15 +40,6 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
ctrllog "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/metrics"

"github.com/go-logr/logr"
"github.com/prometheus/client_golang/prometheus"
appstudioredhatcomv1alpha1 "github.com/redhat-appstudio/application-api/api/v1alpha1"
imagerepositoryv1alpha1 "github.com/redhat-appstudio/image-controller/api/v1alpha1"
l "github.com/redhat-appstudio/image-controller/pkg/logs"
"github.com/redhat-appstudio/image-controller/pkg/quay"
remotesecretv1beta1 "github.com/redhat-appstudio/remote-secret/api/v1beta1"
)

const (
Expand All @@ -51,15 +49,6 @@ const (

buildPipelineServiceAccountName = "appstudio-pipeline"
updateComponentAnnotationName = "image-controller.appstudio.redhat.com/update-component-image"

metricsNamespace = "redhat_appstudio"
metricsSubsystem = "imagecontroller"
)

var (
imageRepositoryProvisionTimeMetric prometheus.Histogram
imageRepositoryProvisionFailureTimeMetric prometheus.Histogram
repositoryTimesForMetrics = map[string]time.Time{}
)

// ImageRepositoryReconciler reconciles a ImageRepository object
Expand All @@ -72,59 +61,17 @@ type ImageRepositoryReconciler struct {
QuayOrganization string
}

func initMetrics() error {
buckets := getProvisionTimeMetricsBuckets()

// don't register it if it was already registered by another controller
if imageRepositoryProvisionTimeMetric != nil {
return nil
}

imageRepositoryProvisionTimeMetric = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Buckets: buckets,
Name: "image_repository_provision_time",
Help: "The time in seconds spent from the moment of Image repository provision request to Image repository is ready to use.",
})

imageRepositoryProvisionFailureTimeMetric = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Buckets: buckets,
Name: "image_repository_provision_failure_time",
Help: "The time in seconds spent from the moment of Image repository provision request to Image repository failure.",
})

if err := metrics.Registry.Register(imageRepositoryProvisionTimeMetric); err != nil {
return fmt.Errorf("failed to register the image_repository_provision_time metric: %w", err)
}
if err := metrics.Registry.Register(imageRepositoryProvisionFailureTimeMetric); err != nil {
return fmt.Errorf("failed to register the image_repository_provision_failure_time metric: %w", err)
}

return nil
}

func getProvisionTimeMetricsBuckets() []float64 {
return []float64{5, 10, 15, 20, 30, 60, 120, 300}
}

// SetupWithManager sets up the controller with the Manager.
func (r *ImageRepositoryReconciler) SetupWithManager(mgr ctrl.Manager) error {
if err := initMetrics(); err != nil {
return err
}

return ctrl.NewControllerManagedBy(mgr).
For(&imagerepositoryv1alpha1.ImageRepository{}).
Complete(r)
}

func setMetricsTime(idForMetrics string, reconcileStartTime time.Time) {
_, timeRecorded := repositoryTimesForMetrics[idForMetrics]
_, timeRecorded := metrics.RepositoryTimesForMetrics[idForMetrics]
if !timeRecorded {
repositoryTimesForMetrics[idForMetrics] = reconcileStartTime
metrics.RepositoryTimesForMetrics[idForMetrics] = reconcileStartTime
}
}

Expand Down Expand Up @@ -156,7 +103,7 @@ func (r *ImageRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Requ

if !imageRepository.DeletionTimestamp.IsZero() {
// remove component from metrics map
delete(repositoryTimesForMetrics, repositoryIdForMetrics)
delete(metrics.RepositoryTimesForMetrics, repositoryIdForMetrics)

// Reread quay token
r.QuayClient = r.BuildQuayClient(log)
Expand All @@ -176,12 +123,12 @@ func (r *ImageRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Requ
}

if imageRepository.Status.State == imagerepositoryv1alpha1.ImageRepositoryStateFailed {
provisionTime, timeRecorded := repositoryTimesForMetrics[repositoryIdForMetrics]
provisionTime, timeRecorded := metrics.RepositoryTimesForMetrics[repositoryIdForMetrics]
if timeRecorded {
imageRepositoryProvisionFailureTimeMetric.Observe(time.Since(provisionTime).Seconds())
metrics.ImageRepositoryProvisionFailureTimeMetric.Observe(time.Since(provisionTime).Seconds())

// remove component from metrics map
delete(repositoryTimesForMetrics, repositoryIdForMetrics)
delete(metrics.RepositoryTimesForMetrics, repositoryIdForMetrics)
}

return ctrl.Result{}, nil
Expand Down Expand Up @@ -274,12 +221,12 @@ func (r *ImageRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Requ

// we are adding to map only for new provision, not for some partial actions,
// so report time only if time was recorded
provisionTime, timeRecorded := repositoryTimesForMetrics[repositoryIdForMetrics]
provisionTime, timeRecorded := metrics.RepositoryTimesForMetrics[repositoryIdForMetrics]
if timeRecorded {
imageRepositoryProvisionTimeMetric.Observe(time.Since(provisionTime).Seconds())
metrics.ImageRepositoryProvisionTimeMetric.Observe(time.Since(provisionTime).Seconds())
}
// remove component from metrics map
delete(repositoryTimesForMetrics, repositoryIdForMetrics)
delete(metrics.RepositoryTimesForMetrics, repositoryIdForMetrics)

return ctrl.Result{}, nil
}
Expand Down
Loading
Loading