From 5138dd51182cf47392b15fc6624f8f1a61957da7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EB=B3=80=EC=9E=AC=ED=95=9C?= Date: Tue, 5 Nov 2024 22:18:36 +0900 Subject: [PATCH] feat: add orphaned resource count metric (#20521) Signed-off-by: jaehanbyun --- controller/appcontroller.go | 1 + controller/metrics/metrics.go | 16 ++++++++++++++++ controller/metrics/metrics_test.go | 25 +++++++++++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/controller/appcontroller.go b/controller/appcontroller.go index a3947ca634a72..5a48f1d41cb09 100644 --- a/controller/appcontroller.go +++ b/controller/appcontroller.go @@ -628,6 +628,7 @@ func (ctrl *ApplicationController) getResourceTree(a *appv1.Application, managed Message: fmt.Sprintf("Application has %d orphaned resources", len(orphanedNodes)), }} } + ctrl.metricsServer.SetOrphanedResourcesMetric(a, len(orphanedNodes)) a.Status.SetConditions(conditions, map[appv1.ApplicationConditionType]bool{appv1.ApplicationConditionOrphanedResourceWarning: true}) sort.Slice(orphanedNodes, func(i, j int) bool { return orphanedNodes[i].ResourceRef.String() < orphanedNodes[j].ResourceRef.String() diff --git a/controller/metrics/metrics.go b/controller/metrics/metrics.go index a9df75aff8015..b8674cd231b09 100644 --- a/controller/metrics/metrics.go +++ b/controller/metrics/metrics.go @@ -33,6 +33,7 @@ type MetricsServer struct { syncCounter *prometheus.CounterVec kubectlExecCounter *prometheus.CounterVec kubectlExecPendingGauge *prometheus.GaugeVec + orphanedResourcesGauge *prometheus.GaugeVec k8sRequestCounter *prometheus.CounterVec clusterEventsCounter *prometheus.CounterVec redisRequestCounter *prometheus.CounterVec @@ -144,6 +145,14 @@ var ( }, []string{"hostname", "initiator"}, ) + + orphanedResourcesGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "argocd_app_orphaned_resources_count", + Help: "Number of orphaned resources per application", + }, + descAppDefaultLabels, + ) ) // NewMetricsServer returns a new prometheus server which collects application metrics @@ -188,6 +197,7 @@ func NewMetricsServer(addr string, appLister applister.ApplicationLister, appFil registry.MustRegister(k8sRequestCounter) registry.MustRegister(kubectlExecCounter) registry.MustRegister(kubectlExecPendingGauge) + registry.MustRegister(orphanedResourcesGauge) registry.MustRegister(reconcileHistogram) registry.MustRegister(clusterEventsCounter) registry.MustRegister(redisRequestCounter) @@ -203,6 +213,7 @@ func NewMetricsServer(addr string, appLister applister.ApplicationLister, appFil k8sRequestCounter: k8sRequestCounter, kubectlExecCounter: kubectlExecCounter, kubectlExecPendingGauge: kubectlExecPendingGauge, + orphanedResourcesGauge: orphanedResourcesGauge, reconcileHistogram: reconcileHistogram, clusterEventsCounter: clusterEventsCounter, redisRequestCounter: redisRequestCounter, @@ -241,6 +252,10 @@ func (m *MetricsServer) DecKubectlExecPending(command string) { m.kubectlExecPendingGauge.WithLabelValues(m.hostname, command).Dec() } +func (m *MetricsServer) SetOrphanedResourcesMetric(app *argoappv1.Application, numOrphanedResources int) { + m.orphanedResourcesGauge.WithLabelValues(app.Namespace, app.Name, app.Spec.GetProject()).Set(float64(numOrphanedResources)) +} + // IncClusterEventsCount increments the number of cluster events func (m *MetricsServer) IncClusterEventsCount(server, group, kind string) { m.clusterEventsCounter.WithLabelValues(server, group, kind).Inc() @@ -290,6 +305,7 @@ func (m *MetricsServer) SetExpiration(cacheExpiration time.Duration) error { m.syncCounter.Reset() m.kubectlExecCounter.Reset() m.kubectlExecPendingGauge.Reset() + m.orphanedResourcesGauge.Reset() m.k8sRequestCounter.Reset() m.clusterEventsCounter.Reset() m.redisRequestCounter.Reset() diff --git a/controller/metrics/metrics_test.go b/controller/metrics/metrics_test.go index e86c6cc6c42f9..383cb0c0c3d5a 100644 --- a/controller/metrics/metrics_test.go +++ b/controller/metrics/metrics_test.go @@ -509,6 +509,31 @@ argocd_app_reconcile_count{dest_server="https://localhost:6443",namespace="argoc assertMetricsPrinted(t, appReconcileMetrics, body) } +func TestOrphanedResourcesMetric(t *testing.T) { + cancel, appLister := newFakeLister() + defer cancel() + metricsServ, err := NewMetricsServer("localhost:8082", appLister, appFilter, noOpHealthCheck, []string{}, []string{}) + require.NoError(t, err) + + expectedMetrics := ` +# HELP argocd_app_orphaned_resources_count Number of orphaned resources per application +# TYPE argocd_app_orphaned_resources_count gauge +argocd_app_orphaned_resources_count{name="my-app-4",namespace="argocd",project="important-project"} 1 +` + app := newFakeApp(fakeApp4) + numOrphanedResources := 1 + metricsServ.SetOrphanedResourcesMetric(app, numOrphanedResources) + + req, err := http.NewRequest(http.MethodGet, "/metrics", nil) + require.NoError(t, err) + rr := httptest.NewRecorder() + metricsServ.Handler.ServeHTTP(rr, req) + assert.Equal(t, http.StatusOK, rr.Code) + body := rr.Body.String() + log.Println(body) + assertMetricsPrinted(t, expectedMetrics, body) +} + func TestMetricsReset(t *testing.T) { cancel, appLister := newFakeLister() defer cancel()