Telemetry metrics: add support for metrics collection to openshift-mo…

…nitor in the controller.

Signed-off-by: Bella Khizgiyaev <[email protected]>

cmd/forklift-controller/BUILD.bazel

@@ -43,6 +43,7 @@ go_library(
         "//vendor/github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1:monitoring",
         "//vendor/github.com/prometheus/client_golang/prometheus/promhttp",
         "//vendor/k8s.io/apimachinery/pkg/api/errors",
+        "//vendor/k8s.io/client-go/kubernetes",
         "//vendor/k8s.io/client-go/plugin/pkg/client/auth/gcp",
         "//vendor/kubevirt.io/api/core/v1:core",
         "//vendor/kubevirt.io/api/export/v1alpha1",

cmd/forklift-controller/main.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"os"
 	"time"
@@ -35,6 +36,7 @@ import (
 	promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/client-go/kubernetes"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	cnv "kubevirt.io/api/core/v1"
 	export "kubevirt.io/api/export/v1alpha1"
@@ -129,50 +131,94 @@ func main() {
 		log.Error(err, "proceeding without optional kubevirt instance type APIs")
 	}
 
-	mgr.Add(manager.RunnableFunc(func(ctx context.Context) error {
-		// Setup Prometheus recording rules
-		log.Info("Setting up Prometheus recording rules")
-		namespace := "konveyor-forklift" // Replace with the appropriate namespace
-		err = rules.SetupRules(namespace)
-		if err != nil {
-			log.Error(err, "unable to set up Prometheus recording rules")
-			return err
-		}
-
-		// Build the PrometheusRule
-		promRule, err := rules.BuildPrometheusRule(namespace)
-		if err != nil {
-			log.Error(err, "unable to build PrometheusRule")
-			return err
-		}
-
-		// Create or Update the PrometheusRule in the Kubernetes cluster
-		existingPromRule := &promv1.PrometheusRule{}
-		err = mgr.GetClient().Get(context.TODO(), client.ObjectKey{
-			Namespace: namespace,
-			Name:      promRule.Name,
-		}, existingPromRule)
-		if err != nil {
-			if errors.IsNotFound(err) {
-				err = mgr.GetClient().Create(context.TODO(), promRule)
-				if err != nil {
-					log.Error(err, "unable to create PrometheusRule")
-					return err
-				}
-			} else {
-				log.Error(err, "unable to get PrometheusRule")
+	openshift := os.Getenv("OPENSHIFT")
+	if openshift == "" {
+		openshift = "false"
+	}
+	// Clusters without OpenShift do not run OpenShift monitoring out of the box,
+	// and hence are not able to be registered to the monitoring services.
+	if openshift == "true" {
+		mgr.Add(manager.RunnableFunc(func(ctx context.Context) error {
+			log.Info("waiting for cache to sync")
+			if !mgr.GetCache().WaitForCacheSync(ctx) {
+				log.Error(fmt.Errorf("failed to wait for cache sync"), "cache sync failed")
+				return fmt.Errorf("failed to wait for cache sync")
+			}
+
+			clientset, err := kubernetes.NewForConfig(cfg)
+			if err != nil {
+				log.Error(err, "unable to create Kubernetes client")
+				os.Exit(1)
+			}
+
+			log.Info("Setting up Prometheus recording rules")
+
+			namespace := os.Getenv("POD_NAMESPACE")
+			if namespace == "" {
+				namespace = "openshift-mtv"
+			}
+			err = rules.SetupRules(namespace)
+			if err != nil {
+				log.Error(err, "unable to set up Prometheus recording rules")
 				return err
 			}
-		} else {
-			promRule.ResourceVersion = existingPromRule.ResourceVersion
-			err = mgr.GetClient().Update(context.TODO(), promRule)
+
+			ownerRef, err := rules.GetDeploymentInfo(clientset, namespace, "forklift-controller")
+			if err != nil {
+				log.Error(err, "Failed to get owner refernce")
+			}
+
+			promRule, err := rules.BuildPrometheusRule(namespace, ownerRef)
 			if err != nil {
-				log.Error(err, "unable to update PrometheusRule")
+				log.Error(err, "unable to build PrometheusRule")
 				return err
 			}
-		}
-		return nil
-	}))
+
+			err = rules.PatchMonitorinLable(namespace, clientset)
+			if err != nil {
+				log.Error(err, "unable to patch monitor label")
+				return err
+			}
+
+			err = rules.CreateMetricsService(clientset, namespace, ownerRef)
+			if err != nil {
+				log.Error(err, "unable to create metrics Service")
+				return err
+			}
+
+			err = rules.CreateServiceMonitor(mgr.GetClient(), namespace, ownerRef)
+			if err != nil {
+				log.Error(err, "unable to create ServiceMonitor")
+				return err
+			}
+
+			existingPromRule := &promv1.PrometheusRule{}
+			err = mgr.GetClient().Get(context.TODO(), client.ObjectKey{
+				Namespace: namespace,
+				Name:      promRule.Name,
+			}, existingPromRule)
+			if err != nil {
+				if errors.IsNotFound(err) {
+					err = mgr.GetClient().Create(context.TODO(), promRule)
+					if err != nil {
+						log.Error(err, "unable to create PrometheusRule")
+						return err
+					}
+				} else {
+					log.Error(err, "unable to get PrometheusRule")
+					return err
+				}
+			} else {
+				promRule.ResourceVersion = existingPromRule.ResourceVersion
+				err = mgr.GetClient().Update(context.TODO(), promRule)
+				if err != nil {
+					log.Error(err, "unable to update PrometheusRule")
+					return err
+				}
+			}
+			return nil
+		}))
+	}
 
 	// Setup all Controllers
 	log.Info("Setting up controller")

operator/config/rbac/forklift-controller_role.yaml

@@ -147,6 +147,10 @@ rules:
   - monitoring.coreos.com
   resources:
   - prometheusrules
+  - servicemonitors
   verbs:
   - get
   - list
+  - create
+  - update
+  - patch

operator/roles/forkliftcontroller/templates/controller/deployment-controller.yml.j2

@@ -21,6 +21,7 @@ spec:
         app: {{ app_name }}
         control-plane: controller-manager
         controller-tools.k8s.io: "1.0"
+        prometheus.forklift.konveyor.io: "true" 
       annotations:
         configHash: "{{ (inventory_volume_path | string) }}"
     spec:

pkg/monitoring/metrics/forklift-controller/metrics.go

@@ -23,7 +23,7 @@ const (
 var (
 	// 'status' - [ succeeded, failed ]
 	// 'provider' - [oVirt, vSphere, Openstack, OVA, openshift]
-	planStatusGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{
+	migrationGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "mtv_workload_plans",
 		Help: "VM migration Plans sorted by status and provider type",
 	},

pkg/monitoring/rules/BUILD.bazel

@@ -9,5 +9,12 @@ go_library(
         "//pkg/monitoring/rules/recordingrules",
         "//vendor/github.com/machadovilaca/operator-observability/pkg/operatorrules",
         "//vendor/github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1:monitoring",
+        "//vendor/k8s.io/api/core/v1:core",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:meta",
+        "//vendor/k8s.io/apimachinery/pkg/types",
+        "//vendor/k8s.io/apimachinery/pkg/util/intstr",
+        "//vendor/k8s.io/client-go/kubernetes",
+        "//vendor/sigs.k8s.io/controller-runtime/pkg/client",
     ],
 )