Add monitor resources creation to forklift operator

Signed-off-by: Bella Khizgiyaev <[email protected]>

cmd/forklift-controller/BUILD.bazel

@@ -33,16 +33,13 @@ go_library(
         "//pkg/apis",
         "//pkg/controller",
         "//pkg/lib/logging",
-        "//pkg/monitoring/metrics",
         "//pkg/settings",
         "//pkg/webhook",
         "//vendor/github.com/go-logr/logr",
         "//vendor/github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1:k8s_cni_cncf_io",
         "//vendor/github.com/openshift/api/template/v1:template",
         "//vendor/github.com/pkg/profile",
-        "//vendor/github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1:monitoring",
         "//vendor/github.com/prometheus/client_golang/prometheus/promhttp",
-        "//vendor/k8s.io/client-go/kubernetes",
         "//vendor/k8s.io/client-go/plugin/pkg/client/auth/gcp",
         "//vendor/kubevirt.io/api/core/v1:core",
         "//vendor/kubevirt.io/api/export/v1alpha1",

cmd/forklift-controller/main.go

@@ -17,8 +17,6 @@ limitations under the License.
 package main
 
 import (
-	"context"
-	"fmt"
 	"net/http"
 	"os"
 	"time"
@@ -28,14 +26,11 @@ import (
 	"github.com/konveyor/forklift-controller/pkg/apis"
 	"github.com/konveyor/forklift-controller/pkg/controller"
 	"github.com/konveyor/forklift-controller/pkg/lib/logging"
-	"github.com/konveyor/forklift-controller/pkg/monitoring/metrics"
 	"github.com/konveyor/forklift-controller/pkg/settings"
 	"github.com/konveyor/forklift-controller/pkg/webhook"
 	template "github.com/openshift/api/template/v1"
 	"github.com/pkg/profile"
-	promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"k8s.io/client-go/kubernetes"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	cnv "kubevirt.io/api/core/v1"
 	export "kubevirt.io/api/export/v1alpha1"
@@ -118,75 +113,12 @@ func main() {
 		log.Error(err, "unable to add kubevirt export APIs to scheme")
 		os.Exit(1)
 	}
-	if err := promv1.AddToScheme(mgr.GetScheme()); err != nil {
-		log.Error(err, "unable to add Prometheus APIs to scheme")
-		os.Exit(1)
-	}
 	if err := template.Install(mgr.GetScheme()); err != nil {
 		log.Error(err, "proceeding without optional OpenShift template APIs")
 	}
 	if err := instancetype.AddToScheme(mgr.GetScheme()); err != nil {
 		log.Error(err, "proceeding without optional kubevirt instance type APIs")
 	}
-
-	openshift := os.Getenv("OPENSHIFT")
-	if openshift == "" {
-		openshift = "false"
-	}
-	// Clusters without OpenShift do not run OpenShift monitoring out of the box,
-	// and hence are not able to be registered to the monitoring services.
-	if openshift == "true" {
-		err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error {
-			log.Info("waiting for cache to sync")
-			if !mgr.GetCache().WaitForCacheSync(ctx) {
-				log.Error(fmt.Errorf("failed to wait for cache sync"), "cache sync failed")
-				return fmt.Errorf("failed to wait for cache sync")
-			}
-
-			clientset, err := kubernetes.NewForConfig(cfg)
-			if err != nil {
-				log.Error(err, "unable to create Kubernetes client")
-				os.Exit(1)
-			}
-
-			log.Info("Setting up Prometheus recording rules")
-
-			namespace := os.Getenv("POD_NAMESPACE")
-			if namespace == "" {
-				namespace = "openshift-mtv"
-			}
-
-			ownerRef, err := metrics.GetDeploymentInfo(clientset, namespace, "forklift-controller")
-			if err != nil {
-				log.Error(err, "Failed to get owner refernce")
-			}
-
-			err = metrics.PatchMonitorinLable(namespace, clientset)
-			if err != nil {
-				log.Error(err, "unable to patch monitor label")
-				return err
-			}
-
-			err = metrics.CreateMetricsService(clientset, namespace, ownerRef)
-			if err != nil {
-				log.Error(err, "unable to create metrics Service")
-				return err
-			}
-
-			err = metrics.CreateServiceMonitor(mgr.GetClient(), namespace, ownerRef)
-			if err != nil {
-				log.Error(err, "unable to create ServiceMonitor")
-				return err
-			}
-
-			return nil
-		}))
-		if err != nil {
-			log.Error(err, "unable to set monitoring services for telemetry")
-			os.Exit(1)
-		}
-	}
-
 	// Setup all Controllers
 	log.Info("Setting up controller")
 	if err := controller.AddToManager(mgr); err != nil {

go.mod

@@ -21,7 +21,6 @@ require (
 	github.com/ovirt/go-ovirt v0.0.0-20230808190322-9fd1992199b2
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/profile v1.7.0
-	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.68.0
 	github.com/prometheus/client_golang v1.17.0
 	github.com/prometheus/client_model v0.5.0
 	github.com/prometheus/common v0.45.0

go.sum

@@ -252,8 +252,6 @@ github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDj
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.68.0 h1:yl9ceUSUBo9woQIO+8eoWpcxZkdZgm89g+rVvu37TUw=
-github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.68.0/go.mod h1:9Uuu3pEU2jB8PwuqkHvegQ0HV/BlZRJUyfTYAqfdVF8=
 github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
 github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=

operator/config/rbac/forklift-controller_role.yaml

@@ -143,16 +143,3 @@ rules:
   verbs:
   - get
   - list
-- apiGroups:
-  - monitoring.coreos.com
-  resources:
-  - prometheusrules
-  - servicemonitors
-  verbs:
-  - get
-  - list
-  - create
-  - update
-  - patch
-  - delete
-

operator/config/rbac/leader_election_role.yaml

@@ -15,6 +15,7 @@ rules:
   - events
   - configmaps
   - secrets
+  - namespaces
   verbs:
   - '*'
 - apiGroups:
@@ -39,6 +40,10 @@ rules:
   verbs:
   - get
   - create
+  - update
+  - patch
+  - list
+  - watch
 - apiGroups:
   - apps.openshift.io
   resources:

operator/config/rbac/role.yaml

@@ -4,6 +4,13 @@ kind: ClusterRole
 metadata:
   name: manager-role
 rules:
+- apiGroups:
+  - ''
+  resources:
+  - namespaces
+  verbs:
+  - list
+  - watch
 - apiGroups:
   - config.openshift.io
   resources:

operator/roles/forkliftcontroller/defaults/main.yml

@@ -113,3 +113,9 @@ virt_v2v_dont_request_kvm: "{{ lookup( 'env', 'VIRT_V2V_DONT_REQUEST_KVM') }}"
 virt_v2v_extra_args: "{{ lookup( 'env', 'VIRT_V2V_EXTRA_ARGS') }}"
 
 ova_provider_server_fqin: "{{ lookup( 'env', 'OVA_PROVIDER_SERVER_IMAGE') or lookup( 'env', 'RELATED_IMAGE_OVA_PROVIDER_SERVER') }}"
+
+metric_service_name: "{{ app_name }}-metrics"
+metric_servicemonitor_name: "{{ app_name }}-metrics"
+metric_interval: "30s"
+metric_port_name: "metrics"
+

operator/roles/forkliftcontroller/tasks/main.yml

@@ -79,6 +79,29 @@
       definition: "{{ lookup('template', 'controller/controller-scc.yml.j2') }}"
     when: not k8s_cluster|bool
 
+  - when: not k8s_cluster|bool
+    block:
+    - name: "Setup metrics service"
+      k8s:
+        state: present
+        definition: "{{ lookup('template', 'monitor/service-metrics.yml.j2') }}"
+
+    - name: "Setup metrics service monitor"
+      k8s:
+        state: present
+        definition: "{{ lookup('template', 'monitor/servicemonitor-metrics.yml.j2') }}"
+
+    - name: "Add monitoring label to namespace"
+      k8s:
+        state: present
+        definition:
+          apiVersion: v1
+          kind: Namespace
+          metadata:
+            name: "{{ app_namespace }}"
+            labels:
+              openshift.io/cluster-monitoring: "true"
+
   - when: feature_volume_populator|bool
     block:
     - name: "Setup populator controller deployment"

operator/roles/forkliftcontroller/templates/monitor/service-metrics.yml.j2

@@ -0,0 +1,22 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: {{ app_name }}
+    service: {{ metric_service_name }}
+    prometheus.forklift.konveyor.io: "true"
+    control-plane: controller-manager
+    controller-tools.k8s.io: "1.0"
+  name: {{ metric_service_name }}
+  namespace: {{ app_namespace }}
+spec:
+  ports:
+  - name: {{ metric_port_name }}
+    port: 2112
+    targetPort: 2112
+    protocol: TCP
+  selector:
+    app: {{ app_name }}
+    prometheus.forklift.konveyor.io: "true"
+    controller-tools.k8s.io: "1.0"

operator/roles/forkliftcontroller/templates/monitor/servicemonitor-metrics.yml.j2

@@ -0,0 +1,21 @@
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ metric_servicemonitor_name }}
+  namespace: {{ app_namespace }}
+  labels:
+    app: {{ app_name }}
+    prometheus.forklift.konveyor.io: "true"
+    control-plane: controller-manager
+spec:
+  endpoints:
+    - interval: {{ metric_interval }}
+      port: {{ metric_port_name }}
+  namespaceSelector:
+    matchNames:
+      - {{ app_namespace }}
+  selector:
+    matchLabels:
+      app: {{ app_name }}
+      prometheus.forklift.konveyor.io: "true"