Migrate to Tempo on OpenShift

Closes #1095 Signed-off-by: Alexander Schwartz <[email protected]>
keycloak · Jan 14, 2025 · 900493c · 900493c
1 parent 1c01d35
commit 900493c
Show file tree

Hide file tree

Showing 10 changed files with 76 additions and 81 deletions.
diff --git a/provision/aws/rosa_create_cluster.sh b/provision/aws/rosa_create_cluster.sh
@@ -75,6 +75,7 @@ cd ${SCRIPT_DIR}
 ./rosa_install_cryotstat_operator.sh
 
 ./rosa_install_openshift_logging.sh
+./rosa_install_tempo_operator.sh
 
 echo "Enabling user alert routing."
 oc apply -f ${SCRIPT_DIR}/../openshift/cluster-monitoring-config.yaml

diff --git a/provision/aws/rosa_install_tempo_operator.sh b/provision/aws/rosa_install_tempo_operator.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+if [[ "$RUNNER_DEBUG" == "1" ]]; then
+  set -x
+fi
+
+INSTALL_NAMESPACE=${INSTALL_NAMESPACE:-openshift-operators}
+
+echo "Installing tempo operator."
+
+oc apply -f - << EOF
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  name: tempo-product
+  namespace: ${INSTALL_NAMESPACE}
+spec:
+  channel: stable
+  installPlanApproval: Automatic
+  name: tempo-product
+  source: redhat-operators
+  sourceNamespace: openshift-marketplace
+EOF
diff --git a/provision/infinispan/ispn-helm/templates/infinispan.yaml b/provision/infinispan/ispn-helm/templates/infinispan.yaml
@@ -26,7 +26,7 @@ data:
         {{- if .Values.tracing.enabled }}
         tracing:
           enabled: true
-          collector-endpoint: "http://jaeger-collector.monitoring.svc:4318"
+          collector-endpoint: "http://tempo-tempo.monitoring.svc:4318"
           exporter-protocol: "OTLP"
           service-name: {{ .Values.tracing.serviceName | quote }}
           security: false

diff --git a/provision/minikube/keycloak/templates/keycloak.yaml b/provision/minikube/keycloak/templates/keycloak.yaml
@@ -94,7 +94,7 @@ spec:
     - name: tracing-enabled
       value: "true"
     - name: tracing-endpoint
-      value: "http://jaeger-collector.monitoring.svc:4317"
+      value: {{ .Values.tracingEndpoint }}
     - name: tracing-sampler-ratio
       value: {{ .Values.otelSamplingPercentage | quote }}
     - name: tracing-sampler-type

diff --git a/provision/minikube/keycloak/values.yaml b/provision/minikube/keycloak/values.yaml
@@ -53,3 +53,4 @@ infinispan:
     username: developer
 networkPolicy:
   enabled: true
+tracingEndpoint: http://jaeger-collector.monitoring.svc:4317
diff --git a/provision/openshift/Taskfile.yaml b/provision/openshift/Taskfile.yaml
@@ -110,27 +110,6 @@ tasks:
       - .task/var-KC_HOSTNAME_SUFFIX
       - .task/var-KC_DATABASE_URL
 
-  jaeger:
-    deps:
-      - common:split
-      - common:env
-      - openshift-env
-      - grafana-sa
-    cmds:
-      - helm repo add jaegertracing https://jaegertracing.github.io/helm-charts
-      - helm repo update
-      - >
-        helm upgrade --install jaeger jaegertracing/jaeger --version 3.1.2 -n monitoring -f ../minikube/jaeger/values.yaml
-        --set allInOne.podSecurityContext.runAsUser=$(cat .task/monitoring-uids)
-        --set allInOne.podSecurityContext.runAsGroup=$(cat .task/monitoring-uids)
-        --set allInOne.podSecurityContext.fsGroup=$(cat .task/monitoring-uids)
-        --set allInOne.extraEnv[0].value={{.KB_RETENTION}}
-    sources:
-      - ../minikube/jaeger/**/*.*
-      - .task/subtask-{{.TASK}}.yaml
-      - .task/var-KB_RETENTION
-      - .task/var-KC_HOSTNAME_SUFFIX
-
   grafana-sa:
     # this setup was inspired by https://zhimin-wen.medium.com/custom-grafana-dashboard-for-user-workload-in-openshift-6dc2d4cad274
     deps:
@@ -181,7 +160,6 @@ tasks:
       - common:env
       - openshift-env
       - grafana
-      - jaeger
     cmds:
       - helm upgrade -n monitoring --install monitoring monitoring
     sources:
@@ -278,6 +256,7 @@ tasks:
         --set multiSite={{ .KC_MULTI_SITE }}
         --set nodePortsEnabled=false
         --set networkPolicy.enabled={{ .KC_NETWORK_POLICY }}
+        --set tracingEndpoint=http://tempo-tempo.monitoring.svc:4317
         ../minikube/keycloak
       - >
         bash -c '

diff --git a/provision/openshift/grafana.yaml b/provision/openshift/grafana.yaml
@@ -45,23 +45,13 @@ datasources:
           httpMethod: "POST"
         secureJsonData:
           httpHeaderValue1: Bearer $__file{/etc/secrets/grafanaserviceaccount/token}
-      - name: Jaeger
-        uid: PC9A941E8F2E49454
-        type: jaeger
-        url: "http://jaeger-query.monitoring.svc:16686"
+      - name: Tempo
+        uid: PD9A941E8F2E49454
+        type: tempo
+        url: "http://tempo-tempo.monitoring.svc:3200"
         access: proxy
         isDefault: false
         editable: false
-        jsonData:
-          tracesToLogs:
-            datasourceUid: P8E80F9AEF21F6940
-            filterByTraceID: true,
-            mapTagNamesEnabled: true
-            mappedTags:
-              - key: "hostname"
-                value: "pod"
-            spanStartTimeShift: "-1h"
-            spanEndTimeShift: "1h"
 extraSecretMounts:
 - name: grafana-sa-secret
   secretName: grafana-sa-secret

diff --git a/provision/openshift/monitoring/templates/jaeger-pvc.yaml b/provision/openshift/monitoring/templates/jaeger-pvc.yaml
diff --git a/provision/openshift/monitoring/templates/tempo.yaml b/provision/openshift/monitoring/templates/tempo.yaml
@@ -0,0 +1,42 @@
+apiVersion: tempo.grafana.com/v1alpha1
+kind: TempoMonolithic
+metadata:
+  name: tempo
+  namespace: monitoring
+spec:
+  ingestion:
+    otlp:
+      grpc:
+        enabled: true
+      http:
+        enabled: true
+  jaegerui:
+    enabled: true
+    ingress:
+      enabled: true
+    resources:
+      limits:
+        cpu: '2'
+        memory: 2Gi
+    route:
+      enabled: true
+  multitenancy:
+    enabled: false
+    mode: static
+  observability:
+    grafana:
+      dataSource:
+        enabled: false
+    metrics:
+      prometheusRules:
+        enabled: false
+      serviceMonitors:
+        enabled: true
+  resources:
+    limits:
+      cpu: '2'
+      memory: 2Gi
+  storage:
+    traces:
+      backend: pv
+      size: 1Gi
diff --git a/provision/rosa-cross-dc/Taskfile.yaml b/provision/rosa-cross-dc/Taskfile.yaml
@@ -513,7 +513,6 @@ tasks:
     internal: true
     cmds:
       - helm repo add grafana https://grafana.github.io/helm-charts
-      - helm repo add jaegertracing https://jaegertracing.github.io/helm-charts
       - helm repo update
 
   create-grafana-service-account:
@@ -571,27 +570,6 @@ tasks:
       - "{{.KC_DIR}}/grafana.yaml"
       - "{{.ISPN_DIR}}/.task/kubecfg/{{.ROSA_CLUSTER_NAME}}"
 
-  install-jaeger:
-    label: "install-jaeger-{{.ROSA_CLUSTER_NAME}}"
-    internal: true
-    requires:
-      vars:
-        - ROSA_CLUSTER_NAME
-    vars:
-      RETENTION: '{{.RETENTION | default "168h"}}'
-      JAEGER_VERSION: '{{.JAEGER_VERSION | default "3.1.2"}}'
-    cmds:
-      - >
-        KUBECONFIG="{{.ISPN_DIR}}/.task/kubecfg/{{.ROSA_CLUSTER_NAME}}"
-        helm upgrade --install jaeger jaegertracing/jaeger --version {{.JAEGER_VERSION}} -n monitoring -f "{{.ROOT_DIR}}/../minikube/jaeger/values.yaml"
-        --set allInOne.podSecurityContext.runAsUser=$(cat .task/monitoring-uids-{{.ROSA_CLUSTER_NAME}})
-        --set allInOne.podSecurityContext.runAsGroup=$(cat .task/monitoring-uids-{{.ROSA_CLUSTER_NAME}})
-        --set allInOne.podSecurityContext.fsGroup=$(cat .task/monitoring-uids-{{.ROSA_CLUSTER_NAME}})
-        --set allInOne.extraEnv[0].value={{.RETENTION}}
-    sources:
-      - "{{.ROOT_DIR}}/../minikube/jaeger/**/*"
-      - "{{.ISPN_DIR}}/.task/kubecfg/{{.ROSA_CLUSTER_NAME}}"
-
   install-grafana-charts:
     label: "install-grafana-charts-{{.ROSA_CLUSTER_NAME}}"
     internal: true
@@ -644,12 +622,6 @@ tasks:
       - task: install-grafana
         vars:
           ROSA_CLUSTER_NAME: "{{.ROSA_CLUSTER_NAME_2}}"
-      - task: install-jaeger
-        vars:
-          ROSA_CLUSTER_NAME: "{{.ROSA_CLUSTER_NAME_1}}"
-      - task: install-jaeger
-        vars:
-          ROSA_CLUSTER_NAME: "{{.ROSA_CLUSTER_NAME_2}}"
       - task: install-grafana-charts
         vars:
           ROSA_CLUSTER_NAME: "{{.ROSA_CLUSTER_NAME_1}}"