Keycloak: Add opentelemetry tracing

.run/frontend-webapp-springboot.run.xml

@@ -0,0 +1,18 @@
+<component name="ProjectRunConfigurationManager">
+    <configuration default="false" name="frontend-webapp-springboot" type="SpringBootApplicationConfigurationType" factoryName="Spring Boot">
+        <option name="ACTIVE_PROFILES" />
+        <envs>
+            <env name="OTEL_EXPORTER_OTLP_ENDPOINT" value="https://ops.acme.test:4317" />
+            <env name="OTEL_METRICS_EXPORTER" value="none" />
+            <env name="OTEL_PROPAGATORS" value="b3multi" />
+            <env name="OTEL_SERVICE_NAME" value="acme-frontend-springboot" />
+        </envs>
+        <module name="frontend-webapp-springboot" />
+        <option name="SPRING_BOOT_MAIN_CLASS" value="com.github.thomasdarimont.keycloak.webapp.WebAppSpringBoot" />
+        <option name="VM_PARAMETERS" value="-javaagent:$ProjectFileDir$/bin/opentelemetry-javaagent-1.20.0.jar" />
+        <option name="WORKING_DIRECTORY" value="file://$ProjectFileDir$" />
+        <method v="2">
+            <option name="Make" enabled="true" />
+        </method>
+    </configuration>
+</component>

config/stage/dev/otel/otel-collector-config-tls.yaml

@@ -0,0 +1,35 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: acme-otel-collector:4317
+        tls:
+          cert_file: /cert.pem
+          key_file: /key.pem
+          min_version: "1.2"
+          max_version: "1.3"
+
+exporters:
+  jaeger:
+    endpoint: ops.acme.test:14250
+    tls:
+      min_version: "1.2"
+      max_version: "1.3"
+      ca_file: /rootca.pem
+      cert_file: /cert.pem
+      key_file: /key.pem
+  logging:
+    loglevel: debug
+processors:
+  batch:
+
+extensions:
+  health_check:
+
+service:
+  extensions: [health_check]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [jaeger]

config/stage/dev/otel/otel-collector-config.yaml

@@ -0,0 +1,25 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: acme-otel-collector:4317
+
+exporters:
+  jaeger:
+    endpoint: ops.acme.test:14250
+    tls:
+      insecure: true
+
+processors:
+  batch:
+
+extensions:
+  health_check:
+
+service:
+  extensions: [health_check]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [jaeger]

deployments/local/dev/docker-compose-keycloakx.yml

@@ -52,6 +52,7 @@ services:
       - "--cache=local"
       - "--proxy=passthrough"
 #      - "--log-console-output=json"
+      - "--log-console-format='%d{HH:mm:ss} %-5p traceId=%X{traceId}, parentId=%X{parentId}, spanId=%X{spanId}, sampled=%X{sampled} [%c{2.}] (%t) %s%e%n'"
       - "--https-protocols=TLSv1.3,TLSv1.2"
       - "--spi-theme-cache-themes=false"
       - "--spi-theme-cache-templates=false"
@@ -75,6 +76,7 @@ services:
       # 172.17.0.1 is host.docker.internal
       - "id.acme.test:172.17.0.1"
       - "apps.acme.test:172.17.0.1"
+      - "ops.acme.test:172.17.0.1"
     ports:
       - "8080:8080"
       - "8443:8443"

deployments/local/dev/docker-compose-tracing-tls.yml

@@ -0,0 +1,29 @@
+services:
+  acme-otel-collector:
+    volumes:
+      - ../../../config/stage/dev/otel/otel-collector-config-tls.yaml:/etc/otel-collector-config.yaml:z
+      - ${CA_ROOT_CERT:-}:/rootca.pem:z
+  acme-jaeger:
+    volumes:
+      - ../../../config/stage/dev/tls/acme.test+1.pem:/cert.pem:z
+      - ../../../config/stage/dev/tls/acme.test+1-key.pem:/key.pem:z
+      - ${CA_ROOT_CERT:-}:/rootca.pem:z
+    command:
+      - "--query.http.tls.enabled"
+      - "--query.http.tls.key=/key.pem"
+      - "--query.http.tls.cert=/cert.pem"
+      - "--query.http.tls.min-version=1.2"
+      - "--query.http.tls.max-version=1.3"
+      - "--collector.grpc.tls.enabled"
+      - "--collector.grpc.tls.key=/key.pem"
+      - "--collector.grpc.tls.cert=/cert.pem"
+      - "--collector.grpc.tls.min-version=1.2"
+      - "--collector.grpc.tls.max-version=1.3"
+      # Jaeger sends traces to itself. If we only allow TLS inbound, we need to do this via the hostname
+      # and validate the certificate
+      - "--reporter.grpc.tls.enabled"
+      - "--reporter.grpc.tls.ca=/rootca.pem"
+      - "--reporter.grpc.host-port=ops.acme.test:14250"
+  acme-keycloak:
+    volumes:
+      - ../../../keycloak/config/quarkus-tracing-tls.properties:/opt/keycloak/conf/quarkus.properties:z

deployments/local/dev/docker-compose-tracing.yml

@@ -0,0 +1,30 @@
+# https://quarkus.io/version/2.7/guides/opentelemetry#run-the-application
+services:
+  acme-otel-collector:
+    build:
+      context: "../../../config/stage/dev/tls"
+      dockerfile: "../../../../deployments/local/dev/otel-collector/Dockerfile"
+    command: ["--config=/etc/otel-collector-config.yaml"]
+    ports:
+      - "13133:13133" # Health_check extension
+      - "4317:4317"   # OTLP gRPC receiver
+    volumes:
+      - ../../../config/stage/dev/otel/otel-collector-config.yaml:/etc/otel-collector-config.yaml:z
+    extra_hosts:
+      # 172.17.0.1 is host.docker.internal
+      - "id.acme.test:172.17.0.1"
+      - "apps.acme.test:172.17.0.1"
+      - "ops.acme.test:172.17.0.1"
+  acme-jaeger:
+    image: jaegertracing/all-in-one:1.39
+    ports:
+      - "16686:16686"
+      - "14250:14250"
+    extra_hosts:
+      # 172.17.0.1 is host.docker.internal
+      - "id.acme.test:172.17.0.1"
+      - "apps.acme.test:172.17.0.1"
+      - "ops.acme.test:172.17.0.1"
+  acme-keycloak:
+    volumes:
+      - ../../../keycloak/config/quarkus-tracing.properties:/opt/keycloak/conf/quarkus.properties:z

deployments/local/dev/keycloakx/Dockerfile

@@ -1,7 +1,9 @@
 #see https://www.keycloak.org/server/containers
 ARG KEYCLOAK_VERSION=20.0.1
 FROM quay.io/keycloak/keycloak:$KEYCLOAK_VERSION
-
+ARG QUARKUS_VERSION=2.13.3.Final
+ARG OTEL_VERSION=1.17.0
+ARG OTEL_ALPHA_VERSION=1.17.0-alpha
 USER root
 
 # Add java-11-openjdk-devel JDK for debugging
@@ -22,4 +24,92 @@ RUN keytool -import -cacerts -noprompt -file /etc/x509/tls.crt.pem -storepass ch
 #     --location \
 #     --output /opt/jboss/keycloak/providers/keycloak-metrics-spi-$AEROGEAR_VERSION.jar
 
+RUN curl https://repo1.maven.org/maven2/io/quarkus/quarkus-opentelemetry/$QUARKUS_VERSION/quarkus-opentelemetry-$QUARKUS_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/quarkus-opentelemetry-$QUARKUS_VERSION.jar
+
+RUN curl https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-sdk/$OTEL_VERSION/opentelemetry-sdk-$OTEL_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/opentelemetry-sdk-$OTEL_VERSION.jar
+
+RUN curl https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-api/$OTEL_VERSION/opentelemetry-api-$OTEL_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/opentelemetry-api-$OTEL_VERSION.jar
+
+RUN curl https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-context/$OTEL_VERSION/opentelemetry-context-$OTEL_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/opentelemetry-context-$OTEL_VERSION.jar
+
+RUN curl https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-sdk-common/$OTEL_VERSION/opentelemetry-sdk-common-$OTEL_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/opentelemetry-sdk-common-$OTEL_VERSION.jar
+
+RUN curl https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-sdk-trace/$OTEL_VERSION/opentelemetry-sdk-trace-$OTEL_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/opentelemetry-sdk-trace-$OTEL_VERSION.jar
+
+RUN curl https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-sdk-metrics/$OTEL_VERSION/opentelemetry-sdk-metrics-$OTEL_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/opentelemetry-sdk-metrics-$OTEL_VERSION.jar
+
+RUN curl https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-sdk-logs/$OTEL_ALPHA_VERSION/opentelemetry-sdk-logs-$OTEL_ALPHA_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/opentelemetry-sdk-logs-$OTEL_ALPHA_VERSION.jar
+
+RUN curl https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-extension-annotations/$OTEL_VERSION/opentelemetry-extension-annotations-$OTEL_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/opentelemetry-extension-annotations-$OTEL_VERSION.jar
+
+RUN curl https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-sdk-extension-autoconfigure-spi/$OTEL_VERSION/opentelemetry-sdk-extension-autoconfigure-spi-$OTEL_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/opentelemetry-sdk-extension-autoconfigure-spi-$OTEL_VERSION.jar
+
+RUN curl https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-semconv/$OTEL_ALPHA_VERSION/opentelemetry-semconv-$OTEL_ALPHA_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/opentelemetry-semconv-$OTEL_ALPHA_VERSION.jar
+
+RUN curl https://repo1.maven.org/maven2/io/opentelemetry/instrumentation/opentelemetry-instrumentation-api/$OTEL_ALPHA_VERSION/opentelemetry-instrumentation-api-$OTEL_ALPHA_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/opentelemetry-instrumentation-api-$OTEL_ALPHA_VERSION.jar
+
+RUN curl https://repo1.maven.org/maven2/io/opentelemetry/instrumentation/opentelemetry-instrumentation-annotations/$OTEL_ALPHA_VERSION/opentelemetry-instrumentation-annotations-$OTEL_ALPHA_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/opentelemetry-instrumentation-annotations-$OTEL_ALPHA_VERSION.jar
+
+RUN curl https://repo1.maven.org/maven2/io/opentelemetry/instrumentation/opentelemetry-instrumentation-annotations-support/$OTEL_ALPHA_VERSION/opentelemetry-instrumentation-annotations-support-$OTEL_ALPHA_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/opentelemetry-instrumentation-annotations-support-$OTEL_ALPHA_VERSION.jar
+
+RUN curl https://repo1.maven.org/maven2/io/opentelemetry/instrumentation/opentelemetry-instrumentation-api-semconv/$OTEL_ALPHA_VERSION/opentelemetry-instrumentation-api-semconv-$OTEL_ALPHA_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/opentelemetry-instrumentation-api-semconv-$OTEL_ALPHA_VERSION.jar
+
+RUN curl https://repo1.maven.org/maven2/io/quarkus/quarkus-opentelemetry-deployment/$QUARKUS_VERSION/quarkus-opentelemetry-deployment-$QUARKUS_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/quarkus-opentelemetry-deployment-$QUARKUS_VERSION.jar
+
+RUN curl https://repo1.maven.org/maven2/io/quarkus/quarkus-opentelemetry-exporter-otlp/$QUARKUS_VERSION/quarkus-opentelemetry-exporter-otlp-$QUARKUS_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/quarkus-opentelemetry-exporter-otlp-$QUARKUS_VERSION.jar
+
+RUN curl https://repo1.maven.org/maven2/io/quarkus/quarkus-opentelemetry-exporter-otlp-deployment/$QUARKUS_VERSION/quarkus-opentelemetry-exporter-otlp-deployment-$QUARKUS_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/quarkus-opentelemetry-exporter-otlp-deployment-$QUARKUS_VERSION.jar
+
+RUN curl https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-exporter-otlp/$OTEL_VERSION/opentelemetry-exporter-otlp-$OTEL_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/opentelemetry-exporter-otlp-$OTEL_VERSION.jar
+
+RUN curl https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-exporter-otlp-common/$OTEL_VERSION/opentelemetry-exporter-otlp-common-$OTEL_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/opentelemetry-exporter-otlp-common-$OTEL_VERSION.jar
+
+RUN curl https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-exporter-common/$OTEL_VERSION/opentelemetry-exporter-common-$OTEL_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/opentelemetry-exporter-common-$OTEL_VERSION.jar
+
+RUN curl https://repo1.maven.org/maven2/io/opentelemetry/opentelemetry-extension-trace-propagators/$OTEL_VERSION/opentelemetry-extension-trace-propagators-$OTEL_VERSION.jar \
+    -L \
+    -o /opt/keycloak/providers/opentelemetry-extension-trace-propagators-$OTEL_VERSION.jar
+
 USER keycloak

deployments/local/dev/otel-collector/Dockerfile

@@ -0,0 +1,9 @@
+ARG OTEL_VERSION=0.64.1
+FROM otel/opentelemetry-collector:$OTEL_VERSION
+
+USER 0
+
+COPY --chown=10001:0 "./acme.test+1.pem" /cert.pem
+COPY --chown=10001:0 "./acme.test+1-key.pem" /key.pem
+
+USER 10001

keycloak/config/quarkus-tracing-tls.properties

@@ -0,0 +1,27 @@
+# Customize log level for the extensions package
+quarkus.log.category."com.github.thomasdarimont.keycloak".level=DEBUG
+
+# see https://quarkus.io/guides/smallrye-metrics
+# quarkus.smallrye-metrics.path=/actuator/metrics
+
+# see https://quarkus.io/guides/smallrye-health
+# quarkus.smallrye-health.root-path=/actuator/health
+
+# Use quarkus access logging
+# See https://quarkus.io/guides/http-reference#quarkus-vertx-http-config-group-access-log-config_quarkus.http.access-log.enabled
+#quarkus.http.access-log.enabled=true
+#quarkus.http.access-log.pattern=%h %l %u %t "%r" %s %b %m "%{i,Referer}" "%{i,User-Agent}" "%{i,X-Request-Id}" "%{i,X-Organization-Id}" %D
+
+quarkus.opentelemetry.enabled=true
+quarkus.opentelemetry.tracer.enabled=true
+quarkus.opentelemetry.tracer.exporter.otlp.enabled=true
+quarkus.opentelemetry.tracer.exporter.otlp.endpoint=https://ops.acme.test:4317
+quarkus.opentelemetry.tracer.resource-attributes=service.name=acme-keycloak
+
+## Sampling Settings
+quarkus.opentelemetry.tracer.sampler=ratio
+# Percentage of traces that are sampled. 1.0 = 100%. Great for testing, not great for production.
+quarkus.opentelemetry.tracer.sampler.ratio=1.0
+# If we're not the root of this trace, defer to the sampling decision made by the parent
+quarkus.opentelemetry.tracer.sampler.parent-based=true
+quarkus.opentelemetry.propagators=b3multi

keycloak/config/quarkus-tracing.properties

@@ -0,0 +1,27 @@
+# Customize log level for the extensions package
+quarkus.log.category."com.github.thomasdarimont.keycloak".level=DEBUG
+
+# see https://quarkus.io/guides/smallrye-metrics
+# quarkus.smallrye-metrics.path=/actuator/metrics
+
+# see https://quarkus.io/guides/smallrye-health
+# quarkus.smallrye-health.root-path=/actuator/health
+
+# Use quarkus access logging
+# See https://quarkus.io/guides/http-reference#quarkus-vertx-http-config-group-access-log-config_quarkus.http.access-log.enabled
+#quarkus.http.access-log.enabled=true
+#quarkus.http.access-log.pattern=%h %l %u %t "%r" %s %b %m "%{i,Referer}" "%{i,User-Agent}" "%{i,X-Request-Id}" "%{i,X-Organization-Id}" %D
+
+quarkus.opentelemetry.enabled=true
+quarkus.opentelemetry.tracer.enabled=true
+quarkus.opentelemetry.tracer.exporter.otlp.enabled=true
+quarkus.opentelemetry.tracer.exporter.otlp.endpoint=http://ops.acme.test:4317
+quarkus.opentelemetry.tracer.resource-attributes=service.name=acme-keycloak
+
+## Sampling Settings
+quarkus.opentelemetry.tracer.sampler=ratio
+# Percentage of traces that are sampled. 1.0 = 100%. Great for testing, not great for production.
+quarkus.opentelemetry.tracer.sampler.ratio=1.0
+# If we're not the root of this trace, defer to the sampling decision made by the parent
+quarkus.opentelemetry.tracer.sampler.parent-based=true
+quarkus.opentelemetry.propagators=b3multi

keycloak/config/quarkus.properties

@@ -10,4 +10,9 @@ quarkus.log.category."com.github.thomasdarimont.keycloak".level=DEBUG
 # Use quarkus access logging
 # See https://quarkus.io/guides/http-reference#quarkus-vertx-http-config-group-access-log-config_quarkus.http.access-log.enabled
 #quarkus.http.access-log.enabled=true
-#quarkus.http.access-log.pattern=%h %l %u %t "%r" %s %b %m "%{i,Referer}" "%{i,User-Agent}" "%{i,X-Request-Id}" "%{i,X-Organization-Id}" %D
+#quarkus.http.access-log.pattern=%h %l %u %t "%r" %s %b %m "%{i,Referer}" "%{i,User-Agent}" "%{i,X-Request-Id}" "%{i,X-Organization-Id}" %D
+
+# Needs to be true if the libraries are present, but disabling the tracer and exporter makes it do effectively nothing
+quarkus.opentelemetry.enabled=true
+quarkus.opentelemetry.tracer.enabled=false
+quarkus.opentelemetry.tracer.exporter.otlp.enabled=false

readme.md

@@ -43,7 +43,8 @@ d) **Operator** configuring realm and server for different stages
 - Examples for running a cluster behind a reverse proxy with examples for [HAProxy](deployments/local/cluster/haproxy), [Apache](deployments/local/cluster/apache), [nginx](deployments/local/cluster/nginx), [caddy](deployments/local/cluster/caddy)
 - Examples for running a Keycloak cluster with an external infinispan cluster with [remote cache store](deployments/local/cluster/haproxy-external-ispn/docker-compose-haproxy-ispn-remote.yml) and [hotrod cache store](deployments/local/cluster/haproxy-external-ispn/docker-compose-haproxy-ispn-hotrod.yml).
 - Example for Keycloak with [Graylog](https://www.graylog.org/) for log analysis, dashboards and alerting.
-- Example for metrics collection and dashboards with [Prometheus](https://prometheus.io) and [Grafana](https://grafana.com/oss). 
+- Example for metrics collection and dashboards with [Prometheus](https://prometheus.io) and [Grafana](https://grafana.com/oss).
+- Example for tracing with [OpenTelemetry](https://opentelemetry.io/) and [Jaeger](https://www.jaegertracing.io/)
 
 ## Usage envcheck
 
@@ -205,6 +206,34 @@ Manual steps when logged in as an Admin (Example User: devops_fallback, Password
     * Add e.g. elastic-search as datasource (http://acme-graylog-lo:9090/) (see [Graylog](#enable-graylog) services)
 * Import Boards of your choice from [Grafana](https://grafana.com/grafana/dashboards) (for testing an [exported board](../../../config/stage/dev/grafana/microprofile-wildfly-16-metrics_rev1.json) can be used) 
 
+### Enable Tracing
+With [OpenTelemetry](https://opentelemetry.io/) and [Jaeger](https://www.jaegertracing.io/), it is possible to trace requests traveling through Keycloak and the systems integrating it.
+This uses the Quarkus OpenTelemetry extension in order to create traces, which are then sent to the [otel-collector](https://opentelemetry.io/docs/collector/).
+The collector then passes the information on to Jaeger, where they can be viewed in the web interface
+
+#### Run with Tracing
+```
+java start.java --tracing
+```
+Open [Jaeger](http://ops.acme.test:16686) or [Jaeger with TLS](https://ops.acme.test:16686), depending on configuration.
+When TLS is enabled, it is enabled for all three of the following:
+* Jaeger UI
+* Keycloak -> Collector communication
+* Collector -> Jaeger communication
+
+#### Instrumentation
+In order to gain additional insights, other applications that integrate with Keycloak can also send traces to the collector.
+The [OpenTelemetry Documentation](https://opentelemetry.io/docs/instrumentation/) contains tools to instrument applications in various languages.
+Quarkus applications like Keycloak can also use the [Quarkus OpenTelemetry extension](https://quarkus.io/guides/opentelemetry) instead of the agent.
+An example for running an instrumented Spring Boot app could look like this:
+```
+OTEL_METRICS_EXPORTER=none OTEL_SERVICE_NAME="frontend-webapp-springboot" OTEL_PROPAGATORS="b3multi" \
+ OTEL_EXPORTER_OTLP_ENDPOINT="http://id.acme.test:4317" java -javaagent:bin/opentelemetry-javaagent-1.20.0.jar \
+ -jar apps/frontend-webapp-springboot/target/frontend-webapp-springboot-0.0.1-SNAPSHOT.jar
+```
+The included IDEA run-config for the frontend-webapp-springboot module contains the necessary configuration to run that module with tracing enabled.
+If you then navigate to the [frontend webapp](https://apps.acme.test:4633/webapp/), you can navigate through the application, and then later check the Jaeger UI for traces.
+
 ### Clustering
 
 Clustering examples can be found in the [deployments/local/cluster](deployments/local/cluster) folder.