From 8c37793d939af2171696beb762fd52fcf71af75b Mon Sep 17 00:00:00 2001 From: Michal Hajas Date: Thu, 14 Nov 2024 16:18:38 +0100 Subject: [PATCH] Add Kubernetes CPU metric to JVM metric guide and dashboard Closes #1042 Signed-off-by: Michal Hajas --- .../pages/running/metrics/jvm_metrics.adoc | 11 + .../dashboards/keycloak-status.json | 1688 +++++++++-------- 2 files changed, 877 insertions(+), 822 deletions(-) diff --git a/doc/kubernetes/modules/ROOT/pages/running/metrics/jvm_metrics.adoc b/doc/kubernetes/modules/ROOT/pages/running/metrics/jvm_metrics.adoc index e34eed2a0..9186f95da 100644 --- a/doc/kubernetes/modules/ROOT/pages/running/metrics/jvm_metrics.adoc +++ b/doc/kubernetes/modules/ROOT/pages/running/metrics/jvm_metrics.adoc @@ -49,3 +49,14 @@ m| jvm_gc_overhead |=== Additional information on the {project_name} `metrics` endpoint can be found https://www.keycloak.org/server/configuration-metrics[here]. + +== CPU Usage in Kubernetes + +|=== +|Metric |Description + +m| container_cpu_usage_seconds_total + +| Cumulative cpu time consumed by the container in core-seconds. + +|=== diff --git a/provision/minikube/monitoring/dashboards/keycloak-status.json b/provision/minikube/monitoring/dashboards/keycloak-status.json index aaa944484..510278199 100644 --- a/provision/minikube/monitoring/dashboards/keycloak-status.json +++ b/provision/minikube/monitoring/dashboards/keycloak-status.json @@ -24,7 +24,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 5, + "id": 6, "links": [], "liveNow": false, "panels": [ @@ -448,7 +448,7 @@ "type": "timeseries" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, @@ -456,839 +456,921 @@ "y": 22 }, "id": 10, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "panels": [], + "title": "JVM Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" }, - "overrides": [ + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "matcher": { - "id": "byName", - "options": "runtime" - }, - "properties": [ - { - "id": "custom.width", - "value": 154 - } - ] + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 } ] - }, - "gridPos": { - "h": 16, - "w": 6, - "x": 0, - "y": 23 - }, - "id": 12, - "options": { - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "runtime" }, - "frameIndex": 0, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "9.4.7", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(jvm_info_total{namespace=\"$namespace\"}) by (pod, vendor, runtime, version)", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "JVM Info", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Value": true - }, - "indexByName": {}, - "renameByName": {} + "properties": [ + { + "id": "custom.width", + "value": 154 } - } + ] + } + ] + }, + "gridPos": { + "h": 16, + "w": 6, + "x": 0, + "y": 23 + }, + "id": 12, + "options": { + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" ], - "type": "table" + "show": false }, + "frameIndex": 0, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.4.7", + "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" + "editorMode": "code", + "exemplar": false, + "expr": "max(jvm_info_total{namespace=\"$namespace\"}) by (pod, vendor, runtime, version)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "JVM Info", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true }, - "overrides": [] - }, - "gridPos": { - "h": 16, - "w": 6, - "x": 6, - "y": 23 + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "This is available only in Kubernetes and only if your pods have cpu limits set", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 20, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 14, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "textMode": "auto" + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "pluginVersion": "9.4.7", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "expr": "sum_over_time((sum by (pod) (jvm_memory_used_bytes))[$__range:$__interval]) / count_over_time((sum by (pod) (jvm_memory_used_bytes))[$__range:$__interval]) ", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "B" - } - ], - "title": "Average memory usage", - "type": "stat" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 6, + "x": 6, + "y": 23 + }, + "id": 346, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 16, - "w": 6, - "x": 12, - "y": 23 + "editorMode": "code", + "expr": "sum(rate(container_cpu_usage_seconds_total{container=\"keycloak\", namespace=\"$namespace\"}[5m])) by (pod) /\nsum(container_spec_cpu_quota{container=\"keycloak\", namespace=\"$namespace\"}/container_spec_cpu_period{container=\"keycloak\", namespace=\"$namespace\"}) by (pod)", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "KUBERNETES - CPU Usage percentage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "id": 18, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "min", - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] }, - "pluginVersion": "9.4.7", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by (pod) (jvm_memory_used_bytes)", - "hide": false, - "legendFormat": "{{pod}} - used", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by (pod) (jvm_memory_committed_bytes)", - "hide": false, - "legendFormat": "{{pod}} - committed", - "range": true, - "refId": "B" - } + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 6, + "x": 12, + "y": 23 + }, + "id": 20, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "title": "JVM memory used vs committed", - "type": "timeseries" + "fields": "", + "values": false }, + "textMode": "auto" + }, + "pluginVersion": "9.4.7", + "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 16, - "w": 6, - "x": 0, - "y": 39 + "editorMode": "code", + "expr": "sum_over_time((sum by (pod) (jvm_memory_used_bytes))[$__range:$__interval]) / count_over_time((sum by (pod) (jvm_memory_used_bytes))[$__range:$__interval]) ", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Average memory usage", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 22, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "textMode": "auto" - }, - "pluginVersion": "9.4.7", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum(\n rate(\n jvm_gc_pause_seconds_sum{\n container=\"keycloak\", \n namespace=\"$namespace\"}\n [$__range] \n )\n) without (action,instance,cause) \n/\nsum(\n rate(\n jvm_gc_pause_seconds_count{\n container=\"keycloak\",\n namespace=\"$namespace\"}\n [$__range] \n )\n) without (action,instance,cause)", - "hide": false, - "legendFormat": "{{pod}} - {{gc}}", - "range": true, - "refId": "B" + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 6, + "x": 0, + "y": 39 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "min", + "max", + "mean" ], - "title": "Average GC time", - "type": "stat" + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.4.7", + "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 41, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] + "editorMode": "code", + "expr": "sum by (pod) (jvm_memory_used_bytes)", + "hide": false, + "legendFormat": "{{pod}} - used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "gridPos": { - "h": 16, - "w": 6, - "x": 6, - "y": 39 + "editorMode": "code", + "expr": "sum by (pod) (jvm_memory_committed_bytes)", + "hide": false, + "legendFormat": "{{pod}} - committed", + "range": true, + "refId": "B" + } + ], + "title": "JVM memory used vs committed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "id": 142, - "options": { - "legend": { - "calcs": [ - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by (pod,cause)(irate(jvm_gc_pause_seconds_sum{container=\"keycloak\", namespace=\"$namespace\"}[5m]))", - "legendFormat": "{{pod}} - {{cause}}", - "range": true, - "refId": "A" - } + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 6, + "x": 6, + "y": 39 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "title": "Changes in average GC times in 5 minutes interval", - "type": "timeseries" + "fields": "", + "values": false }, + "textMode": "auto" + }, + "pluginVersion": "9.4.7", + "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 14, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "gridPos": { - "h": 16, - "w": 6, - "x": 12, - "y": 39 + "editorMode": "code", + "expr": "sum(\n rate(\n jvm_gc_pause_seconds_sum{\n container=\"keycloak\", \n namespace=\"$namespace\"}\n [$__range] \n )\n) without (action,instance,cause) \n/\nsum(\n rate(\n jvm_gc_pause_seconds_count{\n container=\"keycloak\",\n namespace=\"$namespace\"}\n [$__range] \n )\n) without (action,instance,cause)", + "hide": false, + "legendFormat": "{{pod}} - {{gc}}", + "range": true, + "refId": "B" + } + ], + "title": "Average GC time", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 140, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 41, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "expr": "sum by (pod,cause)(irate(jvm_gc_pause_seconds_count{container=\"keycloak\", namespace=\"$namespace\"}[5m]))", - "legendFormat": "{{pod}} - {{cause}}", - "range": true, - "refId": "A" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 6, + "x": 12, + "y": 39 + }, + "id": 142, + "options": { + "legend": { + "calcs": [ + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (pod,cause)(irate(jvm_gc_pause_seconds_sum{container=\"keycloak\", namespace=\"$namespace\"}[5m]))", + "legendFormat": "{{pod}} - {{cause}}", + "range": true, + "refId": "A" + } + ], + "title": "Changes in average GC times in 5 minutes interval", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 14, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 6, + "x": 0, + "y": 55 + }, + "id": 140, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" ], - "title": "Number of GC events in 5 minutes interval", - "type": "timeseries" + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "I am not sure about this metric. I would expect _max to be only increasing number while here it is going up and down so I am not sure what this really means", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 16, - "w": 6, - "x": 0, - "y": 55 + "editorMode": "code", + "expr": "sum by (pod,cause)(irate(jvm_gc_pause_seconds_count{container=\"keycloak\", namespace=\"$namespace\"}[5m]))", + "legendFormat": "{{pod}} - {{cause}}", + "range": true, + "refId": "A" + } + ], + "title": "Number of GC events in 5 minutes interval", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "I am not sure about this metric. I would expect _max to be only increasing number while here it is going up and down so I am not sure what this really means", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 24, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "max by (pod,cause) (jvm_gc_pause_seconds_max{container=\"keycloak\", namespace=\"$namespace\"})", - "legendFormat": "{{pod}} - {{cause}}", - "range": true, - "refId": "A" - } + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 6, + "x": 6, + "y": 55 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" ], - "title": "Maximum GC time and cause", - "type": "timeseries" + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "The percentage of CPU time spent on garbage collection, indicating the impact of GC on application performance in the JVM. It refers to the proportion of the total CPU processing time that is dedicated to executing garbage collection (GC) operations, as opposed to running application code or performing other tasks. This metric helps determine how much overhead GC introduces, affecting the overall performance of the Keycloak’s JVM.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 16, - "w": 6, - "x": 6, - "y": 55 + "editorMode": "code", + "expr": "max by (pod,cause) (jvm_gc_pause_seconds_max{container=\"keycloak\", namespace=\"$namespace\"})", + "legendFormat": "{{pod}} - {{cause}}", + "range": true, + "refId": "A" + } + ], + "title": "Maximum GC time and cause", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The percentage of CPU time spent on garbage collection, indicating the impact of GC on application performance in the JVM. It refers to the proportion of the total CPU processing time that is dedicated to executing garbage collection (GC) operations, as opposed to running application code or performing other tasks. This metric helps determine how much overhead GC introduces, affecting the overall performance of the Keycloak’s JVM.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 26, - "options": { - "legend": { - "calcs": [ - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "tooltip": { - "mode": "single", - "sort": "none" + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by (pod) (jvm_gc_overhead{container=\"keycloak\", namespace=\"$namespace\"})", - "legendFormat": "{{pod}}", - "range": true, - "refId": "A" - } + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 6, + "x": 12, + "y": 55 + }, + "id": 26, + "options": { + "legend": { + "calcs": [ + "max", + "mean" ], - "title": "JVM GC CPU overhead %", - "type": "timeseries" + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 14, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 16, - "w": 6, - "x": 12, - "y": 55 + "editorMode": "code", + "expr": "sum by (pod) (jvm_gc_overhead{container=\"keycloak\", namespace=\"$namespace\"})", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "JVM GC CPU overhead %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 224, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 14, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (pod)(jvm_threads_states_threads{namespace=\"$namespace\", container=\"keycloak\", state=\"waiting\"})", - "format": "time_series", - "hide": true, - "instant": false, - "legendFormat": "{{pod}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "expr": "jvm_threads_states_threads", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "B" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 6, + "x": 0, + "y": 71 + }, + "id": 224, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "lastNotNull" ], - "title": "JVM waiting threads", - "type": "timeseries" + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (pod)(jvm_threads_states_threads{namespace=\"$namespace\", container=\"keycloak\", state=\"waiting\"})", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" } ], - "title": "JVM Metrics", - "type": "row" + "title": "JVM waiting threads", + "type": "timeseries" }, { "collapsed": true, @@ -1296,7 +1378,7 @@ "h": 1, "w": 24, "x": 0, - "y": 23 + "y": 87 }, "id": 28, "panels": [ @@ -1315,8 +1397,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1332,7 +1413,7 @@ "h": 16, "w": 6, "x": 0, - "y": 24 + "y": 67 }, "id": 30, "options": { @@ -1357,7 +1438,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum by (pod) (sum_over_time(agroal_active_count[$__range:$__interval]) / sum_over_time((agroal_active_count + agroal_available_count)[$__range:$__interval]))", + "expr": "avg by (pod) (sum_over_time(agroal_active_count[$__range:$__interval]) / sum_over_time((agroal_active_count + agroal_available_count)[$__range:$__interval]))", "legendFormat": "__auto", "range": true, "refId": "A" @@ -1411,8 +1492,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1427,7 +1507,7 @@ "h": 16, "w": 6, "x": 6, - "y": 24 + "y": 67 }, "id": 32, "options": { @@ -1505,8 +1585,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1521,7 +1600,7 @@ "h": 16, "w": 6, "x": 12, - "y": 24 + "y": 67 }, "id": 34, "options": { @@ -1613,8 +1692,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1629,7 +1707,7 @@ "h": 16, "w": 6, "x": 0, - "y": 40 + "y": 83 }, "id": 183, "options": { @@ -1670,7 +1748,7 @@ "h": 1, "w": 24, "x": 0, - "y": 24 + "y": 88 }, "id": 36, "panels": [ @@ -1719,8 +1797,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1735,7 +1812,7 @@ "h": 15, "w": 6, "x": 0, - "y": 105 + "y": 68 }, "id": 38, "options": { @@ -1814,8 +1891,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1824,38 +1900,13 @@ ] } }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "{outcome=\"SUCCESS\", uri=\"/realms/{realm}/protocol/{protocol}/token\"}" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 15, "w": 12, "x": 6, - "y": 105 + "y": 68 }, "id": 42, "options": { @@ -1917,7 +1968,7 @@ "h": 15, "w": 6, "x": 0, - "y": 120 + "y": 83 }, "id": 40, "options": { @@ -2019,8 +2070,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2035,7 +2085,7 @@ "h": 15, "w": 12, "x": 6, - "y": 120 + "y": 83 }, "id": 50, "options": { @@ -2116,8 +2166,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -2128,7 +2177,7 @@ "h": 15, "w": 6, "x": 0, - "y": 135 + "y": 98 }, "id": 44, "options": { @@ -2206,8 +2255,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2222,7 +2270,7 @@ "h": 15, "w": 12, "x": 6, - "y": 135 + "y": 98 }, "id": 264, "options": { @@ -2304,8 +2352,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2321,7 +2368,7 @@ "h": 15, "w": 6, "x": 0, - "y": 150 + "y": 113 }, "id": 49, "options": { @@ -2371,8 +2418,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -2384,7 +2430,7 @@ "h": 15, "w": 6, "x": 6, - "y": 150 + "y": 113 }, "id": 46, "options": { @@ -2446,8 +2492,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -2459,7 +2504,7 @@ "h": 15, "w": 6, "x": 12, - "y": 150 + "y": 113 }, "id": 47, "options": { @@ -2551,8 +2596,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2568,7 +2612,7 @@ "h": 15, "w": 6, "x": 0, - "y": 165 + "y": 128 }, "id": 51, "options": { @@ -2613,7 +2657,7 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 89 }, "id": 58, "panels": [ @@ -2678,7 +2722,7 @@ "h": 15, "w": 6, "x": 0, - "y": 90 + "y": 176 }, "id": 89, "options": { @@ -2770,7 +2814,7 @@ "h": 15, "w": 6, "x": 6, - "y": 90 + "y": 176 }, "id": 90, "options": { @@ -2862,7 +2906,7 @@ "h": 15, "w": 6, "x": 12, - "y": 90 + "y": 176 }, "id": 99, "options": { @@ -2956,7 +3000,7 @@ "h": 15, "w": 6, "x": 0, - "y": 105 + "y": 191 }, "id": 66, "options": { @@ -3048,7 +3092,7 @@ "h": 15, "w": 6, "x": 6, - "y": 105 + "y": 191 }, "id": 97, "options": { @@ -3070,7 +3114,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum by (pod)((\nirate(vendor_statistics_hit_times_seconds_count{container=\"keycloak\", namespace=\"$namespace\", cache=\"$jdbc_cache_names\"}[5m]) + irate(vendor_statistics_miss_times_seconds_count{container=\"keycloak\", namespace=\"$namespace\", cache=\"$jdbc_cache_names\"}[5m]))\n/\n(\n irate(vendor_statistics_hit_times_seconds_count{container=\"keycloak\", namespace=\"$namespace\", cache=\"$jdbc_cache_names\"}[5m]) + irate(vendor_statistics_miss_times_seconds_count{container=\"keycloak\", namespace=\"$namespace\", cache=\"$jdbc_cache_names\"}[5m])\n + \n irate(vendor_statistics_remove_hit_times_seconds_count{container=\"keycloak\", namespace=\"$namespace\", cache=\"$jdbc_cache_names\"}[5m]) + irate(vendor_statistics_remove_miss_times_seconds_count{container=\"keycloak\", namespace=\"$namespace\", cache=\"$jdbc_cache_names\"}[5m]) \n + \n irate(vendor_statistics_store_times_seconds_count{container=\"keycloak\", namespace=\"$namespace\", cache=\"$jdbc_cache_names\"}[5m])\n))", + "expr": "avg by (pod)((\nirate(vendor_statistics_hit_times_seconds_count{container=\"keycloak\", namespace=\"$namespace\", cache=\"$jdbc_cache_names\"}[5m]) + irate(vendor_statistics_miss_times_seconds_count{container=\"keycloak\", namespace=\"$namespace\", cache=\"$jdbc_cache_names\"}[5m]))\n/\n(\n irate(vendor_statistics_hit_times_seconds_count{container=\"keycloak\", namespace=\"$namespace\", cache=\"$jdbc_cache_names\"}[5m]) + irate(vendor_statistics_miss_times_seconds_count{container=\"keycloak\", namespace=\"$namespace\", cache=\"$jdbc_cache_names\"}[5m])\n + \n irate(vendor_statistics_remove_hit_times_seconds_count{container=\"keycloak\", namespace=\"$namespace\", cache=\"$jdbc_cache_names\"}[5m]) + irate(vendor_statistics_remove_miss_times_seconds_count{container=\"keycloak\", namespace=\"$namespace\", cache=\"$jdbc_cache_names\"}[5m]) \n + \n irate(vendor_statistics_store_times_seconds_count{container=\"keycloak\", namespace=\"$namespace\", cache=\"$jdbc_cache_names\"}[5m])\n))", "legendFormat": "{{pod}}", "range": true, "refId": "A" @@ -3106,7 +3150,7 @@ "h": 15, "w": 6, "x": 12, - "y": 105 + "y": 191 }, "id": 91, "options": { @@ -3131,7 +3175,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum by (pod)(\n rate(\n vendor_statistics_store_times_seconds_sum{\n container=\"keycloak\", \n namespace=\"$namespace\",\n cache=\"$jdbc_cache_names\"}\n [$__range] \n )\n)\n/\nsum by(pod)(\n rate(\n vendor_statistics_store_times_seconds_count{\n container=\"keycloak\",\n namespace=\"$namespace\",\n cache=\"$jdbc_cache_names\"}\n [$__range] \n )\n)", + "expr": "avg by (pod)(\n rate(\n vendor_statistics_store_times_seconds_sum{\n container=\"keycloak\", \n namespace=\"$namespace\",\n cache=\"$jdbc_cache_names\"}\n [$__range] \n )\n)\n/\nsum by(pod)(\n rate(\n vendor_statistics_store_times_seconds_count{\n container=\"keycloak\",\n namespace=\"$namespace\",\n cache=\"$jdbc_cache_names\"}\n [$__range] \n )\n)", "legendFormat": "__auto", "range": true, "refId": "A" @@ -3201,7 +3245,7 @@ "h": 15, "w": 6, "x": 0, - "y": 120 + "y": 206 }, "id": 94, "options": { @@ -3290,7 +3334,7 @@ "h": 15, "w": 6, "x": 6, - "y": 120 + "y": 206 }, "id": 70, "options": { @@ -3316,7 +3360,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum by (pod)(irate(vendor_statistics_hit_times_seconds_count{container=\"keycloak\", cache=\"$jdbc_cache_names\", namespace=\"$namespace\"}[5m]) / (irate(vendor_statistics_hit_times_seconds_count{cache=\"$jdbc_cache_names\", namespace=\"$namespace\", container=\"keycloak\"}[5m]) + irate(vendor_statistics_miss_times_seconds_count{cache=\"$jdbc_cache_names\", namespace=\"$namespace\", container=\"keycloak\"}[5m])))", + "expr": "avg by (pod)(irate(vendor_statistics_hit_times_seconds_count{container=\"keycloak\", cache=\"$jdbc_cache_names\", namespace=\"$namespace\"}[5m]) / (irate(vendor_statistics_hit_times_seconds_count{cache=\"$jdbc_cache_names\", namespace=\"$namespace\", container=\"keycloak\"}[5m]) + irate(vendor_statistics_miss_times_seconds_count{cache=\"$jdbc_cache_names\", namespace=\"$namespace\", container=\"keycloak\"}[5m])))", "hide": false, "legendFormat": "{{pod}}", "range": true, @@ -3387,7 +3431,7 @@ "h": 15, "w": 6, "x": 12, - "y": 120 + "y": 206 }, "id": 53, "options": { @@ -3479,7 +3523,7 @@ "h": 15, "w": 6, "x": 0, - "y": 135 + "y": 221 }, "id": 95, "options": { @@ -3568,7 +3612,7 @@ "h": 15, "w": 6, "x": 6, - "y": 135 + "y": 221 }, "id": 92, "options": { @@ -3594,7 +3638,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum by (pod)(irate(vendor_statistics_remove_hit_times_seconds_count{container=\"keycloak\", cache=\"$jdbc_cache_names\", namespace=\"$namespace\"}[5m]) / (irate(vendor_statistics_remove_hit_times_seconds_count{cache=\"$jdbc_cache_names\", namespace=\"$namespace\", container=\"keycloak\"}[5m]) + irate(vendor_statistics_remove_miss_times_seconds_count{cache=\"$jdbc_cache_names\", namespace=\"$namespace\", container=\"keycloak\"}[5m])))", + "expr": "avg by (pod)(irate(vendor_statistics_remove_hit_times_seconds_count{container=\"keycloak\", cache=\"$jdbc_cache_names\", namespace=\"$namespace\"}[5m]) / (irate(vendor_statistics_remove_hit_times_seconds_count{cache=\"$jdbc_cache_names\", namespace=\"$namespace\", container=\"keycloak\"}[5m]) + irate(vendor_statistics_remove_miss_times_seconds_count{cache=\"$jdbc_cache_names\", namespace=\"$namespace\", container=\"keycloak\"}[5m])))", "hide": false, "legendFormat": "{{pod}}", "range": true, @@ -3661,7 +3705,7 @@ "h": 15, "w": 6, "x": 12, - "y": 135 + "y": 221 }, "id": 87, "options": {