diff --git a/crates/pbs/src/metrics.rs b/crates/pbs/src/metrics.rs index d7f21dd..917e804 100644 --- a/crates/pbs/src/metrics.rs +++ b/crates/pbs/src/metrics.rs @@ -16,7 +16,7 @@ lazy_static! { // FROM RELAYS /// Status code received by relay by endpoint pub static ref RELAY_STATUS_CODE: IntCounterVec = register_int_counter_vec_with_registry!( - "relay_status_code", + "relay_status_code_total", "HTTP status code received by relay", &["http_status_code", "endpoint", "relay_id"], PBS_METRICS_REGISTRY @@ -35,7 +35,7 @@ lazy_static! { // TO BEACON NODE /// Status code returned to beacon node by endpoint pub static ref BEACON_NODE_STATUS: IntCounterVec = register_int_counter_vec_with_registry!( - "beacon_node_status_code", + "beacon_node_status_code_total", "HTTP status code returned to beacon node", &["http_status_code", "endpoint"], PBS_METRICS_REGISTRY diff --git a/grafana/dashboards/dashboard.json b/grafana/dashboards/dashboard.json index 494e204..d729a61 100644 --- a/grafana/dashboards/dashboard.json +++ b/grafana/dashboards/dashboard.json @@ -18,9 +18,23 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 1, "links": [], "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 12, + "panels": [], + "repeat": "endpoint", + "repeatDirection": "h", + "title": "$endpoint calls to relays", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -48,13 +62,16 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, - "pointSize": 2, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "never", - "spanNulls": true, + "showPoints": "auto", + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -64,6 +81,7 @@ } }, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -78,40 +96,114 @@ ] } }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "{__name__=\"cb_pbs_beacon_node_status_code\", endpoint=\"get_header\", http_status_code=\"200\", instance=\"cb_pbs:10000\", job=\"pbs\"}" - ], - "prefix": "All except:", - "readOnly": true - } + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(increase(cb_pbs_relay_status_code_total{http_status_code=~\"2..\", endpoint=\"$endpoint\"}[1h])) by (relay_id)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "$endpoint Relay Success QPH", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "properties": [ + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } + "color": "red", + "value": 80 } ] } - ] + }, + "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 + "h": 11, + "w": 6, + "x": 6, + "y": 1 }, - "id": 5, + "id": 13, "options": { "legend": { "calcs": [], @@ -130,19 +222,15 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "cb_pbs_beacon_node_status_code{http_status_code=\"200\"}", - "fullMetaSearch": false, - "includeNullMetadata": false, + "editorMode": "code", + "expr": "sum(increase(cb_pbs_relay_status_code_total{http_status_code=~\"4..|5..\", endpoint=\"$endpoint\"}[1h])) by (relay_id)", "instant": false, "legendFormat": "__auto", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" } ], - "title": "Beacon node successful calls by endpoint", + "title": "$endpoint Relay Error QPH", "type": "timeseries" }, { @@ -150,7 +238,6 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -173,6 +260,9 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -189,6 +279,7 @@ } }, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -206,18 +297,18 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, + "h": 11, + "w": 6, "x": 12, - "y": 0 + "y": 1 }, - "id": 3, + "id": 43, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { "mode": "single", @@ -230,19 +321,15 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "cb_pbs_relay_status_code{endpoint=\"register_validator\", http_status_code=\"400\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, + "editorMode": "code", + "expr": "sum(increase(cb_pbs_beacon_node_status_code_total{http_status_code=~\"2..\", endpoint=\"$endpoint\"}[1h]))", "instant": false, "legendFormat": "__auto", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" } ], - "title": "Total Failed register_validator failures", + "title": "$endpoint Beacon Node Success QPH", "type": "timeseries" }, { @@ -272,6 +359,9 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -288,6 +378,7 @@ } }, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -302,42 +393,115 @@ ] } }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "{__name__=\"cb_pbs_relay_status_code\", endpoint=\"get_header\", http_status_code=\"200\", instance=\"cb_pbs:10000\", job=\"pbs\", relay_id=\"bloxroute.holesky.blxrbdn.com\"}", - "{__name__=\"cb_pbs_relay_status_code\", endpoint=\"get_header\", http_status_code=\"200\", instance=\"cb_pbs:10000\", job=\"pbs\", relay_id=\"holesky.titanrelay.xyz\"}", - "{__name__=\"cb_pbs_relay_status_code\", endpoint=\"get_header\", http_status_code=\"200\", instance=\"cb_pbs:10000\", job=\"pbs\", relay_id=\"relay-stag.ultrasound.money\"}" - ], - "prefix": "All except:", - "readOnly": true - } + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(increase(cb_pbs_beacon_node_status_code_total{http_status_code=~\"4..|5..\", endpoint=\"$endpoint\"}[1h]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "$endpoint Beacon Node Error QPH", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } + "color": "red", + "value": 80 } ] - } - ] + }, + "unit": "s" + }, + "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, + "h": 11, + "w": 6, "x": 0, - "y": 8 + "y": 12 }, - "id": 2, + "id": 20, "options": { "legend": { "calcs": [], @@ -346,7 +510,7 @@ "showLegend": true }, "tooltip": { - "mode": "none", + "mode": "single", "sort": "none" } }, @@ -357,10 +521,10 @@ "uid": "PBFA97CFB590B2093" }, "disableTextWrap": false, - "editorMode": "builder", - "expr": "cb_pbs_relay_status_code{endpoint=\"get_header\"}", + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(cb_pbs_relay_latency_bucket{endpoint=\"$endpoint\"}[1m])) by (le, relay_id))", "fullMetaSearch": false, - "includeNullMetadata": false, + "includeNullMetadata": true, "instant": false, "legendFormat": "__auto", "range": true, @@ -368,7 +532,7 @@ "useBackend": false } ], - "title": "get_header Total calls by Status and Relay", + "title": "$endpoint Relay P50", "type": "timeseries" }, { @@ -376,7 +540,6 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -399,13 +562,16 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": false, + "showPoints": "always", + "spanNulls": true, "stacking": { "group": "A", "mode": "none" @@ -415,6 +581,7 @@ } }, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -427,23 +594,24 @@ "value": 80 } ] - } + }, + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 + "h": 11, + "w": 6, + "x": 6, + "y": 12 }, - "id": 4, + "id": 29, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { "mode": "single", @@ -457,8 +625,8 @@ "uid": "PBFA97CFB590B2093" }, "disableTextWrap": false, - "editorMode": "builder", - "expr": "cb_pbs_beacon_node_status_code{endpoint=\"submit_blinded_block\"}", + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(cb_pbs_relay_latency_bucket{endpoint=\"$endpoint\"}[1m])) by (le, relay_id))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -468,7 +636,7 @@ "useBackend": false } ], - "title": "submit_blinded_block calls by status", + "title": "$endpoint Relay P90", "type": "timeseries" }, { @@ -498,13 +666,16 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": false, + "showPoints": "always", + "spanNulls": true, "stacking": { "group": "A", "mode": "none" @@ -514,6 +685,7 @@ } }, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -526,23 +698,24 @@ "value": 80 } ] - } + }, + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 + "h": 11, + "w": 6, + "x": 12, + "y": 12 }, - "id": 1, + "id": 30, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { "mode": "single", @@ -556,10 +729,10 @@ "uid": "PBFA97CFB590B2093" }, "disableTextWrap": false, - "editorMode": "builder", - "expr": "cb_pbs_beacon_node_status_code{endpoint=\"register_validator\"}", + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(cb_pbs_relay_latency_bucket{endpoint=\"$endpoint\"}[1m])) by (le, relay_id))", "fullMetaSearch": false, - "includeNullMetadata": false, + "includeNullMetadata": true, "instant": false, "legendFormat": "__auto", "range": true, @@ -567,7 +740,7 @@ "useBackend": false } ], - "title": "Registered validators", + "title": "$endpoint Relay P99", "type": "timeseries" } ], @@ -575,16 +748,49 @@ "schemaVersion": 39, "tags": [], "templating": { - "list": [] + "list": [ + { + "current": { + "selected": true, + "text": [ + "get_header", + "register_validator", + "submit_blinded_block" + ], + "value": [ + "get_header", + "register_validator", + "submit_blinded_block" + ] + }, + "definition": "label_values(endpoint)", + "description": "BuilderAPI endpoint", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "endpoint", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(endpoint)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] }, "time": { - "from": "now-6h", + "from": "now-2d", "to": "now" }, "timepicker": {}, "timezone": "browser", - "title": "Commit-Boost metrics", + "title": "PBS Metrics", "uid": "PBFA97CFB590B2093", - "version": 7, + "version": 1, "weekStart": "" } \ No newline at end of file diff --git a/grafana/datasources/datasources.yml b/grafana/datasources/datasources.yml index aab1d1b..de15442 100644 --- a/grafana/datasources/datasources.yml +++ b/grafana/datasources/datasources.yml @@ -1,7 +1,7 @@ apiVersion: 1 datasources: - - name: Prometheus + - name: cb-prometheus type: prometheus access: proxy orgId: 1