diff --git a/.github/workflows/test-molecule.yml b/.github/workflows/test-molecule.yml index 0c47828f5..46942b81b 100644 --- a/.github/workflows/test-molecule.yml +++ b/.github/workflows/test-molecule.yml @@ -68,11 +68,6 @@ jobs: matrix: tests: [ - { role: "update-changes", test: "20-rc23" }, - { role: "update-changes", test: "202" }, - { role: "update-changes", test: "203" }, - { role: "update-changes", test: "213" }, - { role: "update-changes", test: "214" }, { role: "update-changes", test: "215" }, ] fail-fast: false diff --git a/controls/roles/manage-service/files/grafana/CharonService/dashboards/dash_charon_overview.json b/controls/roles/manage-service/files/grafana/CharonService/dashboards/dash_charon_overview.json index fb4b528fd..b3947e8ae 100644 --- a/controls/roles/manage-service/files/grafana/CharonService/dashboards/dash_charon_overview.json +++ b/controls/roles/manage-service/files/grafana/CharonService/dashboards/dash_charon_overview.json @@ -1,6293 +1,6750 @@ { - "annotations": { - "list": [ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 2, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "cluster-labels" + ], + "targetBlank": false, + "title": "", + "tooltip": "", + "type": "dashboards", + "url": "" + }, + { + "asDropdown": false, + "icon": "info", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "Explore Charon Logs", + "tooltip": "", + "type": "link", + "url": "/explore?schemaVersion=1&panes=%7B\"xlh\"%3A%7B\"datasource\"%3A\"loki\"%2C\"queries\"%3A%5B%7B\"refId\"%3A\"A\"%2C\"expr\"%3A\"\"%2C\"queryType\"%3A\"range\"%2C\"datasource\"%3A%7B\"type\"%3A\"loki\"%2C\"uid\"%3A\"loki\"%7D%7D%5D%2C\"range\"%3A%7B\"from\"%3A\"now-1h\"%2C\"to\"%3A\"now\"%7D%7D%7D&orgId=1" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 168, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Status returned from the `/readyz` health status endpoint. Unhealthy reasons include beacon node problems, charon p2p problems, or validator client problems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" + "options": { + "0": { + "color": "text", + "index": 1, + "text": "Unknown" + }, + "1": { + "color": "green", + "index": 0, + "text": "OK" + }, + "2": { + "color": "red", + "index": 3, + "text": "BeaconNode Down" + }, + "3": { + "color": "orange", + "index": 4, + "text": "BeaconNode Syncing" + }, + "4": { + "color": "orange", + "index": 5, + "text": "Insufficient Peers" }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" + "5": { + "color": "orange", + "index": 6, + "text": "VC not connected" }, - "type": "dashboard" + "6": { + "color": "orange", + "index": 7, + "text": "VC missing validators" + }, + "7": { + "color": "orange", + "index": 8, + "text": "BeaconNode has zero peers" + }, + "8": { + "color": "orange", + "index": 9, + "text": "BeaconNode far behind" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "text", + "index": 2, + "text": "Dead" + } + }, + "type": "special" } - ] + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 170, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Health Status", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Successful cluster duty percentage by type over the whole time range", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "#EAB839", + "value": 0.9 + }, + { + "color": "green", + "value": 0.99 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 14, + "x": 4, + "y": 1 + }, + "id": 176, + "options": { + "minVizHeight": 200, + "minVizWidth": 200, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(\n sum(increase(core_tracker_success_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range])) by (duty)\n) \n/ \n( \n sum(increase(core_tracker_expect_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range])) by (duty) \n) > 0", + "instant": true, + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": false, + "refId": "A" + } + ], + "title": "Cluster Duty Success(%) ", + "type": "gauge" }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 10, - "links": [ + { + "datasource": "${DS_PROMETHEUS}", + "description": "Average beacon node score for the whole time window. The score is a combination of:\n - **errors**: the ratio or errors returned by the beacon node \n - **latency**: how fast or slow the beacon node is responding\n\n\nThe performance of the Beacon node is critical to the performance of the validators and therefore rewards. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "green", + "value": 0.8 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 174, + "options": { + "minVizHeight": 200, + "minVizWidth": 200, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.4.2", + "targets": [ { - "asDropdown": false, - "icon": "external link", - "includeVars": false, - "keepTime": false, - "tags": [], - "targetBlank": false, - "title": "Explore Charon Logs", - "tooltip": "Navigate to Charon Logs Explorer", - "type": "link", - "url": "/explore?orgId=1&left=%7B%22datasource%22:%22loki%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bcluster_name%3D%5C%22$cluster_name%5C%22,%20cluster_hash%3D%5C%22$cluster_hash%5C%22,%20cluster_peer%3D%5C%22$cluster_peer%5C%22%7D%20%7C%20logfmt%20%7C%20line_format%20%60%7B%7B.pretty%7D%7D%60%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22loki%22%7D,%22editorMode%22:%22builder%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(clamp(app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 1 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 4 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 5 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 6, 1, 1) OR on() vector(0))\n", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" }, { - "asDropdown": false, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "cluster-labels" - ], - "targetBlank": false, - "title": "New link", - "tooltip": "", - "type": "dashboards", - "url": "" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "(\n (\n0.5 * (1.0 - 10*(sum(increase(app_eth2_errors_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range])) / (sum(increase(app_eth2_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", endpoint !=\"submit_validator_registrations\"}[$__range]))))) OR on() vector(0.5)\n )\n +\n (\n 0.5 * (1.0 - clamp_max(histogram_quantile(0.99, sum(rate(app_eth2_latency_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", endpoint !=\"submit_validator_registrations\"}[$__range])) by (le)),1))\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + }, + { + "datasource": { + "name": "Expression", + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "$A * $B", + "hide": false, + "refId": "C", + "type": "math" + } + ], + "title": "Beacon Node Score", + "transformations": [ + { + "id": "filterByRefId", + "options": { + "include": "C" + } } - ], - "liveNow": false, - "panels": [ + ], + "type": "gauge" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Information about the charon cluster.\n\n- **Network**: The blockchain network \n- **Operators/Nodes**: The number of operators or nodes or peers in the charon DV cluster.\n- **Threshold**: The minimum number of participating nodes (also quorum).\n- **Validators**: The number of validators added to the cluster, each validator is staking 32 ETH. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Network" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "1": { + "color": "blue", + "index": 0, + "text": "Mainnet" + }, + "2": { + "color": "purple", + "index": 1, + "text": "Goerli" + }, + "3": { + "color": "purple", + "index": 2, + "text": "Sepolia" + }, + "4": { + "color": "purple", + "index": 3, + "text": "Gnosis" + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 115, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 168, - "panels": [], - "title": "Overview", - "type": "row" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "( 1 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"mainnet\"} )\nor \n( 2 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"goerli\"} )\nor\n( 3 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"sepolia\"} )\nor\n( 4 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"gnosis\"} )", + "format": "time_series", + "hide": false, + "instant": true, + "legendFormat": "Network", + "range": false, + "refId": "D" }, { - "datasource": "${DS_PROMETHEUS}", - "description": "Status returned from the `/readyz` health status endpoint. Unhealthy reasons include beacon node problems, charon p2p problems, or validator client problems.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "text", - "index": 1, - "text": "Unknown" - }, - "1": { - "color": "green", - "index": 0, - "text": "OK" - }, - "2": { - "color": "red", - "index": 3, - "text": "BeaconNode Down" - }, - "3": { - "color": "orange", - "index": 4, - "text": "BeaconNode Syncing" - }, - "4": { - "color": "orange", - "index": 5, - "text": "Insufficient Peers" - }, - "5": { - "color": "orange", - "index": 6, - "text": "VC not connected" - }, - "6": { - "color": "orange", - "index": 7, - "text": "VC missing validators" - }, - "7": { - "color": "orange", - "index": 8, - "text": "BeaconNode has zero peers" - }, - "8": { - "color": "orange", - "index": 9, - "text": "BeaconNode far behind" - } - }, - "type": "value" - }, - { - "options": { - "match": "null", - "result": { - "color": "text", - "index": 2, - "text": "Dead" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 170, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Health Status", - "type": "stat" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "cluster_operators{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "instant": true, + "legendFormat": "Operators/Nodes", + "range": false, + "refId": "A" }, { - "datasource": "${DS_PROMETHEUS}", - "description": "Successful cluster duty percentage by type over the whole time range", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "links": [], - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "#EAB839", - "value": 0.9 - }, - { - "color": "green", - "value": 0.99 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 14, - "x": 4, - "y": 1 - }, - "id": 176, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": true - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} - }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_success_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range])) by (duty)\n) \n/ \n( \n sum(increase(core_tracker_expect_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range])) by (duty) \n) > 0", - "instant": true, - "interval": "$interval", - "legendFormat": "{{duty}}", - "range": false, - "refId": "A" - } - ], - "title": "Cluster Duty Success(%) ", - "type": "gauge" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "cluster_threshold{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "hide": false, + "instant": true, + "legendFormat": "Threshold", + "range": false, + "refId": "B" }, { - "datasource": "${DS_PROMETHEUS}", - "description": "Average beacon node score for the whole time window. The score is a combination of:\n - **errors**: the ratio or errors returned by the beacon node \n - **latency**: how fast or slow the beacon node is responding\n\n\nThe performance of the Beacon node is critical to the performance of the validators and therefore rewards. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "orange", - "value": 0.5 - }, - { - "color": "green", - "value": 0.8 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 1 - }, - "id": 174, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(clamp(app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 1 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 4 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 5 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 6, 1, 1) OR on() vector(0))\n*\non()\n(\n (\n0.5 * (1.0 - 10*(sum(increase(app_eth2_errors_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range])) / (sum(increase(app_eth2_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range]))))) OR on() vector(0.5)\n )\n +\n (\n 0.5 * (1.0 - clamp_max(histogram_quantile(0.99, sum(rate(app_eth2_latency_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range])) by (le)),1))\n )\n)", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Beacon Node Score", - "type": "gauge" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "core_scheduler_validators_active{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "hide": false, + "instant": true, + "legendFormat": "Validators Active", + "range": false, + "refId": "E" }, { - "datasource": "${DS_PROMETHEUS}", - "description": "Information about the charon cluster.\n\n- **Network**: The blockchain network \n- **Operators/Nodes**: The number of operators or nodes or peers in the charon DV cluster.\n- **Threshold**: The minimum number of participating nodes (also quorum).\n- **Validators**: The number of validators added to the cluster, each validator is staking 32 ETH. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Network" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "options": { - "1": { - "color": "blue", - "index": 0, - "text": "Mainnet" - }, - "2": { - "color": "purple", - "index": 1, - "text": "Goerli" - }, - "3": { - "color": "purple", - "index": 2, - "text": "Sepolia" - }, - "4": { - "color": "purple", - "index": 3, - "text": "Gnosis" - } - }, - "type": "value" - } - ] - } - ] - } + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "cluster_validators{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "hide": false, + "instant": true, + "legendFormat": "Validators Total", + "range": false, + "refId": "C" + } + ], + "title": "Cluster Info", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This panels shows any health checks that are failing. See for help troubleshooting. \n\nNote the different severities:\n- **info**: Only informational, might not be an actual problem.\n- **warning**: Performance is degraded, operator intervention only required if this persists for a long time.\n- **critical**: Significant problem, validators probably not operation at all. Immediate operator intervention required.\n \n\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "noValue": "All checks are passing ๐ŸŽ‰", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Severity" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "critical": { + "color": "red", + "index": 2, + "text": "critical" + }, + "info": { + "color": "blue", + "index": 0, + "text": "info" + }, + "warning": { + "color": "orange", + "index": 1, + "text": "warning" + } + }, + "type": "value" + } ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "custom.width", + "value": 75 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Check Name" }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 4 - }, - "id": 115, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto" - }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "( 1 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"mainnet\"} )\nor \n( 2 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"goerli\"} )\nor\n( 3 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"sepolia\"} )\nor\n( 4 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"gnosis\"} )", - "format": "time_series", - "hide": false, - "instant": true, - "legendFormat": "Network", - "range": false, - "refId": "D" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "cluster_operators{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "instant": true, - "legendFormat": "Operators/Nodes", - "range": false, - "refId": "A" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "cluster_threshold{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, - "instant": true, - "legendFormat": "Threshold", - "range": false, - "refId": "B" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "core_scheduler_validators_active{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, - "instant": true, - "legendFormat": "Validators Active", - "range": false, - "refId": "E" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "cluster_validators{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, - "instant": true, - "legendFormat": "Validators Total", - "range": false, - "refId": "C" + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" } - ], - "title": "Cluster Info", - "transformations": [], - "type": "stat" + }, + { + "id": "color", + "value": { + "fixedColor": "super-light-yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 0, + "y": 8 + }, + "id": 233, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false }, + "showHeader": true + }, + "pluginVersion": "10.4.2", + "targets": [ { - "datasource": "${DS_PROMETHEUS}", - "description": "- **Peer**: The peer's name (inferred from their *charon-enr-private-key*)\n- **You**: โญ๏ธ is this local charon node\n- **Connected**: Whether you are currently connected to this peer.\n- **Direct**: Whether the connection is *direct* (๐Ÿ‘) or *relay* (๐Ÿ‘Ž) \n- **Latency**: The time messages take to travel to/from the peer.\n- **Version**: The version of the peer's charon node\n- **Supported**: Whether the peers charon version is supported/compatible or not.\n- **Uptime**\" The duration since peer was restarted.\n- **๐ŸšงAttest**: Number of attestation duties missed in the time window.\n- **๐ŸšงPropose**: Number of block proposal duties missed in the time window.\n- **๐ŸšงOther**: Number of other duties missed in the time window.\n\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "center", - "cellOptions": { - "type": "auto" - }, - "inspect": false, - "minWidth": 0, - "width": 90 - }, - "decimals": 0, - "mappings": [], - "noValue": "--", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Latency (ms)" - }, - "properties": [ - { - "id": "unit", - "value": "none" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "from": 0, - "result": { - "color": "green", - "index": 0 - }, - "to": 150 - }, - "type": "range" - }, - { - "options": { - "from": 150, - "result": { - "color": "yellow", - "index": 1 - }, - "to": 300 - }, - "type": "range" - }, - { - "options": { - "from": 300, - "result": { - "color": "orange", - "index": 2 - }, - "to": 500 - }, - "type": "range" - }, - { - "options": { - "from": 500, - "result": { - "color": "red", - "index": 3 - }, - "to": 100000 - }, - "type": "range" - } - ] - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "lcd", - "type": "gauge", - "valueDisplayMode": "color" - } - }, - { - "id": "custom.width", - "value": 200 - }, - { - "id": "max", - "value": 700 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "You" - }, - "properties": [ - { - "id": "noValue", - "value": "โญ" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "auto" - } - }, - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "--" - }, - "1": { - "index": 1, - "text": "--" - } - }, - "type": "value" - } - ] - }, - { - "id": "custom.width", - "value": 60 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Connected" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "ย โŒ" - }, - "1": { - "index": 1, - "text": "โœ…" - } - }, - "type": "value" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "ClockDiff" - }, - "properties": [ - { - "id": "unit", - "value": "ms" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "๐Ÿ†—" - } - }, - "type": "value" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Direct" - }, - "properties": [ - { - "id": "custom.width", - "value": 55 - }, - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "--" - } - }, - "type": "value" - }, - { - "options": { - "from": 1, - "result": { - "index": 1, - "text": "๐Ÿ‘Ž" - }, - "to": 9 - }, - "type": "range" - }, - { - "options": { - "from": 10, - "result": { - "index": 2, - "text": "๐Ÿ‘" - }, - "to": 1000 - }, - "type": "range" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Peer" - }, - "properties": [ - { - "id": "custom.width", - "value": 150 - } - ] + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(app_health_checks{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (name,severity) > 0", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Failing Health Checks", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true + }, + "indexByName": {}, + "renameByName": { + "Value": "", + "name": "Check Name", + "severity": "Severity" + } + } + } + ], + "type": "table" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "- **Peer**: The peer's name (inferred from their *charon-enr-private-key*)\n- **You**: โญ๏ธ is this local charon node\n- **Connected**: Whether you are currently connected to this peer.\n- **Direct**: Whether the connection is *direct* (๐Ÿ‘) or *relay* (๐Ÿ‘Ž) \n- **Latency**: The time messages take to travel to/from the peer.\n- **Version**: The version of the peer's charon node\n- **Supported**: Whether the peers charon version is supported/compatible or not.\n- **Uptime**\" The duration since peer was restarted.\n- **๐ŸšงAttest**: Number of attestation duties missed in the time window.\n- **๐ŸšงPropose**: Number of block proposal duties missed in the time window.\n- **๐ŸšงOther**: Number of other duties missed in the time window.\n\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "minWidth": 0, + "width": 90 + }, + "decimals": 0, + "mappings": [], + "noValue": "--", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Latency (ms)" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "from": 0, + "result": { + "color": "green", + "index": 0 + }, + "to": 150 + }, + "type": "range" + }, + { + "options": { + "from": 150, + "result": { + "color": "yellow", + "index": 1 + }, + "to": 300 + }, + "type": "range" + }, + { + "options": { + "from": 300, + "result": { + "color": "orange", + "index": 2 + }, + "to": 500 + }, + "type": "range" + }, + { + "options": { + "from": 500, + "result": { + "color": "red", + "index": 3 + }, + "to": 100000 + }, + "type": "range" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "lcd", + "type": "gauge", + "valueDisplayMode": "color" + } + }, + { + "id": "custom.width", + "value": 200 + }, + { + "id": "max", + "value": 700 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "You" + }, + "properties": [ + { + "id": "noValue", + "value": "โญ" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "auto" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "--" + }, + "1": { + "index": 1, + "text": "--" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.width", + "value": 60 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Connected" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": " โŒ" + }, + "1": { + "index": 1, + "text": "โœ…" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ClockDiff" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "๐Ÿ†—" + } }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Direct" + }, + "properties": [ + { + "id": "custom.width", + "value": 55 + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "--" + } + }, + "type": "value" + }, + { + "options": { + "from": 1, + "result": { + "index": 1, + "text": "๐Ÿ‘Ž" + }, + "to": 9 + }, + "type": "range" + }, + { + "options": { + "from": 10, + "result": { + "index": 2, + "text": "๐Ÿ‘" + }, + "to": 1000 + }, + "type": "range" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Peer" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Uptime" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Index" + }, + "properties": [ + { + "id": "custom.width", + "value": 30 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "๐Ÿšง.*" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "thresholds" + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ { - "matcher": { - "id": "byName", - "options": "Uptime" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] + "color": "#585858", + "value": null }, { - "matcher": { - "id": "byName", - "options": "Index" - }, - "properties": [ - { - "id": "custom.width", - "value": 30 - } - ] + "color": "super-light-green", + "value": 0.00001 }, { - "matcher": { - "id": "byRegexp", - "options": "๐Ÿšง.*" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "thresholds" - } - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "decimals", - "value": 1 - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "#585858", - "value": null - }, - { - "color": "super-light-green", - "value": 0.00001 - }, - { - "color": "yellow", - "value": 0.01 - }, - { - "color": "red", - "value": 0.1 - } - ] - } - }, - { - "id": "noValue", - "value": "0%" - } - ] + "color": "yellow", + "value": 0.01 }, { - "matcher": { - "id": "byName", - "options": "Supported" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "โŒ" - }, - "1": { - "index": 1, - "text": "๐Ÿ†—" - } - }, - "type": "value" - } - ] - } - ] + "color": "red", + "value": 0.1 } + ] + } + }, + { + "id": "noValue", + "value": "0%" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Supported" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "โŒ" + }, + "1": { + "index": 1, + "text": "๐Ÿ†—" + } + }, + "type": "value" + } ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Builder API" }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 85, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "frameIndex": 0, - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "You" - } + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "โŒ" + }, + "1": { + "index": 1, + "text": "โœ…" + } + }, + "type": "value" + } ] + }, + { + "id": "custom.width", + "value": 99 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Version" }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", - "format": "table", - "instant": true, - "range": false, - "refId": "A" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__rate_interval])) by (le,peer)) * 1000", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "B" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "", - "range": false, - "refId": "F" + "properties": [ + { + "id": "custom.width", + "value": 104 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 19, + "x": 5, + "y": 8 + }, + "id": 85, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "You" + } + ] + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", + "format": "table", + "instant": true, + "range": false, + "refId": "A" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__rate_interval])) by (le,peer)) * 1000", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "B" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "", + "range": false, + "refId": "F" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer)*10 + \nmax(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "", + "range": false, + "refId": "K" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer,version) ", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "L" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "time() - max(app_peerinfo_start_time_secs{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} > 0) by (peer)", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "Q" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(app_peerinfo_index{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "R" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"attester\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"attester\"}[$__range])) by (peer) \n) OR on() vector(0)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "C" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=~\".*proposer\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=~\".*proposer\"}[$__range])) by (peer) \n) OR on() vector(0)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "D" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty!~\"(proposer|attester)\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty!~\"(proposer|attester)\"}[$__range])) by (peer) \n) OR on() vector(0)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "E" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(app_peerinfo_version_support{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer) ", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "", + "range": false, + "refId": "G" + }, + { + "datasource": "${DS_PROMETHEUS}", + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "max(app_peerinfo_builder_api_enabled{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", + "format": "table", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "H", + "useBackend": false + } + ], + "title": "Peer Connectivity and Missed Duties (๐Ÿšง)", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "peer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time 1": true, + "Time 10": true, + "Time 11": true, + "Time 12": true, + "Time 13": true, + "Time 14": true, + "Time 15": true, + "Time 16": true, + "Time 17": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "Time 6": true, + "Time 7": true, + "Time 8": true, + "Time 9": true, + "Value #F": false, + "Value #L": true, + "Value #M": true + }, + "includeByName": {}, + "indexByName": { + "Time 1": 9, + "Time 10": 22, + "Time 11": 23, + "Time 2": 11, + "Time 3": 12, + "Time 4": 13, + "Time 5": 14, + "Time 6": 15, + "Time 7": 16, + "Time 8": 17, + "Time 9": 21, + "Value #A": 3, + "Value #B": 5, + "Value #C": 18, + "Value #D": 19, + "Value #E": 20, + "Value #F": 2, + "Value #G": 7, + "Value #K": 4, + "Value #L": 10, + "Value #Q": 8, + "Value #R": 1, + "peer": 0, + "version": 6 + }, + "renameByName": { + "Value #A": "Connected", + "Value #B": "Latency (ms)", + "Value #C": "๐ŸšงAttest", + "Value #D": "๐ŸšงPropose", + "Value #E": "๐ŸšงOther", + "Value #F": "You", + "Value #G": "Supported", + "Value #H": "Builder API", + "Value #I": "Exit", + "Value #J": "ClockDiff", + "Value #K": "Direct", + "Value #L": "", + "Value #N": "SyncMsg", + "Value #O": "SyncContrib", + "Value #P": "PrepareContrib", + "Value #Q": "Uptime", + "Value #R": "Index", + "git_hash": "GitCommit", + "peer": "Peer", + "version": "Version" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 193, + "panels": [], + "title": "Overview Advanced", + "type": "row" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 16 + }, + "id": 83, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "mode": "none", + "sort": "none" + } + }, + "pluginVersion": "9.2.3", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (version)", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{version}}", + "range": true, + "refId": "A" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(app_git_commit{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (git_hash)", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{git_hash}}", + "range": true, + "refId": "B" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "rate(app_start_time_secs{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) > 0", + "hide": false, + "instant": false, + "interval": "$interval", + "legendFormat": "Restarts", + "range": true, + "refId": "C" + } + ], + "title": "Versions and Git Hashes", + "type": "state-timeline" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Status returned from the `/readyz` health status endpoint. Unhealthy reasons include beacon node problems, charon p2p problems, or validator client problems.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#ffffff", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "color": "text", + "index": 1, + "text": "Unknown" }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer)*10 + \nmax(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "", - "range": false, - "refId": "K" + "1": { + "color": "green", + "index": 0, + "text": "OK" }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "sum(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer,version) ", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "L" + "2": { + "color": "red", + "index": 3, + "text": "BeaconNode Down" }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "time() - max(app_peerinfo_start_time_secs{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} > 0) by (peer)", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "Q" + "3": { + "color": "orange", + "index": 4, + "text": "BeaconNode Syncing" }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(app_peerinfo_index{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "R" + "4": { + "color": "orange", + "index": 5, + "text": "Insufficient Peers" }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"attester\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"attester\"}[$__range])) by (peer) \n) OR on() vector(0)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "C" + "5": { + "color": "light-orange", + "index": 6, + "text": "VC not connected" }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=~\".*proposer\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=~\".*proposer\"}[$__range])) by (peer) \n) OR on() vector(0)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "D" + "6": { + "color": "orange", + "index": 7, + "text": "VC missing validators" }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty!~\"(proposer|attester)\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty!~\"(proposer|attester)\"}[$__range])) by (peer) \n) OR on() vector(0)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "E" + "7": { + "color": "orange", + "index": 8, + "text": "BeaconNode has zero peers" }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(app_peerinfo_version_support{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer) ", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "", - "range": false, - "refId": "G" + "8": { + "index": 9, + "text": "BeaconNode far behind" } - ], - "title": "Peer Connectivity and Missed Duties (๐Ÿšง)", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "peer" - } - }, - { - "id": "organize", + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "text", + "index": 2, + "text": "Dead" + } + }, + "type": "special" + } + ], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#ccccdb", + "value": null + }, + { + "color": "green", + "value": 0.5 + }, + { + "color": "red", + "value": 1.5 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 7, + "y": 16 + }, + "id": 119, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.3", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "hide": false, + "instant": false, + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Historical Health Status", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Type of connections to each peer in the cluster during the time window:\n - ๐ŸŸข Green: Direct connection (this is the preferred)\n - ๐ŸŸก Yellow: Indirect relay connection (this is ok if only temporary)\n - ๐Ÿ”ด Red: No connection to peer (this is a problem)\n\nSee https://docs.obol.tech/docs/charon/networking#external-p2p-network", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "green", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 9, + "x": 15, + "y": 16 + }, + "id": 87, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "(max(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer))\n+\n(1 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))\n+ \n(100 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer))", + "format": "time_series", + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Peer Connections (๐ŸŸข=direct, ๐ŸŸก=relay, ๐Ÿ”ด=not connected)", + "type": "state-timeline" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "fixed" + }, + "mappings": [], + "max": 10, + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "P2P Reachability" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { "options": { - "excludeByName": { - "Time 1": true, - "Time 10": true, - "Time 11": true, - "Time 12": true, - "Time 13": true, - "Time 14": true, - "Time 15": true, - "Time 16": true, - "Time 17": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "Time 6": true, - "Time 7": true, - "Time 8": true, - "Time 9": true, - "Value #F": false, - "Value #L": true, - "Value #M": true - }, - "indexByName": { - "Time 1": 9, - "Time 10": 22, - "Time 11": 23, - "Time 2": 11, - "Time 3": 12, - "Time 4": 13, - "Time 5": 14, - "Time 6": 15, - "Time 7": 16, - "Time 8": 17, - "Time 9": 21, - "Value #A": 3, - "Value #B": 5, - "Value #C": 18, - "Value #D": 19, - "Value #E": 20, - "Value #F": 2, - "Value #G": 7, - "Value #K": 4, - "Value #L": 10, - "Value #Q": 8, - "Value #R": 1, - "peer": 0, - "version": 6 - }, - "renameByName": { - "Value #A": "Connected", - "Value #B": "Latency (ms)", - "Value #C": "๐ŸšงAttest", - "Value #D": "๐ŸšงPropose", - "Value #E": "๐ŸšงOther", - "Value #F": "You", - "Value #G": "Supported", - "Value #H": "Aggregate", - "Value #I": "Exit", - "Value #J": "ClockDiff", - "Value #K": "Direct", - "Value #L": "", - "Value #N": "SyncMsg", - "Value #O": "SyncContrib", - "Value #P": "PrepareContrib", - "Value #Q": "Uptime", - "Value #R": "Index", - "git_hash": "GitCommit", - "peer": "Peer", - "version": "Version" - } - } + "0": { + "color": "text", + "index": 0, + "text": "Unknown" + }, + "1": { + "color": "green", + "index": 1, + "text": "Public" + }, + "2": { + "color": "yellow", + "index": 2, + "text": "Private" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Uptime" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "color", + "value": { + "fixedColor": "text", + "mode": "fixed" } - ], - "type": "table" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 21 + }, + "id": 113, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "titleSize": 15, + "valueSize": 15 }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 193, - "panels": [ - { - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 0, - "spanNulls": false - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 7, - "x": 0, - "y": 16 - }, - "id": 83, + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "time() - max(app_start_time_secs{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "hide": false, + "instant": true, + "legendFormat": "Uptime", + "range": false, + "refId": "G" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "core_scheduler_current_slot{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Slot", + "range": false, + "refId": "A" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "core_scheduler_current_epoch{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "hide": false, + "instant": true, + "legendFormat": "Epoch", + "range": false, + "refId": "B" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "core_scheduler_validators_active{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "hide": false, + "instant": true, + "legendFormat": "Validators Active", + "range": false, + "refId": "C" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "hide": false, + "instant": true, + "legendFormat": "Charon Peers Connected", + "range": false, + "refId": "D" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(p2p_relay_connections{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) ", + "hide": false, + "instant": true, + "legendFormat": "Relays Connected", + "range": false, + "refId": "E" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "app_beacon_node_peers{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "hide": false, + "instant": true, + "legendFormat": "Beacon Node Peers", + "range": false, + "refId": "H" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "p2p_reachability_status{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "hide": false, + "instant": true, + "legendFormat": "P2P Reachability", + "range": false, + "refId": "F" + } + ], + "title": "Current Status", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Status and balance of each validator by public key with links to beaconcha.in.\n\nIf present, the \"โ—๏ธPeer Exits\" column indicate the number of peers that submitted partially signed exits for each validator.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-blue" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Balance" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "mappings", + "value": [ + { "options": { - "alignValue": "left", - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "mergeValues": true, - "rowHeight": 0.9, - "showValue": "never", - "tooltip": { - "mode": "none", - "sort": "none" - } - }, - "pluginVersion": "9.2.3", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (version)", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{version}}", - "range": true, - "refId": "A" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(app_git_commit{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (git_hash)", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{git_hash}}", - "range": true, - "refId": "B" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "rate(app_start_time_secs{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) > 0", - "hide": false, - "instant": false, - "interval": "$interval", - "legendFormat": "Restarts", - "range": true, - "refId": "C" - } - ], - "title": "Versions and Git Hashes", - "type": "state-timeline" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Status returned from the `/readyz` health status endpoint. Unhealthy reasons include beacon node problems, charon p2p problems, or validator client problems.", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "#ffffff", - "mode": "fixed" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "stepAfter", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "area" - } - }, - "decimals": 0, - "mappings": [ - { - "options": { - "0": { - "color": "text", - "index": 1, - "text": "Unknown" - }, - "1": { - "color": "green", - "index": 0, - "text": "OK" - }, - "2": { - "color": "red", - "index": 3, - "text": "BeaconNode Down" - }, - "3": { - "color": "orange", - "index": 4, - "text": "BeaconNode Syncing" - }, - "4": { - "color": "orange", - "index": 5, - "text": "Insufficient Peers" - }, - "5": { - "color": "light-orange", - "index": 6, - "text": "VC not connected" - }, - "6": { - "color": "orange", - "index": 7, - "text": "VC missing validators" - }, - "7": { - "color": "orange", - "index": 8, - "text": "BeaconNode has zero peers" - }, - "8": { - "index": 9, - "text": "BeaconNode far behind" - } - }, - "type": "value" - }, - { - "options": { - "match": "null", - "result": { - "color": "text", - "index": 2, - "text": "Dead" - } - }, - "type": "special" - } - ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#ccccdb", - "value": null - }, - { - "color": "green", - "value": 0.5 - }, - { - "color": "red", - "value": 1.5 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 7, - "y": 16 - }, - "id": 119, + "0": { + "color": "transparent", + "index": 3 + } + }, + "type": "value" + }, + { "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.3", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", - "hide": false, - "instant": false, - "interval": "$interval", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Historical Health Status", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Type of connections to each peer in the cluster during the time window:\n - ๐ŸŸข Green: Direct connection (this is the preferred)\n - ๐ŸŸก Yellow: Indirect relay connection (this is ok if only temporary)\n - ๐Ÿ”ด Red: No connection to peer (this is a problem)\n\nSee https://docs.obol.tech/docs/charon/networking#external-p2p-network", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 0, - "spanNulls": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "#EAB839", - "value": 1 - }, - { - "color": "green", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 9, - "x": 15, - "y": 16 - }, - "id": 87, + "from": 32, + "result": { + "color": "green", + "index": 0 + }, + "to": 100 + }, + "type": "range" + }, + { "options": { - "alignValue": "left", - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "mergeValues": true, - "rowHeight": 0.9, - "showValue": "never", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "(max(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer))\n+\n(1 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))\n+ \n(100 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer))", - "format": "time_series", - "interval": "$interval", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Peer Connections (๐ŸŸข=direct, ๐ŸŸก=relay, ๐Ÿ”ด=not connected)", - "type": "state-timeline" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "text", - "mode": "fixed" - }, - "mappings": [], - "max": 10, - "min": 0, - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "P2P Reachability" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "color": "text", - "index": 0, - "text": "Unknown" - }, - "1": { - "color": "green", - "index": 1, - "text": "Public" - }, - "2": { - "color": "yellow", - "index": 2, - "text": "Private" - } - }, - "type": "value" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Uptime" - }, - "properties": [ - { - "id": "unit", - "value": "s" - }, - { - "id": "decimals", - "value": 1 - }, - { - "id": "color", - "value": { - "fixedColor": "text", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 0, - "y": 21 - }, - "id": 113, + "from": 30, + "result": { + "color": "orange", + "index": 1 + }, + "to": 32 + }, + "type": "range" + }, + { "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto" - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "time() - max(app_start_time_secs{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", - "hide": false, - "instant": true, - "legendFormat": "Uptime", - "range": false, - "refId": "G" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "core_scheduler_current_slot{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Slot", - "range": false, - "refId": "A" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "core_scheduler_current_epoch{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, - "instant": true, - "legendFormat": "Epoch", - "range": false, - "refId": "B" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "core_scheduler_validators_active{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, - "instant": true, - "legendFormat": "Validators Active", - "range": false, - "refId": "C" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", - "hide": false, - "instant": true, - "legendFormat": "Charon Peers Connected", - "range": false, - "refId": "D" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "sum(p2p_relay_connections{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) ", - "hide": false, - "instant": true, - "legendFormat": "Relays Connected", - "range": false, - "refId": "E" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "app_beacon_node_peers{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, - "instant": true, - "legendFormat": "Beacon Node Peers", - "range": false, - "refId": "H" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "p2p_reachability_status{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, - "instant": true, - "legendFormat": "P2P Reachability", - "range": false, - "refId": "F" - } - ], - "title": "Current Status", - "transformations": [], - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Status and balance of each validator by public key with links to beaconcha.in.\n\nIf present, the \"โ—๏ธPeer Exits\" column indicate the number of peers that submitted partially signed exits for each validator.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "light-blue", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Balance" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "color": "transparent", - "index": 3 - } - }, - "type": "value" - }, - { - "options": { - "from": 32, - "result": { - "color": "green", - "index": 0 - }, - "to": 100 - }, - "type": "range" - }, - { - "options": { - "from": 30, - "result": { - "color": "orange", - "index": 1 - }, - "to": 32 - }, - "type": "range" - }, - { - "options": { - "from": 1, - "result": { - "color": "red", - "index": 2 - }, - "to": 30 - }, - "type": "range" - } - ] - }, - { - "id": "unit", - "value": "ETH" - }, - { - "id": "custom.width", - "value": 150 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": " Public Key" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "targetBlank": true, - "title": "beaconcha.in", - "url": "http://${__data.fields[\"cluster_network\"]}.beaconcha.in/validator/${__data.fields[\"pubkey_full\"]}" - } - ] - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "custom.width" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Status" - }, - "properties": [ - { - "id": "custom.width", - "value": 150 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*Exits" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "__hide__" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 9, - "x": 4, - "y": 21 - }, - "id": 134, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(core_scheduler_validator_balance_gwei{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} / 1000000000) by (pubkey_full,pubkey, cluster_network)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "{{pubkey}}", - "range": false, - "refId": "B" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(core_scheduler_validator_status{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 1) by (pubkey, status)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(core_parsigdb_exit_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (pubkey) ", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "C" - } - ], - "title": "Validators", - "transformations": [ - { - "id": "joinByField", - "options": { - "byField": "pubkey", - "mode": "outer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "Time 3": true, - "Value #A": true, - "Value #B": false, - "cluster_network": false, - "job": true, - "pubkey_full": false - }, - "indexByName": { - "Time 1": 2, - "Time 2": 7, - "Time 3": 9, - "Value #A": 8, - "Value #B": 6, - "Value #C": 4, - "cluster_network": 3, - "pubkey": 0, - "pubkey_full": 1, - "status": 5 - }, - "renameByName": { - "Value #A": "", - "Value #B": "Balance", - "Value #C": "โ—๏ธPeer Exits", - "cluster_network": "__hide__", - "pubkey": " Public Key", - "pubkey_full": "__hide__", - "status": "Status" - } - } - } - ], - "type": "table" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Warning (w) and error (e) logs grouped by topic. \n\nOccasional warns and errors are normal, but a significant increase indicate problems that require operators to investigate by looking at the actual logs entries.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "area" - } - }, - "links": [ - { - "targetBlank": true, - "title": "Show in log explorer", - "url": "/explore?orgId=1&left=%7B%22datasource%22:%22loki%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcompose_service%3D%5C%22$job%5C%22%7D%20%7C%3D%20%60${__field.labels.msg}%60%20%7C%20logfmt%20%7C%20line_format%20%60%7B%7B.pretty%7D%7D%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D" - } - ], - "mappings": [], - "min": 0, - "noValue": "No errors or warnings", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 100 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 11, - "x": 13, - "y": 21 - }, - "id": 117, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "none", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "sum(increase(app_log_error_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (topic) > 0 ", - "hide": false, - "interval": "$interval", - "legendFormat": "e:{{topic}}", - "range": true, - "refId": "B" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "sum(increase(app_log_warn_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (topic) > 0 ", - "hide": false, - "interval": "$interval", - "legendFormat": "w:{{topic}}", - "range": true, - "refId": "A" - } - ], - "title": "Warning(w) and Error(e) logs (per $interval)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "- **Peer**: The peer's name (inferred from their *charon-enr-private-key*)\n- **You**: โญ๏ธ is this local charon node\n- **Connected**: Whether you are currently connected to this peer.\n- **Direct**: Whether the connection is *direct* (๐Ÿ‘) or *relay* (๐Ÿ‘Ž) \n- **Latency**: The time messages take to travel to/from the peer.\n- **Uptime**\" The duration since peer was restarted.\n- **ClockDiff**: Difference between local and peer's clock time. More than 2s is bad.\n- **๐ŸšงAttest**: Number of attestation duties missed in the time window.\n- **๐ŸšงPropose**: Number of block proposal duties missed in the time window.\n- **๐ŸšงOther**: Number of other duties missed in the time window.\n\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "center", - "cellOptions": { - "type": "auto" - }, - "inspect": false, - "minWidth": 0, - "width": 90 - }, - "decimals": 0, - "mappings": [], - "noValue": "--", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent" - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Latency" - }, - "properties": [ - { - "id": "unit", - "value": "none" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "from": 0, - "result": { - "color": "green", - "index": 0 - }, - "to": 150 - }, - "type": "range" - }, - { - "options": { - "from": 150, - "result": { - "color": "yellow", - "index": 1 - }, - "to": 300 - }, - "type": "range" - }, - { - "options": { - "from": 300, - "result": { - "color": "orange", - "index": 2 - }, - "to": 500 - }, - "type": "range" - }, - { - "options": { - "from": 500, - "result": { - "color": "red", - "index": 3 - }, - "to": 100000 - }, - "type": "range" - } - ] - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "lcd", - "type": "gauge" - } - }, - { - "id": "custom.width", - "value": 200 - }, - { - "id": "max", - "value": 700 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "You" - }, - "properties": [ - { - "id": "noValue", - "value": "โญ" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "auto" - } - }, - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "--" - }, - "1": { - "index": 1, - "text": "--" - } - }, - "type": "value" - } - ] - }, - { - "id": "custom.width", - "value": 60 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Connected" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "ย โŒ" - }, - "1": { - "index": 1, - "text": "โœ…" - } - }, - "type": "value" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "ClockDiff" - }, - "properties": [ - { - "id": "unit", - "value": "ms" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "๐Ÿ†—" - } - }, - "type": "value" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Direct" - }, - "properties": [ - { - "id": "custom.width", - "value": 55 - }, - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "--" - } - }, - "type": "value" - }, - { - "options": { - "from": 1, - "result": { - "index": 1, - "text": "๐Ÿ‘Ž" - }, - "to": 9 - }, - "type": "range" - }, - { - "options": { - "from": 10, - "result": { - "index": 2, - "text": "๐Ÿ‘" - }, - "to": 1000 - }, - "type": "range" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Peer" - }, - "properties": [ - { - "id": "custom.width", - "value": 150 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Uptime" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Index" - }, - "properties": [ - { - "id": "custom.width", - "value": 30 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "๐Ÿšง.*" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "thresholds" - } - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "decimals", - "value": 1 - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "#585858" - }, - { - "color": "super-light-green", - "value": 0.00001 - }, - { - "color": "yellow", - "value": 0.01 - }, - { - "color": "red", - "value": 0.1 - } - ] - } - }, - { - "id": "noValue", - "value": "0%" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 194, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "frameIndex": 0, - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "You" - } - ] - }, - "pluginVersion": "9.5.2", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", - "format": "table", - "instant": true, - "range": false, - "refId": "A" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__rate_interval])) by (le,peer)) * 1000", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "B" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "", - "range": false, - "refId": "F" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(round(app_peerinfo_clock_offset_seconds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"},0.1)*1000) by (peer) ", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "J" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer)*10 + \nmax(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "", - "range": false, - "refId": "K" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "sum(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer,version) ", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "L" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(app_peerinfo_git_commit{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer,git_hash) > 0", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "M" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "time() - max(app_peerinfo_start_time_secs{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} > 0) by (peer)", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "Q" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(app_peerinfo_index{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "R" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"attester\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"attester\"}[$__range])) by (peer) \n) > 0 ", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "C" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"proposer\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"proposer\"}[$__range])) by (peer) \n) > 0 ", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "D" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty!~\"(proposer|attester)\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty!~\"(proposer|attester)\"}[$__range])) by (peer) \n) > 0 ", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "E" - } - ], - "title": "Peer Connectivity and Missed Duties (๐Ÿšง)", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "peer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time 1": true, - "Time 10": true, - "Time 11": true, - "Time 12": true, - "Time 13": true, - "Time 14": true, - "Time 15": true, - "Time 16": true, - "Time 17": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "Time 6": true, - "Time 7": true, - "Time 8": true, - "Time 9": true, - "Value #F": false, - "Value #L": true, - "Value #M": true - }, - "indexByName": { - "Time 1": 10, - "Time 10": 29, - "Time 11": 30, - "Time 12": 31, - "Time 13": 32, - "Time 14": 33, - "Time 15": 34, - "Time 16": 35, - "Time 17": 36, - "Time 2": 12, - "Time 3": 13, - "Time 4": 14, - "Time 5": 15, - "Time 6": 16, - "Time 7": 17, - "Time 8": 18, - "Time 9": 28, - "Value #A": 3, - "Value #B": 5, - "Value #C": 20, - "Value #D": 21, - "Value #E": 22, - "Value #F": 2, - "Value #G": 23, - "Value #H": 24, - "Value #J": 6, - "Value #K": 4, - "Value #L": 11, - "Value #M": 19, - "Value #N": 25, - "Value #O": 27, - "Value #P": 26, - "Value #Q": 9, - "Value #R": 1, - "git_hash": 8, - "peer": 0, - "version": 7 - }, - "renameByName": { - "Value #A": "Connected", - "Value #B": "Latency", - "Value #C": "๐ŸšงAttest", - "Value #D": "๐ŸšงPropose", - "Value #E": "๐ŸšงOther", - "Value #F": "You", - "Value #G": "PrepareAgg", - "Value #H": "Aggregate", - "Value #I": "Exit", - "Value #J": "ClockDiff", - "Value #K": "Direct", - "Value #L": "", - "Value #N": "SyncMsg", - "Value #O": "SyncContrib", - "Value #P": "PrepareContrib", - "Value #Q": "Uptime", - "Value #R": "Index", - "git_hash": "GitCommit", - "peer": "Peer", - "version": "Version" - } - } - } - ], - "type": "table" + "from": 1, + "result": { + "color": "red", + "index": 2 + }, + "to": 30 + }, + "type": "range" + } + ] + }, + { + "id": "unit", + "value": "ETH" + }, + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": " Public Key" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "beaconcha.in", + "url": "http://${__data.fields[\"cluster_network\"]}.beaconcha.in/validator/${__data.fields[\"pubkey_full\"]}" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" } - ], - "title": "Overview Advanced", - "type": "row" + }, + { + "id": "custom.width" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Exits" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__hide__" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 9, + "x": 4, + "y": 21 + }, + "id": 134, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": [ + "Value #B" + ], + "reducer": [ + "sum" + ], + "show": true }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Balance" + } + ] + }, + "pluginVersion": "10.4.2", + "targets": [ { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 178, - "panels": [ - { - "datasource": "${DS_PROMETHEUS}", - "description": "Completed duties by type over time.", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "red", - "mode": "palette-classic" - }, - "custom": { - "fillOpacity": 100, - "lineWidth": 0 - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 17 - }, - "id": 92, - "options": { - "colWidth": 0.5, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "rowHeight": 0.43, - "showValue": "auto", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "sum(increase(core_bcast_broadcast_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) > 0", - "instant": false, - "interval": "$interval", - "legendFormat": "{{duty}}", - "range": true, - "refId": "A" - } - ], - "title": "โœ… Completed duties by type ", - "type": "status-history" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Successful duty participation per peer (filtered by selected duty) over time.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 17 - }, - "id": 81, - "options": { - "colWidth": 0.9, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "rowHeight": 0.9, - "showValue": "never", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(core_tracker_participation_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\",duty=~\"$duty\"}[$interval])) by (peer) > 0 ", - "interval": "$interval", - "legendFormat": "{{peer}}", - "range": true, - "refId": "A" - } - ], - "title": "โœ… Successful Peer Participation: Duty=$duty", - "type": "status-history" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Failed duties by type over time. Use data link to show logs.", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "red", - "mode": "palette-classic" - }, - "custom": { - "fillOpacity": 100, - "lineWidth": 0 - }, - "links": [ - { - "targetBlank": true, - "title": "Show in log explorer", - "url": "/explore?orgId=1&left=%7B%22datasource%22:%22loki%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcompose_service%3D%5C%22$job%5C%22%7D%20%7C%3D%20%60Duty%20failed%60%20%7C%3D%20%60${__field.labels.duty}%60%20%7C%20logfmt%20%7C%20line_format%20%60%7B%7B.pretty%7D%7D%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D" - } - ], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 23 - }, - "id": 91, - "options": { - "colWidth": 0.5, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "rowHeight": 0.43, - "showValue": "auto", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "sum(increase(core_tracker_failed_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) > 0", - "instant": false, - "interval": "$interval", - "legendFormat": "{{duty}}", - "range": true, - "refId": "A" - } - ], - "title": "โŒ Failed duties by type (per $interval)", - "type": "status-history" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Missed duty participation per peer (filtered by selected duty) over time.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 23 - }, - "id": 185, - "options": { - "colWidth": 0.9, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "rowHeight": 0.9, - "showValue": "never", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\",duty=~\"$duty\"}[$interval])) by (peer) > 0 ", - "interval": "$interval", - "legendFormat": "{{peer}}", - "range": true, - "refId": "A" - } - ], - "title": "๐Ÿšง Missed Peer Participation: Duty=$duty", - "type": "status-history" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Failed duties percentage by type over time", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "red", - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 29 - }, - "id": 180, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_failed_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty)\n) \n/ \n(\n sum(increase(core_tracker_failed_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) \n + \n sum(increase(core_bcast_broadcast_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) \n) > 0 ", - "instant": false, - "interval": "$interval", - "legendFormat": "{{duty}}", - "range": true, - "refId": "A" - } - ], - "title": "โŒ Failed duty percentage by type (per $interval)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Missed duty participation per duty (filtered by peer) over time.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 29 - }, - "id": 186, - "options": { - "colWidth": 0.9, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "rowHeight": 0.9, - "showValue": "never", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "repeat": "peer", - "repeatDirection": "v", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=\"$peer\"}[$interval])) by (duty) > 0 ", - "interval": "$interval", - "legendFormat": "{{duty}}", - "range": true, - "refId": "A" - } - ], - "title": "๐Ÿšง Missed Peer Participation: Peer=$peer", - "type": "status-history" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Number of failed duties by reason per time window. Note this is only available from v0.16.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "mappings": [], - "noValue": "No failures ๐ŸŽ‰", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 35 - }, - "id": 222, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "sum(increase(core_tracker_failed_duty_reasons_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval])) by (reason) > 0 ", - "hide": false, - "interval": "$interval", - "legendFormat": "{{.duty}}", - "range": true, - "refId": "A" - } - ], - "title": "๐Ÿšจ Duty failure reason: $duty (per $interval)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Duties that was successfully broadcast to the Beacon Node, but that was never included on-chain. This could be due to broadcasting too late (too slow), but it could also just be to aspects of consensus layer out-of-our-control. See https://eth2book.info/bellatrix/part2/incentives/rewards/#remarks.", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "red", - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 41 - }, - "id": 203, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(sum(increase(core_tracker_inclusion_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=~\"$duty\"}[$interval])) by (duty)) > 0 ", - "instant": false, - "interval": "$interval", - "legendFormat": "{{duty}}", - "range": true, - "refId": "A" - } - ], - "title": "๐Ÿ’ฅ Broadcasted duties never included on-chain (per $interval)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "yellow", - "value": 2 - }, - { - "color": "orange", - "value": 5 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "slots" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 47 - }, - "id": 184, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "sum(core_tracker_inclusion_delay{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", - "interval": "$interval", - "legendFormat": "Instant", - "range": true, - "refId": "A" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "max(avg_over_time(core_tracker_inclusion_delay{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval]))", - "hide": false, - "interval": "$interval", - "legendFormat": "Avg", - "range": true, - "refId": "B" - } - ], - "title": "Attestation Inclusion Delay (slots)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "The delay after the start of the slot when the duty was successfully broadcasted. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "orange", - "value": 6 - }, - { - "color": "red", - "value": 15 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 53 - }, - "id": 218, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "max(histogram_quantile(0.90, rate(core_bcast_broadcast_delay_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval]))) ", - "hide": false, - "interval": "$interval", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Broadcast delay: $duty", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "orange", - "value": 6 - }, - { - "color": "red", - "value": 15 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 59 - }, - "id": 208, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "max(histogram_quantile(0.90, increase(core_consensus_duration_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval]))) by (timer)", - "hide": false, - "interval": "$interval", - "legendFormat": "{{timer}}", - "range": true, - "refId": "A" - } - ], - "title": "Consensus duration: $duty", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "yellow", - "value": 2 - }, - { - "color": "orange", - "value": 4 - }, - { - "color": "red", - "value": 8 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 65 - }, - "id": 182, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "max(core_consensus_decided_rounds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}) by (timer)", - "hide": false, - "interval": "$interval", - "legendFormat": "{{timer}}", - "range": true, - "refId": "A" - } - ], - "title": "Consensus Rounds: $duty", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Number of consensus timeouts per time window.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "mappings": [], - "noValue": "No timeouts ๐ŸŽ‰", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 71 - }, - "id": 189, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "sum(increase(core_consensus_timeout_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval])) by (timer) > 0", - "hide": false, - "interval": "$interval", - "legendFormat": "{{.duty}}", - "range": true, - "refId": "B" - } - ], - "title": "๐Ÿšจ Consensus Timeouts (per $interval)", - "type": "timeseries" - } - ], - "title": "Duties", - "type": "row" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(core_scheduler_validator_balance_gwei{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} / 1000000000) by (pubkey_full,pubkey, cluster_network)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "{{pubkey}}", + "range": false, + "refId": "B" }, { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 123, - "panels": [ - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 34 - }, - "id": 127, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "max(go_memstats_alloc_bytes{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", - "interval": "$interval", - "legendFormat": "Allocated Memory", - "range": true, - "refId": "A" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "max(go_memstats_heap_inuse_bytes{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", - "hide": true, - "legendFormat": "Inuse Heap", - "range": true, - "refId": "B" - } - ], - "title": "Memory", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 34 - }, - "id": 129, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.4", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "max(irate(process_cpu_seconds_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval]))", - "interval": "$interval", - "intervalFactor": 2, - "legendFormat": "CPU", - "metric": "go_gc_duration_seconds", - "range": true, - "refId": "A", - "step": 4 - } - ], - "title": "CPU", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "File Descriptors" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 41 - }, - "id": 125, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "max(go_goroutines{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", - "interval": "$interval", - "legendFormat": "Go Routines", - "range": true, - "refId": "A" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "max(process_open_fds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", - "hide": false, - "interval": "$interval", - "legendFormat": "File Descriptors", - "range": true, - "refId": "B" - } - ], - "title": "Go Routines and File Descriptors", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Total time spent on garbage collection per interval", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 41 - }, - "id": 130, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.4", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "max(increase(go_gc_duration_seconds_sum{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) ", - "interval": "$interval", - "intervalFactor": 2, - "legendFormat": "__auto", - "metric": "go_gc_duration_seconds", - "range": true, - "refId": "A", - "step": 4 - } - ], - "title": "Garbage Collection Duration (per $interval)", - "type": "timeseries" - } - ], - "title": "Process", - "type": "row" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(core_scheduler_validator_status{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 1) by (pubkey, status)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" }, { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 141, - "panels": [ - { - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 191, - "options": { - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "frameIndex": 0, - "showHeader": false - }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(app_beacon_node_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (version)\n)\n ", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "__auto", - "range": false, - "refId": "B" - } - ], - "transformations": [ - { - "id": "labelsToFields", - "options": { - "keepLabels": [ - "version" - ], - "mode": "rows" - } - } - ], - "type": "table" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Upstream beacon node API request rate per second by endpoint", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "validators_by_pub_key", - "validators" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 29 - }, - "id": 142, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "max(rate(app_eth2_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0 ", - "interval": "$interval", - "legendFormat": "{{endpoint}}", - "range": true, - "refId": "A" - } - ], - "title": "Beacon Node API requests rate", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Upstream beacon node API request latency (90th percentile) by endpoint by timewindow", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 29 - }, - "id": 138, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.90, sum(rate(app_eth2_latency_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (le,endpoint)) ", - "interval": "$interval", - "legendFormat": "{{endpoint}}", - "range": true, - "refId": "A" - } - ], - "title": "Beacon Node API request latency (90%)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Number of errors per endpoint per time window", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 36 - }, - "id": 144, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "max(increase(app_eth2_errors_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0 ", - "interval": "$interval", - "legendFormat": "{{endpoint}}", - "range": true, - "refId": "A" - } - ], - "title": "๐Ÿšจ Beacon Node API error count (per $interval)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Beacon node score over time", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "#ffffff", - "mode": "fixed" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 32, - "gradientMode": "hue", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - }, - { - "color": "orange", - "value": 0.5 - }, - { - "color": "green", - "value": 0.8 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 36 - }, - "id": 145, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "clamp_min((clamp(app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 1 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 4 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 5 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 6, 1, 1) OR on() vector(0))\n*\non()\n(\n (\n0.5 * (1.0 - 10*(sum(increase(app_eth2_errors_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) / (sum(increase(app_eth2_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval]))))) OR on() vector(0.5)\n )\n +\n (\n 0.5 * (1.0 - clamp_max(histogram_quantile(0.99, sum(rate(app_eth2_latency_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (le)),1))\n )\n), 0)", - "interval": "$interval", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Beacon Node Score", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Number of consensus layer P2P peers the beacon node is connected to", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "#fffefe", - "mode": "fixed" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - }, - { - "color": "#EAB839", - "value": 20 - }, - { - "color": "green", - "value": 40 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 43 - }, - "id": 147, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "app_beacon_node_peers{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "interval": "$interval", - "legendFormat": "Peer count", - "range": true, - "refId": "A" - } - ], - "title": "Beacon node peer count", - "type": "timeseries" - } - ], - "title": "Upstream Beacon Node", - "type": "row" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(core_parsigdb_exit_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (pubkey) ", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "C" + } + ], + "title": "Validators", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "pubkey", + "mode": "outer" + } }, { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 19 - }, - "id": 132, - "panels": [ - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 158 - }, - "id": 136, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "max(rate(core_validatorapi_request_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0", - "interval": "$interval", - "legendFormat": "{{endpoint}}", - "range": true, - "refId": "A" - } - ], - "title": "Validator API requests rate", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 158 - }, - "id": 143, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.90, sum(rate(core_validatorapi_request_latency_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (le,endpoint)) ", - "interval": "$interval", - "legendFormat": "{{endpoint}}", - "range": true, - "refId": "A" - } - ], - "title": "Validator API request latency (90%)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Number of errors per endpoint per time window", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 165 - }, - "id": 139, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "max(increase(core_validatorapi_request_error_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0 ", - "interval": "$interval", - "legendFormat": "{{endpoint}}", - "range": true, - "refId": "A" - } - ], - "title": "๐Ÿšจ Validator API error count (per $interval)", - "type": "timeseries" - } - ], - "title": "Downstream Validator", - "type": "row" + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Value #A": true, + "Value #B": false, + "cluster_network": false, + "job": true, + "pubkey_full": false + }, + "indexByName": { + "Time 1": 2, + "Time 2": 7, + "Time 3": 9, + "Value #A": 8, + "Value #B": 6, + "Value #C": 4, + "cluster_network": 3, + "pubkey": 0, + "pubkey_full": 1, + "status": 5 + }, + "renameByName": { + "Value #A": "", + "Value #B": "Balance", + "Value #C": "โ—๏ธPeer Exits", + "cluster_network": "__hide__", + "pubkey": " Public Key", + "pubkey_full": "__hide__", + "status": "Status" + } + } + } + ], + "type": "table" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Warning (w) and error (e) logs grouped by topic. \n\nOccasional warns and errors are normal, but a significant increase indicate problems that require operators to investigate by looking at the actual logs entries.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Show in log explorer", + "url": "/explore?orgId=1&left=%7B%22datasource%22:%22loki%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcompose_service%3D%5C%22$job%5C%22%7D%20%7C%3D%20%60${__field.labels.msg}%60%20%7C%20logfmt%20%7C%20line_format%20%60%7B%7B.pretty%7D%7D%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D" + } + ], + "mappings": [], + "min": 0, + "noValue": "No errors or warnings", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 100 + } + ] + } }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 13, + "y": 21 + }, + "id": 117, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "none", + "sort": "none" + } + }, + "targets": [ { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 149, - "panels": [ - { - "datasource": "${DS_PROMETHEUS}", - "description": "Type of connections to each peer in the cluster during the time window:\n - ๐ŸŸข Green: Direct connection (this is the preferred)\n - ๐ŸŸก Yellow: Indirect relay connection (this is ok if only temporary)\n - ๐Ÿ”ด Red: No connection to peer (this is a problem)\n\nSee https://docs.obol.tech/docs/charon/networking#external-p2p-network", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 0, - "spanNulls": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - }, - { - "color": "#EAB839", - "value": 1 - }, - { - "color": "green", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 21 - }, - "id": 195, - "options": { - "alignValue": "left", - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "mergeValues": true, - "rowHeight": 0.9, - "showValue": "never", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "(max(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer))\n+\n(1 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))\n+ \n(100 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer))", - "format": "time_series", - "interval": "$interval", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Peer Connections (๐ŸŸข=direct, ๐ŸŸก=relay, ๐Ÿ”ด=not connected)", - "type": "state-timeline" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "yellow", - "value": 0.15 - }, - { - "color": "orange", - "value": 0.3 - }, - { - "color": "red", - "value": 0.5 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 21 - }, - "id": 25, + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "sum(increase(app_log_error_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (topic) > 0 ", + "hide": false, + "interval": "$interval", + "legendFormat": "e:{{topic}}", + "range": true, + "refId": "B" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "sum(increase(app_log_warn_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (topic) > 0 ", + "hide": false, + "interval": "$interval", + "legendFormat": "w:{{topic}}", + "range": true, + "refId": "A" + } + ], + "title": "Warning(w) and Error(e) logs (per $interval)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "- **Peer**: The peer's name (inferred from their *charon-enr-private-key*)\n- **You**: โญ๏ธ is this local charon node\n- **Connected**: Whether you are currently connected to this peer.\n- **Direct**: Whether the connection is *direct* (๐Ÿ‘) or *relay* (๐Ÿ‘Ž) \n- **Latency**: The time messages take to travel to/from the peer.\n- **Uptime**\" The duration since peer was restarted.\n- **ClockDiff**: Difference between local and peer's clock time. More than 2s is bad.\n- **๐ŸšงAttest**: Number of attestation duties missed in the time window.\n- **๐ŸšงPropose**: Number of block proposal duties missed in the time window.\n- **๐ŸšงOther**: Number of other duties missed in the time window.\n\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "minWidth": 0, + "width": 90 + }, + "decimals": 0, + "mappings": [], + "noValue": "--", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Latency" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "mappings", + "value": [ + { "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (le,peer)) ", - "interval": "$interval", - "legendFormat": "{{peer}}", - "range": true, - "refId": "A" - } - ], - "title": "Peer ping latency (90%)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Time windows that contain ping errors per peer", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 0, - "spanNulls": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 27 - }, - "id": 166, + "from": 0, + "result": { + "color": "green", + "index": 0 + }, + "to": 150 + }, + "type": "range" + }, + { "options": { - "alignValue": "left", - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "mergeValues": true, - "rowHeight": 0.9, - "showValue": "never", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "sum(increase(p2p_ping_error_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (peer) > 0", - "interval": "$interval", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Ping Errors", - "type": "state-timeline" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 27 - }, - "id": 160, + "from": 150, + "result": { + "color": "yellow", + "index": 1 + }, + "to": 300 + }, + "type": "range" + }, + { "options": { - "calculate": false, - "cellGap": 5, - "cellValues": { - "unit": "s" - }, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "max": 0.5, - "min": 0, - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdYlGn", - "steps": 82 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "show": true, - "yHistogram": false - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false - } - }, - "pluginVersion": "9.5.2", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (le,peer))", - "interval": "", - "legendFormat": "{{peer}}", - "range": true, - "refId": "A" - } - ], - "title": "Peer ping latency (90%) (heatmap)", - "type": "heatmap" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 158, + "from": 300, + "result": { + "color": "orange", + "index": 2 + }, + "to": 500 + }, + "type": "range" + }, + { "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "max(rate(p2p_ping_latency_secs_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (peer) ", - "interval": "$interval", - "legendFormat": "{{peer}}", - "range": true, - "refId": "A" - } - ], - "title": "Ping Rate", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Number of new libp2p connections per peer per time window", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 33 - }, - "id": 163, + "from": 500, + "result": { + "color": "red", + "index": 3 + }, + "to": 100000 + }, + "type": "range" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "lcd", + "type": "gauge" + } + }, + { + "id": "custom.width", + "value": 200 + }, + { + "id": "max", + "value": 700 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "You" + }, + "properties": [ + { + "id": "noValue", + "value": "โญ" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "auto" + } + }, + { + "id": "mappings", + "value": [ + { "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "max(increase(p2p_peer_connection_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (peer) > 0 ", - "interval": "$interval", - "legendFormat": "{{peer}}", - "range": true, - "refId": "A" - } - ], - "title": "New P2P Connections (per $interval)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "area" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - }, - { - "color": "yellow", - "value": -0.5 - }, - { - "color": "green", - "value": -0.2 - }, - { - "color": "#EAB839", - "value": 0.2 - }, - { - "color": "red", - "value": 0.5 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 39 - }, - "id": 151, + "0": { + "index": 0, + "text": "--" + }, + "1": { + "index": 1, + "text": "--" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.width", + "value": 60 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Connected" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "max(app_peerinfo_clock_offset_seconds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=~\"$cluster_peer\"}) by (peer) ", - "interval": "$interval", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Clock Offset", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Avg Rate of P2P network bytes sent plus received per peer (bytes/second) ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue" - } - ] - }, - "unit": "Bps" - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "B" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "decbytes" - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 39 - }, - "id": 153, + "0": { + "index": 0, + "text": " โŒ" + }, + "1": { + "index": 1, + "text": "โœ…" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ClockDiff" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + }, + { + "id": "mappings", + "value": [ + { "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "0": { + "index": 0, + "text": "๐Ÿ†—" + } }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "sum(rate(p2p_peer_network_receive_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer) + sum(rate(p2p_peer_network_sent_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer)", - "interval": "$interval", - "legendFormat": "{{peer}}", - "range": true, - "refId": "A" - } - ], - "title": "P2P Network Avg Rate Rx+Tx", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 0, - "spanNulls": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 45 - }, - "id": 156, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Direct" + }, + "properties": [ + { + "id": "custom.width", + "value": 55 + }, + { + "id": "mappings", + "value": [ + { "options": { - "alignValue": "left", - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "mergeValues": true, - "rowHeight": 0.9, - "showValue": "never", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "sum(p2p_relay_connections{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer) > 0 ", - "interval": "$interval", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Connected Relays", - "type": "state-timeline" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Total P2P network bytes sent plus received per peer per time window (bytes per interval)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue" - } - ] - }, - "unit": "decbytes" - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "B" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "decbytes" - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 45 - }, - "id": 154, + "0": { + "index": 0, + "text": "--" + } + }, + "type": "value" + }, + { "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "sum(increase(p2p_peer_network_receive_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer) + sum(increase(p2p_peer_network_sent_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer)", - "interval": "$interval", - "legendFormat": "{{peer}}", - "range": true, - "refId": "A" - } - ], - "title": "P2P Network Bytes Rx+Tx (per $interval)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "The number of active libp2p streams by peer or relay. Note this can be further grouped by direction and protocol.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 64 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 51 - }, - "id": 226, + "from": 1, + "result": { + "index": 1, + "text": "๐Ÿ‘Ž" + }, + "to": 9 + }, + "type": "range" + }, + { "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "sum(p2p_peer_streams{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", - "interval": "$interval", - "legendFormat": "{{peer}}", - "range": true, - "refId": "A" - } - ], - "title": "Active libp2p streams", - "type": "timeseries" - } - ], - "title": "P2P Networking", - "type": "row" - } - ], - "refresh": "", - "revision": 1, - "schemaVersion": 38, - "style": "dark", - "tags": [ - "cluster-labels", - "charon" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "hide": 0, - "includeAll": false, - "label": "datasource", - "multi": false, - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "selected": false, - "text": "None", - "value": "" - }, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(app_peer_name, cluster_name)", - "description": "", - "hide": 0, - "includeAll": false, - "label": "Cluster Name", - "multi": false, - "name": "cluster_name", - "options": [], - "query": { - "query": "label_values(app_peer_name, cluster_name)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" + "from": 10, + "result": { + "index": 2, + "text": "๐Ÿ‘" + }, + "to": 1000 + }, + "type": "range" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Peer" }, - { - "current": { - "selected": false, - "text": "8439eb3", - "value": "8439eb3" - }, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\"}, cluster_hash)", - "description": "", - "hide": 0, - "includeAll": false, - "label": "Cluster Hash", - "multi": false, - "name": "cluster_hash", - "options": [], - "query": { - "query": "label_values(app_peer_name{cluster_name=\"$cluster_name\"}, cluster_hash)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Uptime" }, - { - "current": { - "selected": false, - "text": "excited-group", - "value": "excited-group" - }, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\"}, cluster_peer)", - "description": "", - "hide": 0, - "includeAll": false, - "label": "Cluster Peer", - "multi": false, - "name": "cluster_peer", - "options": [], - "query": { - "query": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\"}, cluster_peer)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Index" }, - { - "current": { - "selected": false, - "text": "charon", - "value": "charon" - }, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"},job)", - "description": "Prometheus job (can be ignored)", - "hide": 2, - "includeAll": false, - "label": "--", - "multi": false, - "name": "job", - "options": [], - "query": { - "query": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"},job)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" + "properties": [ + { + "id": "custom.width", + "value": 30 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "๐Ÿšง.*" }, - { - "allValue": "", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\", cluster_peer=\"$cluster_peer\"}, peer)", - "description": "", - "hide": 2, - "includeAll": true, - "label": "P2P Peer", - "multi": false, - "name": "peer", - "options": [], - "query": { - "query": "label_values(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\", cluster_peer=\"$cluster_peer\"}, peer)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "auto": true, - "auto_count": 30, - "auto_min": "1m", - "current": { - "selected": false, - "text": "auto", - "value": "$__auto_interval_interval" - }, - "hide": 0, - "label": "Interval", - "name": "interval", - "options": [ - { - "selected": true, - "text": "auto", - "value": "$__auto_interval_interval" - }, - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "12h", - "value": "12h" - }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "thresholds" + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ { - "selected": false, - "text": "1d", - "value": "1d" + "color": "#585858" }, { - "selected": false, - "text": "7d", - "value": "7d" + "color": "super-light-green", + "value": 0.00001 }, { - "selected": false, - "text": "14d", - "value": "14d" + "color": "yellow", + "value": 0.01 }, { - "selected": false, - "text": "30d", - "value": "30d" + "color": "red", + "value": 0.1 } - ], - "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", - "queryValue": "", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" + ] + } + }, + { + "id": "noValue", + "value": "0%" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 194, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "You" + } + ] + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", + "format": "table", + "instant": true, + "range": false, + "refId": "A" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__rate_interval])) by (le,peer)) * 1000", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "B" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "", + "range": false, + "refId": "F" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(round(app_peerinfo_clock_offset_seconds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"},0.1)*1000) by (peer) ", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "J" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer)*10 + \nmax(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "", + "range": false, + "refId": "K" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer,version) ", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "L" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(app_peerinfo_git_commit{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer,git_hash) > 0", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "M" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "time() - max(app_peerinfo_start_time_secs{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} > 0) by (peer)", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "Q" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(app_peerinfo_index{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "R" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"attester\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"attester\"}[$__range])) by (peer) \n) > 0 ", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "C" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"proposer\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"proposer\"}[$__range])) by (peer) \n) > 0 ", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "D" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty!~\"(proposer|attester)\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty!~\"(proposer|attester)\"}[$__range])) by (peer) \n) > 0 ", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "E" + } + ], + "title": "Peer Connectivity and Missed Duties (๐Ÿšง)", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "peer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time 1": true, + "Time 10": true, + "Time 11": true, + "Time 12": true, + "Time 13": true, + "Time 14": true, + "Time 15": true, + "Time 16": true, + "Time 17": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "Time 6": true, + "Time 7": true, + "Time 8": true, + "Time 9": true, + "Value #F": false, + "Value #L": true, + "Value #M": true + }, + "indexByName": { + "Time 1": 10, + "Time 10": 29, + "Time 11": 30, + "Time 12": 31, + "Time 13": 32, + "Time 14": 33, + "Time 15": 34, + "Time 16": 35, + "Time 17": 36, + "Time 2": 12, + "Time 3": 13, + "Time 4": 14, + "Time 5": 15, + "Time 6": 16, + "Time 7": 17, + "Time 8": 18, + "Time 9": 28, + "Value #A": 3, + "Value #B": 5, + "Value #C": 20, + "Value #D": 21, + "Value #E": 22, + "Value #F": 2, + "Value #G": 23, + "Value #H": 24, + "Value #J": 6, + "Value #K": 4, + "Value #L": 11, + "Value #M": 19, + "Value #N": 25, + "Value #O": 27, + "Value #P": 26, + "Value #Q": 9, + "Value #R": 1, + "git_hash": 8, + "peer": 0, + "version": 7 + }, + "renameByName": { + "Value #A": "Connected", + "Value #B": "Latency", + "Value #C": "๐ŸšงAttest", + "Value #D": "๐ŸšงPropose", + "Value #E": "๐ŸšงOther", + "Value #F": "You", + "Value #G": "PrepareAgg", + "Value #H": "Aggregate", + "Value #I": "Exit", + "Value #J": "ClockDiff", + "Value #K": "Direct", + "Value #L": "", + "Value #N": "SyncMsg", + "Value #O": "SyncContrib", + "Value #P": "PrepareContrib", + "Value #Q": "Uptime", + "Value #R": "Index", + "git_hash": "GitCommit", + "peer": "Peer", + "version": "Version" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 178, + "panels": [], + "title": "Duties", + "type": "row" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Completed duties by type over time.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 100, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0 + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 92, + "options": { + "colWidth": 0.5, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "rowHeight": 0.43, + "showValue": "auto", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(core_bcast_broadcast_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) > 0", + "instant": false, + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "โœ… Completed duties by type ", + "type": "status-history" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Successful duty participation per peer (filtered by selected duty) over time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 81, + "options": { + "colWidth": 0.9, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(core_tracker_participation_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\",duty=~\"$duty\"}[$interval])) by (peer) > 0 ", + "interval": "$interval", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "โœ… Successful Peer Participation: Duty=$duty", + "type": "status-history" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Failed duties by type over time. Use data link to show logs.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 100, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0 + }, + "links": [ + { + "targetBlank": true, + "title": "Show in log explorer", + "url": "/explore?orgId=1&left=%7B%22datasource%22:%22loki%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcompose_service%3D%5C%22$job%5C%22%7D%20%7C%3D%20%60Duty%20failed%60%20%7C%3D%20%60${__field.labels.duty}%60%20%7C%20logfmt%20%7C%20line_format%20%60%7B%7B.pretty%7D%7D%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 91, + "options": { + "colWidth": 0.5, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "rowHeight": 0.43, + "showValue": "auto", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(core_tracker_failed_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) > 0", + "instant": false, + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "โŒ Failed duties by type (per $interval)", + "type": "status-history" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Missed duty participation per peer (filtered by selected duty) over time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 185, + "options": { + "colWidth": 0.9, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\",duty=~\"$duty\"}[$interval])) by (peer) > 0 ", + "interval": "$interval", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "๐Ÿšง Missed Peer Participation: Duty=$duty", + "type": "status-history" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Failed duties percentage by type over time", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 180, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(\n sum(increase(core_tracker_failed_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty)\n) \n/ \n(\n sum(increase(core_tracker_failed_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) \n + \n sum(increase(core_bcast_broadcast_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) \n) > 0 ", + "instant": false, + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "โŒ Failed duty percentage by type (per $interval)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Missed duty participation per duty (filtered by peer) over time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 186, + "options": { + "colWidth": 0.9, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "repeat": "peer", + "repeatDirection": "v", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=\"$peer\"}[$interval])) by (duty) > 0 ", + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "๐Ÿšง Missed Peer Participation: Peer=$peer", + "type": "status-history" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Number of failed duties by reason per time window. Note this is only available from v0.16.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "noValue": "No failures ๐ŸŽ‰", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 55 + }, + "id": 222, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "sum(increase(core_tracker_failed_duty_reasons_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval])) by (reason) > 0 ", + "hide": false, + "interval": "$interval", + "legendFormat": "{{.duty}}", + "range": true, + "refId": "A" + } + ], + "title": "๐Ÿšจ Duty failure reason: $duty (per $interval)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Which operators are having a growing number of relay registrations failures, indicating there may be a risk to an upcoming MEV block production opportunity. Be conscious that absence of an operator from this list may mean their MEV registration is working, or it may mean it is not enabled at all and thus not failing. ", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#ffffff", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "green", + "value": 0.1 + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "orange", + "value": 200 + }, + { + "color": "red", + "value": 500 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "none" + }, + { + "id": "color", + "value": { + "fixedColor": "#808080c4", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 61 + }, + "id": 242, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "max(increase(app_eth2_errors_total{endpoint=~\"submit_validator_registrations\", cluster_name=~\"$cluster_name\"}[$__range])) by (cluster_peer, cluster_name)", + "hide": false, + "legendFormat": "{{cluster_name}}: {{cluster_peer}}", + "range": true, + "refId": "A" + } + ], + "title": "โš ๏ธ MEV Relay Submit Registrations Failure Growth by Operator", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Duties that was successfully broadcast to the Beacon Node, but that was never included on-chain. This could be due to broadcasting too late (too slow), but it could also just be to aspects of consensus layer out-of-our-control. See https://eth2book.info/bellatrix/part2/incentives/rewards/#remarks.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 203, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(sum(increase(core_tracker_inclusion_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=~\"$duty\"}[$interval])) by (duty)) > 0 ", + "instant": false, + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "๐Ÿ’ฅ Broadcasted duties never included on-chain (per $interval)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 2 + }, + { + "color": "orange", + "value": 5 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "slots" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 73 + }, + "id": 184, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "sum(core_tracker_inclusion_delay{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "interval": "$interval", + "legendFormat": "Instant", + "range": true, + "refId": "A" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "max(avg_over_time(core_tracker_inclusion_delay{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval]))", + "hide": false, + "interval": "$interval", + "legendFormat": "Avg", + "range": true, + "refId": "B" + } + ], + "title": "Attestation Inclusion Delay (slots)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "The delay after the start of the slot when the duty was successfully broadcasted. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "orange", + "value": 6 + }, + { + "color": "red", + "value": 15 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 79 + }, + "id": 218, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "max(histogram_quantile(0.90, rate(core_bcast_broadcast_delay_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval]))) ", + "hide": false, + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Broadcast delay: $duty", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "orange", + "value": 6 + }, + { + "color": "red", + "value": 15 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 85 + }, + "id": 208, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "max(histogram_quantile(0.90, increase(core_consensus_duration_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval]))) by (timer)", + "hide": false, + "interval": "$interval", + "legendFormat": "{{timer}}", + "range": true, + "refId": "A" + } + ], + "title": "Consensus duration: $duty", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 2 + }, + { + "color": "orange", + "value": 4 + }, + { + "color": "red", + "value": 8 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 91 + }, + "id": 182, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "max(core_consensus_decided_rounds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}) by (timer)", + "hide": false, + "interval": "$interval", + "legendFormat": "{{timer}}", + "range": true, + "refId": "A" + } + ], + "title": "Consensus Rounds: $duty", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Number of consensus timeouts per time window.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "noValue": "No timeouts ๐ŸŽ‰", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 97 + }, + "id": 189, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "sum(increase(core_consensus_timeout_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval])) by (timer) > 0", + "hide": false, + "interval": "$interval", + "legendFormat": "{{.duty}}", + "range": true, + "refId": "B" + } + ], + "title": "๐Ÿšจ Consensus Timeouts (per $interval)", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 103 + }, + "id": 123, + "panels": [ + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 50 + }, + "id": 127, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "max(go_memstats_alloc_bytes{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "interval": "$interval", + "legendFormat": "Allocated Memory", + "range": true, + "refId": "A" }, { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "max(go_memstats_heap_inuse_bytes{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "hide": true, + "legendFormat": "Inuse Heap", + "range": true, + "refId": "B" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "hide": 0, - "includeAll": true, - "label": "Duty", - "multi": false, - "name": "duty", - "options": [ - { - "selected": true, - "text": "All", - "value": "$__all" - }, - { - "selected": false, - "text": "attester", - "value": "attester" - }, - { - "selected": false, - "text": "prepare_aggregator", - "value": "prepare_aggregator" - }, - { - "selected": false, - "text": "aggregator", - "value": "aggregator" - }, - { - "selected": false, - "text": "sync_message", - "value": "sync_message" - }, - { - "selected": false, - "text": "prepare_sync_contribution", - "value": "prepare_sync_contribution" - }, - { - "selected": false, - "text": "sync_contribution", - "value": "sync_contribution" - }, - { - "selected": false, - "text": "proposer", - "value": "proposer" - }, - { - "selected": false, - "text": "randao", - "value": "randao" - }, - { - "selected": false, - "text": "exit", - "value": "exit" - }, - { - "selected": false, - "text": "builder_proposer", - "value": "builder_proposer" - }, - { - "selected": false, - "text": "builder_registration", - "value": "builder_registration" - }, - { - "selected": false, - "text": "info_sync", - "value": "info_sync" - } + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 50 + }, + "id": 129, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.4", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "max(irate(process_cpu_seconds_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval]))", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "CPU", + "metric": "go_gc_duration_seconds", + "range": true, + "refId": "A", + "step": 4 + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "File Descriptors" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 57 + }, + "id": 125, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "max(go_goroutines{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "interval": "$interval", + "legendFormat": "Go Routines", + "range": true, + "refId": "A" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "max(process_open_fds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "hide": false, + "interval": "$interval", + "legendFormat": "File Descriptors", + "range": true, + "refId": "B" + } + ], + "title": "Go Routines and File Descriptors", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Total time spent on garbage collection per interval", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 130, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.4", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "max(increase(go_gc_duration_seconds_sum{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) ", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "__auto", + "metric": "go_gc_duration_seconds", + "range": true, + "refId": "A", + "step": 4 + } + ], + "title": "Garbage Collection Duration (per $interval)", + "type": "timeseries" + } + ], + "title": "Process", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 104 + }, + "id": 141, + "panels": [], + "title": "Upstream Beacon Node", + "type": "row" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 105 + }, + "id": 191, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": false + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(\n sum(app_beacon_node_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (version)\n)\n ", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "__auto", + "range": false, + "refId": "B" + } + ], + "transformations": [ + { + "id": "labelsToFields", + "options": { + "keepLabels": [ + "version" + ], + "mode": "rows" + } + } + ], + "type": "table" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Upstream beacon node API request rate per second by endpoint", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 107 + }, + "id": 142, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "max(rate(app_eth2_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0 ", + "interval": "$interval", + "legendFormat": "{{endpoint}}", + "range": true, + "refId": "A" + } + ], + "title": "Beacon Node API requests rate", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Upstream beacon node API request latency (90th percentile) by endpoint by timewindow", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 107 + }, + "id": 138, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.90, sum(rate(app_eth2_latency_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (le,endpoint)) ", + "interval": "$interval", + "legendFormat": "{{endpoint}}", + "range": true, + "refId": "A" + } + ], + "title": "Beacon Node API request latency (90%)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Number of errors per endpoint per time window", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "submit_beacon_block" ], - "query": "attester,prepare_aggregator,aggregator,sync_message,prepare_sync_contribution,sync_contribution,proposer,randao,exit,builder_proposer,builder_registration,info_sync", - "queryValue": "", - "skipUrlSync": false, - "type": "custom" + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 114 + }, + "id": 144, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "max(increase(app_eth2_errors_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0 ", + "interval": "$interval", + "legendFormat": "{{endpoint}}", + "range": true, + "refId": "A" + } + ], + "title": "๐Ÿšจ Beacon Node API error count (per $interval)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Beacon node score over time", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#ffffff", + "mode": "fixed" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "green", + "value": 0.8 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 114 + }, + "id": 145, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "clamp_min((clamp(app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 1 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 4 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 5 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 6, 1, 1) OR on() vector(0))\n*\n(\n (\n0.5 * (1.0 - 10*(sum(increase(app_eth2_errors_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) / (sum(increase(app_eth2_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval]))))) OR on() vector(0.5)\n )\n +\n (\n 0.5 * (1.0 - clamp_max(histogram_quantile(0.99, sum(rate(app_eth2_latency_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (le)),1))\n )\n), 0)", + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Beacon Node Score", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Number of consensus layer P2P peers the beacon node is connected to", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#fffefe", + "mode": "fixed" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "#EAB839", + "value": 20 + }, + { + "color": "green", + "value": 40 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 121 + }, + "id": 147, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "app_beacon_node_peers{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "interval": "$interval", + "legendFormat": "Peer count", + "range": true, + "refId": "A" + } + ], + "title": "Beacon node peer count", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 128 + }, + "id": 132, + "panels": [ + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 137 + }, + "id": 136, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "max(rate(core_validatorapi_request_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0", + "interval": "$interval", + "legendFormat": "{{endpoint}}", + "range": true, + "refId": "A" + } + ], + "title": "Validator API requests rate", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 137 + }, + "id": 143, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ { - "current": { - "selected": false, - "text": "goerli", - "value": "goerli" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.90, sum(rate(core_validatorapi_request_latency_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (le,endpoint)) ", + "interval": "$interval", + "legendFormat": "{{endpoint}}", + "range": true, + "refId": "A" + } + ], + "title": "Validator API request latency (90%)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Number of errors per endpoint per time window", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"},cluster_network)", - "description": "", - "hide": 2, - "includeAll": false, - "label": "Cluster Network", - "multi": false, - "name": "cluster_network", - "options": [], - "query": { - "query": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"},cluster_network)", - "refId": "StandardVariableQuery" + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 144 + }, + "id": 139, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "max(increase(core_validatorapi_request_error_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0 ", + "interval": "$interval", + "legendFormat": "{{endpoint}}", + "range": true, + "refId": "A" } + ], + "title": "๐Ÿšจ Validator API error count (per $interval)", + "type": "timeseries" + } + ], + "title": "Downstream Validator", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 129 + }, + "id": 149, + "panels": [], + "title": "P2P Networking", + "type": "row" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Type of connections to each peer in the cluster during the time window:\n - ๐ŸŸข Green: Direct connection (this is the preferred)\n - ๐ŸŸก Yellow: Indirect relay connection (this is ok if only temporary)\n - ๐Ÿ”ด Red: No connection to peer (this is a problem)\n\nSee https://docs.obol.tech/docs/charon/networking#external-p2p-network", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 70, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "green", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 130 + }, + "id": 195, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "(max(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer))\n+\n(1 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))\n+ \n(100 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer))", + "format": "time_series", + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Peer Connections (๐ŸŸข=direct, ๐ŸŸก=relay, ๐Ÿ”ด=not connected)", + "type": "state-timeline" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.15 + }, + { + "color": "orange", + "value": 0.3 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 130 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (le,peer)) ", + "interval": "$interval", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "Peer ping latency (90%)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Time windows that contain ping errors per peer", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 70, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 136 + }, + "id": 166, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "sum(increase(p2p_ping_error_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (peer) > 0", + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Ping Errors", + "type": "state-timeline" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 136 + }, + "id": 160, + "options": { + "calculate": false, + "cellGap": 5, + "cellValues": { + "unit": "s" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "max": 0.5, + "min": 0, + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 82 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (le,peer))", + "interval": "", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "Peer ping latency (90%) (heatmap)", + "type": "heatmap" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 142 + }, + "id": 158, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "max(rate(p2p_ping_latency_secs_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (peer) ", + "interval": "$interval", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "Ping Rate", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Number of new libp2p connections per peer per time window", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 142 + }, + "id": 163, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "max(increase(p2p_peer_connection_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (peer) > 0 ", + "interval": "$interval", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "New P2P Connections (per $interval)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "yellow", + "value": -0.5 + }, + { + "color": "green", + "value": -0.2 + }, + { + "color": "#EAB839", + "value": 0.2 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 148 + }, + "id": 151, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "max(app_peerinfo_clock_offset_seconds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=~\"$cluster_peer\"}) by (peer) ", + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Clock Offset", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Avg Rate of P2P network bytes sent plus received per peer (bytes/second) ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "decbytes" + } + ] + } ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 148 + }, + "id": 153, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "sum(rate(p2p_peer_network_receive_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer) + sum(rate(p2p_peer_network_sent_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer)", + "interval": "$interval", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "P2P Network Avg Rate Rx+Tx", + "type": "timeseries" }, - "time": { - "from": "now-1h", - "to": "now" + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 70, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 154 + }, + "id": 156, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "sum(p2p_relay_connections{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer) > 0 ", + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Connected Relays", + "type": "state-timeline" }, - "timepicker": {}, - "timezone": "", - "title": "Charon Overview", - "uid": "charon_overview_dashboard", - "version": 1, - "weekStart": "" + { + "datasource": "${DS_PROMETHEUS}", + "description": "Total P2P network bytes sent plus received per peer per time window (bytes per interval)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "decbytes" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 154 + }, + "id": 154, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "sum(increase(p2p_peer_network_receive_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer) + sum(increase(p2p_peer_network_sent_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer)", + "interval": "$interval", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "P2P Network Bytes Rx+Tx (per $interval)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "The number of active libp2p streams by peer or relay. Note this can be further grouped by direction and protocol.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 64 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 160 + }, + "id": 226, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "sum(p2p_peer_streams{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", + "interval": "$interval", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "Active libp2p streams", + "type": "timeseries" + } + ], + "refresh": "1m", + "revision": 1, + "schemaVersion": 39, + "tags": [ + "cluster-labels", + "charon" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "description": "", + "hide": 0, + "includeAll": true, + "label": "Cluster Network", + "multi": true, + "name": "cluster_network", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "mainnet", + "value": "mainnet" + }, + { + "selected": false, + "text": "goerli", + "value": "goerli" + }, + { + "selected": false, + "text": "holesky", + "value": "holesky" + }, + { + "selected": false, + "text": "sepolia", + "value": "sepolia" + }, + { + "selected": false, + "text": "gnosis", + "value": "gnosis" + } + ], + "query": "mainnet,goerli,holesky,sepolia,gnosis", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "test-cluster", + "value": "test-cluster" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(app_peer_name{cluster_network=~\"$cluster_network\"},cluster_name)", + "description": "", + "hide": 0, + "includeAll": false, + "label": "Cluster Name", + "multi": false, + "name": "cluster_name", + "options": [], + "query": { + "query": "label_values(app_peer_name{cluster_network=~\"$cluster_network\"},cluster_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "41eb9d9", + "value": "41eb9d9" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\"}, cluster_hash)", + "description": "", + "hide": 0, + "includeAll": false, + "label": "Cluster Hash", + "multi": false, + "name": "cluster_hash", + "options": [], + "query": { + "query": "label_values(app_peer_name{cluster_name=\"$cluster_name\"}, cluster_hash)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "calm-game", + "value": "calm-game" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\"}, cluster_peer)", + "description": "", + "hide": 0, + "includeAll": false, + "label": "Cluster Peer", + "multi": false, + "name": "cluster_peer", + "options": [], + "query": { + "query": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\"}, cluster_peer)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "charon", + "value": "charon" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"},job)", + "description": "Prometheus job (can be ignored)", + "hide": 2, + "includeAll": false, + "label": "--", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"},job)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\", cluster_peer=\"$cluster_peer\"}, peer)", + "description": "", + "hide": 2, + "includeAll": true, + "label": "P2P Peer", + "multi": false, + "name": "peer", + "options": [], + "query": { + "query": "label_values(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\", cluster_peer=\"$cluster_peer\"}, peer)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "auto": true, + "auto_count": 30, + "auto_min": "1m", + "current": { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "includeAll": true, + "label": "Duty", + "multi": false, + "name": "duty", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "attester", + "value": "attester" + }, + { + "selected": false, + "text": "prepare_aggregator", + "value": "prepare_aggregator" + }, + { + "selected": false, + "text": "aggregator", + "value": "aggregator" + }, + { + "selected": false, + "text": "sync_message", + "value": "sync_message" + }, + { + "selected": false, + "text": "prepare_sync_contribution", + "value": "prepare_sync_contribution" + }, + { + "selected": false, + "text": "sync_contribution", + "value": "sync_contribution" + }, + { + "selected": false, + "text": "proposer", + "value": "proposer" + }, + { + "selected": false, + "text": "randao", + "value": "randao" + }, + { + "selected": false, + "text": "exit", + "value": "exit" + }, + { + "selected": false, + "text": "builder_proposer", + "value": "builder_proposer" + }, + { + "selected": false, + "text": "builder_registration", + "value": "builder_registration" + }, + { + "selected": false, + "text": "info_sync", + "value": "info_sync" + } + ], + "query": "attester,prepare_aggregator,aggregator,sync_message,prepare_sync_contribution,sync_contribution,proposer,randao,exit,builder_proposer,builder_registration,info_sync", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Charon Overview", + "uid": "d6qujIJVk", + "version": 3, + "weekStart": "" } \ No newline at end of file diff --git a/controls/roles/update-changes/molecule/20-rc23/converge.yml b/controls/roles/update-changes/molecule/20-rc23/converge.yml deleted file mode 100644 index 3f84cb4a2..000000000 --- a/controls/roles/update-changes/molecule/20-rc23/converge.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: Converge - hosts: all - vars_files: - - ../../../../defaults/stereum_defaults.yaml - - tasks: - - name: "Include update-changes" - include_role: - name: "update-changes" diff --git a/controls/roles/update-changes/molecule/20-rc23/molecule.yml b/controls/roles/update-changes/molecule/20-rc23/molecule.yml deleted file mode 100644 index dfc1cfe42..000000000 --- a/controls/roles/update-changes/molecule/20-rc23/molecule.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -#dependency: -# name: galaxy -driver: - name: docker -platforms: - - name: "update-changes--20-rc23--ubuntu-22.04" - image: ubuntu:jammy - # - name: "configure-updates--default--centos-8" - # image: geerlingguy/docker-centos8-ansible -provisioner: - name: ansible - env: - ANSIBLE_PIPELINING: "True" -lint: | - set -e - yamllint . - ansible-lint . -scenario: - test_sequence: - - destroy - - create - - prepare - - converge - - idempotence - - verify - - destroy diff --git a/controls/roles/update-changes/molecule/20-rc23/playbook.yml b/controls/roles/update-changes/molecule/20-rc23/playbook.yml deleted file mode 100644 index 710c596cf..000000000 --- a/controls/roles/update-changes/molecule/20-rc23/playbook.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Converge - hosts: all - tasks: - - name: "Include update-changes" - include_role: - name: "update-changes" diff --git a/controls/roles/update-changes/molecule/20-rc23/prepare.yml b/controls/roles/update-changes/molecule/20-rc23/prepare.yml deleted file mode 100644 index cea93e6c8..000000000 --- a/controls/roles/update-changes/molecule/20-rc23/prepare.yml +++ /dev/null @@ -1,63 +0,0 @@ ---- -- name: Prepare - hosts: all - tasks: - - name: Make sure Stereum's config path exists - file: - path: "/etc/stereum/services" - state: directory - owner: "root" - group: "root" - mode: 0644 - become: yes - - - name: Create Nethermind service config - copy: - dest: "/etc/stereum/services/1d5a0a1b-feff-a954-77e8-9a19adef072c.yaml" - owner: "root" - group: "root" - mode: 0644 - content: | - service: NethermindService - id: 1d5a0a1b-feff-a954-77e8-9a19adef072c - configVersion: 1 - command: - - --config=goerli - - --log=info - - --datadir=/opt/app/data - - --Network.DiscoveryPort=30303 - - --Network.P2PPort=30303 - - --Merge.Enabled=true - - --JsonRpc.Enabled=true - - --JsonRpc.JwtSecretFile=/engine.jwt - - --JsonRpc.Host=0.0.0.0 - - --JsonRpc.EngineHost=0.0.0.0 - - --Init.WebSocketsEnabled=true - - --JsonRpc.WebSocketsPort=8546 - - --JsonRpc.EnabledModules=[web3,eth,subscribe,net] - - --JsonRpc.AdditionalRpcUrls=http://0.0.0.0:8551|http;ws|engine;eth;subscribe - - --Metrics.Enabled=true - - --Metrics.ExposePort=6060 - - --HealthChecks.Enabled=true - - --Pruning.Mode=Hybrid - entrypoint: - - ./Nethermind.Runner - env: {} - image: nethermind/nethermind:1.18.0 - ports: - - 0.0.0.0:30303:30303/tcp - - 0.0.0.0:30303:30303/udp - - 127.0.0.1:8545:8545/tcp - - 0.0.0.0:8546:8546/tcp - volumes: - - /opt/stereum/nethermind-1d5a0a1b-feff-a954-77e8-9a19adef072c/data:/opt/app/data - - /opt/stereum/nethermind-1d5a0a1b-feff-a954-77e8-9a19adef072c/engine.jwt:/engine.jwt - user: root - autoupdate: true - network: goerli - dependencies: - executionClients: [] - consensusClients: [] - mevboost: [] - become: yes -# EOF diff --git a/controls/roles/update-changes/molecule/20-rc23/verify.yml b/controls/roles/update-changes/molecule/20-rc23/verify.yml deleted file mode 100644 index ce230723d..000000000 --- a/controls/roles/update-changes/molecule/20-rc23/verify.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -- name: Verify - hosts: all - gather_facts: false - tasks: - # Nethermind - - name: Read NethermindService file - slurp: - src: "/etc/stereum/services/1d5a0a1b-feff-a954-77e8-9a19adef072c.yaml" - register: nethermind_service_configuration_raw - - - name: Parse NethermindService configuration - set_fact: - nethermind_service_configuration: "{{ nethermind_service_configuration_raw['content'] | b64decode | from_yaml }}" - - - debug: - msg: "{{ nethermind_service_configuration }}" - - - assert: - that: - - (nethermind_service_configuration.entrypoint | select('match', "./Nethermind.Runner") | length) == 0 - - (nethermind_service_configuration.entrypoint | select('match', "./nethermind") | length) == 1 -# EOF diff --git a/controls/roles/update-changes/molecule/202/converge.yml b/controls/roles/update-changes/molecule/202/converge.yml deleted file mode 100644 index 3f84cb4a2..000000000 --- a/controls/roles/update-changes/molecule/202/converge.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: Converge - hosts: all - vars_files: - - ../../../../defaults/stereum_defaults.yaml - - tasks: - - name: "Include update-changes" - include_role: - name: "update-changes" diff --git a/controls/roles/update-changes/molecule/202/molecule.yml b/controls/roles/update-changes/molecule/202/molecule.yml deleted file mode 100644 index 92ae4ff03..000000000 --- a/controls/roles/update-changes/molecule/202/molecule.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -#dependency: -# name: galaxy -driver: - name: docker -platforms: - - name: "update-changes--2.0.2--ubuntu-23.04" - image: ubuntu:lunar - # - name: "configure-updates--default--centos-8" - # image: geerlingguy/docker-centos8-ansible -provisioner: - name: ansible - env: - ANSIBLE_PIPELINING: "True" -lint: | - set -e - yamllint . - ansible-lint . -scenario: - test_sequence: - - destroy - - create - - prepare - - converge - - idempotence - - verify - - destroy diff --git a/controls/roles/update-changes/molecule/202/playbook.yml b/controls/roles/update-changes/molecule/202/playbook.yml deleted file mode 100644 index 710c596cf..000000000 --- a/controls/roles/update-changes/molecule/202/playbook.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Converge - hosts: all - tasks: - - name: "Include update-changes" - include_role: - name: "update-changes" diff --git a/controls/roles/update-changes/molecule/202/prepare.yml b/controls/roles/update-changes/molecule/202/prepare.yml deleted file mode 100644 index 4ce4067f2..000000000 --- a/controls/roles/update-changes/molecule/202/prepare.yml +++ /dev/null @@ -1,116 +0,0 @@ ---- -- name: Prepare - hosts: all - tasks: - - name: Make sure Stereum's config path exists - file: - path: "/etc/stereum/services" - state: directory - owner: "root" - group: "root" - mode: 0644 - become: yes - - - name: Create Erigon service config - copy: - dest: "/etc/stereum/services/c3c822b5-5b0c-b779-8a04-fae3563ccfa1.yaml" - owner: "root" - group: "root" - mode: 0644 - content: | - service: ErigonService - id: c3c822b5-5b0c-b779-8a04-fae3563ccfa1 - configVersion: 1 - command: - - erigon - - --chain=holesky - - --datadir=/opt/data/erigon - - --authrpc.addr=0.0.0.0 - - --authrpc.vhosts=* - - --authrpc.port=8551 - - --authrpc.jwtsecret=/engine.jwt - - --rpc.returndata.limit=1000000 - - --ws - - --http - - --http.vhosts=* - - --http.corsdomain=* - - --http.addr=0.0.0.0 - - --http.port=8545 - - --http.api=engine,net,eth,web3 - - --metrics - - --metrics.addr=0.0.0.0 - - --metrics.port=6060 - - --db.pagesize=16K - - --db.size.limit=4TB - - --prune=htc - entrypoint: [] - env: {} - image: thorax/erigon:v2.54.0 - ports: - - 0.0.0.0:30303:30303/tcp - - 0.0.0.0:30303:30303/udp - - 127.0.0.1:8545:8545/tcp - - 127.0.0.1:8546:8546/tcp - volumes: - - /opt/stereum/erigon-c3c822b5-5b0c-b779-8a04-fae3563ccfa1/data:/opt/data/erigon - - /opt/stereum/erigon-c3c822b5-5b0c-b779-8a04-fae3563ccfa1/engine.jwt:/engine.jwt - user: root - autoupdate: true - network: holesky - dependencies: - executionClients: [] - consensusClients: [] - mevboost: [] - become: yes - - - name: Create Erigon service config - copy: - dest: "/etc/stereum/services/da69776a-2468-4e2d-b3fb-7a6dfd9eabf1.yaml" - owner: "root" - group: "root" - mode: 0644 - content: | - service: ErigonService - id: da69776a-2468-4e2d-b3fb-7a6dfd9eabf1 - configVersion: 1 - command: - - erigon - - --chain=holesky - - --datadir=/opt/data/erigon - - --authrpc.addr=0.0.0.0 - - --authrpc.vhosts=* - - --authrpc.port=8551 - - --authrpc.jwtsecret=/engine.jwt - - --rpc.returndata.limit=1000000 - - --ws - - --http - - --http.vhosts=* - - --http.corsdomain=* - - --http.addr=0.0.0.0 - - --http.port=8545 - - --http.api=engine,net,eth,web3 - - --metrics - - --metrics.addr=0.0.0.0 - - --metrics.port=6060 - - --db.pagesize=16K - - --prune=htc - entrypoint: [] - env: {} - image: thorax/erigon:v2.54.0 - ports: - - 0.0.0.0:30303:30303/tcp - - 0.0.0.0:30303:30303/udp - - 127.0.0.1:8545:8545/tcp - - 127.0.0.1:8546:8546/tcp - volumes: - - /opt/stereum/erigon-da69776a-2468-4e2d-b3fb-7a6dfd9eabf1/data:/opt/data/erigon - - /opt/stereum/erigon-da69776a-2468-4e2d-b3fb-7a6dfd9eabf1/engine.jwt:/engine.jwt - user: root - autoupdate: true - network: holesky - dependencies: - executionClients: [] - consensusClients: [] - mevboost: [] - become: yes -# EOF diff --git a/controls/roles/update-changes/molecule/202/verify.yml b/controls/roles/update-changes/molecule/202/verify.yml deleted file mode 100644 index 9d3604fa0..000000000 --- a/controls/roles/update-changes/molecule/202/verify.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- -- name: Verify - hosts: all - gather_facts: false - tasks: - # Erigon - - name: Read ErigonService file 1 - slurp: - src: "/etc/stereum/services/c3c822b5-5b0c-b779-8a04-fae3563ccfa1.yaml" - register: erigon_service_configuration1_raw - - # Erigon - - name: Read ErigonService file 2 - slurp: - src: "/etc/stereum/services/da69776a-2468-4e2d-b3fb-7a6dfd9eabf1.yaml" - register: erigon_service_configuration2_raw - - - name: Parse ErigonService configurations - set_fact: - erigon_service_configuration1: "{{ erigon_service_configuration1_raw['content'] | b64decode | from_yaml }}" - erigon_service_configuration2: "{{ erigon_service_configuration2_raw['content'] | b64decode | from_yaml }}" - - - debug: - msg: "{{ erigon_service_configuration1 }}" - - - debug: - msg: "{{ erigon_service_configuration2 }}" - - - assert: - that: - - erigon_service_configuration1.command | select('match', '--db.size.limit.*') | length == 1 - - erigon_service_configuration2.command | select('match', '--db.size.limit.*') | length == 1 - - erigon_service_configuration1.command | select('match', '--db.size.limit=4TB') | length == 1 - - erigon_service_configuration2.command | select('match', '--db.size.limit.8TB') | length == 1 -# EOF diff --git a/controls/roles/update-changes/molecule/203/converge.yml b/controls/roles/update-changes/molecule/203/converge.yml deleted file mode 100644 index 3f84cb4a2..000000000 --- a/controls/roles/update-changes/molecule/203/converge.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: Converge - hosts: all - vars_files: - - ../../../../defaults/stereum_defaults.yaml - - tasks: - - name: "Include update-changes" - include_role: - name: "update-changes" diff --git a/controls/roles/update-changes/molecule/203/molecule.yml b/controls/roles/update-changes/molecule/203/molecule.yml deleted file mode 100644 index cdafc0354..000000000 --- a/controls/roles/update-changes/molecule/203/molecule.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -#dependency: -# name: galaxy -driver: - name: docker -platforms: - - name: "update-changes--2.0.3--ubuntu-23.04" - image: ubuntu:lunar - # - name: "configure-updates--default--centos-8" - # image: geerlingguy/docker-centos8-ansible -provisioner: - name: ansible - env: - ANSIBLE_PIPELINING: "True" -lint: | - set -e - yamllint . - ansible-lint . -scenario: - test_sequence: - - destroy - - create - - prepare - - converge - - idempotence - - verify - - destroy diff --git a/controls/roles/update-changes/molecule/203/playbook.yml b/controls/roles/update-changes/molecule/203/playbook.yml deleted file mode 100644 index 710c596cf..000000000 --- a/controls/roles/update-changes/molecule/203/playbook.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Converge - hosts: all - tasks: - - name: "Include update-changes" - include_role: - name: "update-changes" diff --git a/controls/roles/update-changes/molecule/203/prepare.yml b/controls/roles/update-changes/molecule/203/prepare.yml deleted file mode 100644 index 81decc0ab..000000000 --- a/controls/roles/update-changes/molecule/203/prepare.yml +++ /dev/null @@ -1,114 +0,0 @@ ---- -- name: Prepare - hosts: all - tasks: - - name: Make sure Stereum's config path exists - file: - path: "/etc/stereum/services" - state: directory - owner: "root" - group: "root" - mode: 0644 - become: yes - - - name: Create Nethermind service config - copy: - dest: "/etc/stereum/services/357870cb-8d06-49ee-a204-400080ee34db.yaml" - owner: "root" - group: "root" - mode: 0644 - content: | - service: NethermindService - id: 357870cb-8d06-49ee-a204-400080ee34db - configVersion: 1 - command: - - --config=holesky - - --log=info - - --datadir=/opt/app/data - - --Network.DiscoveryPort=30303 - - --Network.P2PPort=30303 - - --Merge.Enabled=true - - --JsonRpc.Enabled=true - - --JsonRpc.JwtSecretFile=/engine.jwt - - --JsonRpc.Host=0.0.0.0 - - --JsonRpc.EngineHost=0.0.0.0 - - --Init.WebSocketsEnabled=true - - --JsonRpc.WebSocketsPort=8546 - - --JsonRpc.EnabledModules=[web3,eth,subscribe,net] - - --JsonRpc.AdditionalRpcUrls=http://0.0.0.0:8551|http;ws|engine;eth;subscribe - - --Metrics.Enabled=true - - --Metrics.ExposePort=6060 - - --HealthChecks.Enabled=true - - --Pruning.Mode=Hybrid - entrypoint: - - ./nethermind - env: {} - image: nethermind/nethermind:1.25.0 - ports: - - 0.0.0.0:30304:30303/tcp - - 0.0.0.0:30304:30303/udp - - 127.0.0.1:8547:8545/tcp - - 127.0.0.1:8548:8546/tcp - volumes: - - /opt/stereum/nethermind-357870cb-8d06-49ee-a204-400080ee34db/data:/opt/app/data - - /opt/stereum/nethermind-357870cb-8d06-49ee-a204-400080ee34db/engine.jwt:/engine.jwt - user: root - autoupdate: true - network: holesky - dependencies: - executionClients: [] - consensusClients: [] - mevboost: [] - become: yes - - - name: Create Nethermind service config - copy: - dest: "/etc/stereum/services/1b7714e5-37ab-4656-bf99-8b8d0d3b9082.yaml" - owner: "root" - group: "root" - mode: 0644 - content: | - service: NethermindService - id: 1b7714e5-37ab-4656-bf99-8b8d0d3b9082 - configVersion: 1 - command: - - --config=holesky - - --log=info - - --datadir=/opt/app/data - - --Network.DiscoveryPort=30303 - - --Network.P2PPort=30303 - - --Merge.Enabled=true - - --JsonRpc.Enabled=true - - --JsonRpc.JwtSecretFile=/engine.jwt - - --JsonRpc.Host=0.0.0.0 - - --JsonRpc.EngineHost=0.0.0.0 - - --Init.WebSocketsEnabled=true - - --JsonRpc.WebSocketsPort=8546 - - --JsonRpc.EnabledModules=[web3,eth,subscribe,net] - - --JsonRpc.AdditionalRpcUrls=http://0.0.0.0:8551|http;ws|engine;eth;subscribe - - --Metrics.Enabled=true - - --Metrics.ExposePort=6060 - - --HealthChecks.Enabled=true - - --Pruning.Mode=Hybrid - - --Pruning.FullPruningTrigger=VolumeFreeSpace - entrypoint: - - ./nethermind - env: {} - image: nethermind/nethermind:1.25.0 - ports: - - 0.0.0.0:30304:30303/tcp - - 0.0.0.0:30304:30303/udp - - 127.0.0.1:8547:8545/tcp - - 127.0.0.1:8548:8546/tcp - volumes: - - /opt/stereum/nethermind-1b7714e5-37ab-4656-bf99-8b8d0d3b9082/data:/opt/app/data - - /opt/stereum/nethermind-1b7714e5-37ab-4656-bf99-8b8d0d3b9082/engine.jwt:/engine.jwt - user: root - autoupdate: true - network: holesky - dependencies: - executionClients: [] - consensusClients: [] - mevboost: [] - become: yes -# EOF diff --git a/controls/roles/update-changes/molecule/203/verify.yml b/controls/roles/update-changes/molecule/203/verify.yml deleted file mode 100644 index 8eb62c5b4..000000000 --- a/controls/roles/update-changes/molecule/203/verify.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- -- name: Verify - hosts: all - gather_facts: false - tasks: - # Nethermind - - name: Read NethermindService file 1 - slurp: - src: "/etc/stereum/services/357870cb-8d06-49ee-a204-400080ee34db.yaml" - register: nethermind_service_configuration1_raw - - # Nethermind - - name: Read NethermindService file 2 - slurp: - src: "/etc/stereum/services/1b7714e5-37ab-4656-bf99-8b8d0d3b9082.yaml" - register: nethermind_service_configuration2_raw - - - name: Parse NethermindService configurations - set_fact: - nethermind_service_configuration1: "{{ nethermind_service_configuration1_raw['content'] | b64decode | from_yaml }}" - nethermind_service_configuration2: "{{ nethermind_service_configuration2_raw['content'] | b64decode | from_yaml }}" - - - debug: - msg: "{{ nethermind_service_configuration1 }}" - - - debug: - msg: "{{ nethermind_service_configuration2 }}" - - - assert: - that: - - nethermind_service_configuration1.command | select('match', '--Pruning.FullPruningTrigger.*') | length == 1 - - nethermind_service_configuration2.command | select('match', '--Pruning.FullPruningTrigger.*') | length == 1 - - nethermind_service_configuration1.command | select('match', '--Pruning.FullPruningTrigger=StateDbSize') | length == 1 - - nethermind_service_configuration2.command | select('match', '--Pruning.FullPruningTrigger=VolumeFreeSpace') | length == 1 -# EOF diff --git a/controls/roles/update-changes/molecule/213/converge.yml b/controls/roles/update-changes/molecule/213/converge.yml deleted file mode 100644 index 3f84cb4a2..000000000 --- a/controls/roles/update-changes/molecule/213/converge.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: Converge - hosts: all - vars_files: - - ../../../../defaults/stereum_defaults.yaml - - tasks: - - name: "Include update-changes" - include_role: - name: "update-changes" diff --git a/controls/roles/update-changes/molecule/213/molecule.yml b/controls/roles/update-changes/molecule/213/molecule.yml deleted file mode 100644 index 075488536..000000000 --- a/controls/roles/update-changes/molecule/213/molecule.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -#dependency: -# name: galaxy -driver: - name: docker -platforms: - - name: "update-changes--2.1.3--ubuntu-23.04" - image: ubuntu:lunar - # - name: "configure-updates--default--centos-8" - # image: geerlingguy/docker-centos8-ansible -provisioner: - name: ansible - env: - ANSIBLE_PIPELINING: "True" -lint: | - set -e - yamllint . - ansible-lint . -scenario: - test_sequence: - - destroy - - create - - prepare - - converge - - idempotence - - verify - - destroy diff --git a/controls/roles/update-changes/molecule/213/playbook.yml b/controls/roles/update-changes/molecule/213/playbook.yml deleted file mode 100644 index 710c596cf..000000000 --- a/controls/roles/update-changes/molecule/213/playbook.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Converge - hosts: all - tasks: - - name: "Include update-changes" - include_role: - name: "update-changes" diff --git a/controls/roles/update-changes/molecule/213/prepare.yml b/controls/roles/update-changes/molecule/213/prepare.yml deleted file mode 100644 index aa1827593..000000000 --- a/controls/roles/update-changes/molecule/213/prepare.yml +++ /dev/null @@ -1,126 +0,0 @@ ---- -- name: Prepare - hosts: all - tasks: - - name: Make sure Stereum's config path exists - file: - path: "/etc/stereum/services" - state: directory - owner: "root" - group: "root" - mode: 0644 - become: yes - - - name: Create Besu service config - copy: - dest: "/etc/stereum/services/13e313c5-3209-c32b-439b-88300e79e251.yaml" - owner: "root" - group: "root" - mode: 0644 - content: | - service: BesuService - id: 13e313c5-3209-c32b-439b-88300e79e251 - configVersion: 1 - command: - - --network=holesky - - --data-path=/opt/app/data - - --data-storage-format=BONSAI - - --sync-mode=X_SNAP - - --p2p-port=30303 - - --p2p-host=0.0.0.0 - - --rpc-http-enabled=true - - --rpc-http-host=0.0.0.0 - - --rpc-http-cors-origins=* - - --rpc-http-port=8545 - - --rpc-ws-enabled=true - - --rpc-ws-host=0.0.0.0 - - --rpc-ws-port=8546 - - --host-allowlist=* - - --metrics-enabled - - --metrics-host=0.0.0.0 - - --metrics-port=9545 - - --logging=INFO - - --engine-rpc-enabled=true - - --engine-host-allowlist=* - - --engine-rpc-port=8551 - - --engine-jwt-enabled=true - - --engine-jwt-secret=/engine.jwt - entrypoint: - - besu - env: - JAVA_OPTS: -Xmx4g - image: hyperledger/besu:24.1.2 - ports: - - 0.0.0.0:30303:30303/tcp - - 0.0.0.0:30303:30303/udp - - 127.0.0.1:8545:8545/tcp - - 127.0.0.1:8546:8546/tcp - volumes: - - /opt/stereum/besu-13e313c5-3209-c32b-439b-88300e79e251/data:/opt/app/data - - /opt/stereum/besu-13e313c5-3209-c32b-439b-88300e79e251/engine.jwt:/engine.jwt - user: "2000" - autoupdate: true - network: holesky - dependencies: - executionClients: [] - consensusClients: [] - mevboost: [] - otherServices: [] - become: yes - - - name: Create Besu service config - copy: - dest: "/etc/stereum/services/5ece5920-32c0-6b23-1747-3f223e61bede.yaml" - owner: "root" - group: "root" - mode: 0644 - content: | - service: BesuService - id: 5ece5920-32c0-6b23-1747-3f223e61bede - configVersion: 1 - command: - - --network=holesky - - --data-path=/opt/app/data - - --data-storage-format=BONSAI - - --sync-mode=X_CHECKPOINT - - --p2p-port=30303 - - --p2p-host=0.0.0.0 - - --rpc-http-enabled=true - - --rpc-http-host=0.0.0.0 - - --rpc-http-cors-origins=* - - --rpc-http-port=8545 - - --rpc-ws-enabled=true - - --rpc-ws-host=0.0.0.0 - - --rpc-ws-port=8546 - - --host-allowlist=* - - --metrics-enabled - - --metrics-host=0.0.0.0 - - --metrics-port=9545 - - --logging=INFO - - --engine-host-allowlist=* - - --engine-rpc-port=8551 - - --engine-jwt-enabled=true - - --engine-jwt-secret=/engine.jwt - entrypoint: - - besu - env: - JAVA_OPTS: -Xmx4g - image: hyperledger/besu:24.1.2 - ports: - - 0.0.0.0:30303:30303/tcp - - 0.0.0.0:30303:30303/udp - - 127.0.0.1:8545:8545/tcp - - 127.0.0.1:8546:8546/tcp - volumes: - - /opt/stereum/besu-5ece5920-32c0-6b23-1747-3f223e61bede/data:/opt/app/data - - /opt/stereum/besu-5ece5920-32c0-6b23-1747-3f223e61bede/engine.jwt:/engine.jwt - user: "2000" - autoupdate: true - network: holesky - dependencies: - executionClients: [] - consensusClients: [] - mevboost: [] - otherServices: [] - become: yes -# EOF diff --git a/controls/roles/update-changes/molecule/213/verify.yml b/controls/roles/update-changes/molecule/213/verify.yml deleted file mode 100644 index 1dfa97a5f..000000000 --- a/controls/roles/update-changes/molecule/213/verify.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- -- name: Verify - hosts: all - gather_facts: false - tasks: - # Besu - - name: Read Besu file 1 - slurp: - src: "/etc/stereum/services/13e313c5-3209-c32b-439b-88300e79e251.yaml" - register: Besu_service_configuration1_raw - - # Besu - - name: Read Besu file 2 - slurp: - src: "/etc/stereum/services/5ece5920-32c0-6b23-1747-3f223e61bede.yaml" - register: Besu_service_configuration2_raw - - - name: Parse Besu configurations - set_fact: - Besu_service_configuration1: "{{ Besu_service_configuration1_raw['content'] | b64decode | from_yaml }}" - Besu_service_configuration2: "{{ Besu_service_configuration2_raw['content'] | b64decode | from_yaml }}" - - - debug: - msg: "{{ Besu_service_configuration1 }}" - - - debug: - msg: "{{ Besu_service_configuration2 }}" - - - assert: - that: - - Besu_service_configuration1.command | select('match', '--sync-mode=SNAP') | length == 1 - - Besu_service_configuration2.command | select('match', '--sync-mode=CHECKPOINT') | length == 1 - - Besu_service_configuration1.command | select('match', '--engine-jwt-enabled') | length == 0 - - Besu_service_configuration2.command | select('match', '--engine-jwt-enabled') | length == 0 -# EOF diff --git a/controls/roles/update-changes/molecule/214/converge.yml b/controls/roles/update-changes/molecule/214/converge.yml deleted file mode 100644 index 3f84cb4a2..000000000 --- a/controls/roles/update-changes/molecule/214/converge.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: Converge - hosts: all - vars_files: - - ../../../../defaults/stereum_defaults.yaml - - tasks: - - name: "Include update-changes" - include_role: - name: "update-changes" diff --git a/controls/roles/update-changes/molecule/214/molecule.yml b/controls/roles/update-changes/molecule/214/molecule.yml deleted file mode 100644 index 9217beb16..000000000 --- a/controls/roles/update-changes/molecule/214/molecule.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -#dependency: -# name: galaxy -driver: - name: docker -platforms: - - name: "update-changes--2.1.4--ubuntu-23.04" - image: ubuntu:lunar - # - name: "configure-updates--default--centos-8" - # image: geerlingguy/docker-centos8-ansible -provisioner: - name: ansible - env: - ANSIBLE_PIPELINING: "True" -lint: | - set -e - yamllint . - ansible-lint . -scenario: - test_sequence: - - destroy - - create - - prepare - - converge - - idempotence - - verify - - destroy diff --git a/controls/roles/update-changes/molecule/214/playbook.yml b/controls/roles/update-changes/molecule/214/playbook.yml deleted file mode 100644 index 710c596cf..000000000 --- a/controls/roles/update-changes/molecule/214/playbook.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Converge - hosts: all - tasks: - - name: "Include update-changes" - include_role: - name: "update-changes" diff --git a/controls/roles/update-changes/molecule/214/prepare.yml b/controls/roles/update-changes/molecule/214/prepare.yml deleted file mode 100644 index 2f75f73cb..000000000 --- a/controls/roles/update-changes/molecule/214/prepare.yml +++ /dev/null @@ -1,199 +0,0 @@ ---- -- name: Prepare - hosts: all - tasks: - - name: Make sure Stereum's config path exists - file: - path: "/etc/stereum/services" - state: directory - owner: "root" - group: "root" - mode: 0644 - become: yes - - - name: Create Lodestar service config - copy: - dest: "/etc/stereum/services/81ce4f8c-1e73-4201-9889-c5c2c8bb53d5.yaml" - owner: "root" - group: "root" - mode: 0644 - content: | - service: LodestarValidatorService - id: 81ce4f8c-1e73-4201-9889-c5c2c8bb53d5 - configVersion: 1 - command: - - validator - - --network=holesky - - --dataDir=/opt/app/validator - - --keymanager=true - - --keymanager.address=0.0.0.0 - - --keymanager.port=5062 - - --metrics=true - - --metrics.port=5064 - - --metrics.address=0.0.0.0 - - --builder - - --suggestedFeeRecipient=0x0000000000000000000000000000000000000000 - - --doppelgangerProtection=true - - --distributed - - --beaconNodes=http://stereum-314b878c-0589-f5b1-5a59-46b07066e53a:3600 - entrypoint: - - node - - ./packages/cli/bin/lodestar - env: {} - image: chainsafe/lodestar:v1.18.0 - ports: - - 127.0.0.1:5063:5062/tcp - volumes: - - /opt/stereum/lodestar-81ce4f8c-1e73-4201-9889-c5c2c8bb53d5/validator:/opt/app/validator - user: "2000" - autoupdate: true - network: holesky - dependencies: - executionClients: [] - consensusClients: - - service: CharonService - id: 314b878c-0589-f5b1-5a59-46b07066e53a - mevboost: [] - otherServices: [] - become: yes - - - name: Create Teku service config - copy: - dest: "/etc/stereum/services/9a02c5a4-26f5-4670-93bb-d40ad655b913.yaml" - owner: "root" - group: "root" - mode: 0644 - content: | - service: TekuValidatorService - id: 9a02c5a4-26f5-4670-93bb-d40ad655b913 - configVersion: 1 - command: - - vc - - --network=holesky - - --logging=INFO - - --data-path=/opt/app/data - - --validators-keystore-locking-enabled=true - - --validators-proposer-default-fee-recipient=0x0000000000000000000000000000000000000000 - - --validators-graffiti-file=/opt/app/graffitis/graffitis.yaml - - --log-destination=CONSOLE - - --metrics-enabled=true - - --metrics-port=8008 - - --metrics-interface=0.0.0.0 - - --metrics-host-allowlist=* - - --metrics-publish-interval=10 - - --validator-api-enabled=true - - --validator-api-port=5052 - - --validator-api-host-allowlist=* - - --validator-api-cors-origins=* - - --validator-api-keystore-file=/opt/app/data/teku_api_keystore - - --validator-api-keystore-password-file=/opt/app/data/teku_api_password.txt - - --validators-builder-registration-default-enabled=true - - --validators-proposer-blinded-blocks-enabled=true - - --doppelganger-detection-enabled=true - - --beacon-node-api-endpoint=http://stereum-314b878c-0589-f5b1-5a59-46b07066e53a:3600 - entrypoint: - - /opt/teku/bin/teku - env: - JAVA_OPTS: -Xmx4g - image: consensys/teku:24.4.0 - ports: [] - volumes: - - /opt/stereum/teku-9a02c5a4-26f5-4670-93bb-d40ad655b913/data:/opt/app/data - - /opt/stereum/teku-9a02c5a4-26f5-4670-93bb-d40ad655b913/graffitis:/opt/app/graffitis - user: "2000" - autoupdate: true - network: holesky - dependencies: - executionClients: [] - consensusClients: - - service: CharonService - id: 314b878c-0589-f5b1-5a59-46b07066e53a - mevboost: [] - otherServices: [] - become: yes - - - name: Create Lighthouse service config - copy: - dest: "/etc/stereum/services/40e6f174-062b-45f0-8f61-83b568b96d6f.yaml" - owner: "root" - group: "root" - mode: 0644 - content: | - service: LighthouseValidatorService - id: 40e6f174-062b-45f0-8f61-83b568b96d6f - configVersion: 1 - command: - - lighthouse - - vc - - --enable-doppelganger-protection - - --debug-level=debug - - --network=holesky - - --suggested-fee-recipient=0x0000000000000000000000000000000000000000 - - --datadir=/opt/app/validator - - --init-slashing-protection - - --graffiti-file=/opt/app/graffitis/graffitis.yaml - - --metrics - - --metrics-address=0.0.0.0 - - --http - - --http-address=0.0.0.0 - - --http-port=5062 - - --unencrypted-http-transport - - --builder-proposals - - --beacon-nodes=http://stereum-314b878c-0589-f5b1-5a59-46b07066e53a:3600 - entrypoint: [] - env: {} - image: sigp/lighthouse:v5.1.3 - ports: - - 127.0.0.1:5062:5062/tcp - volumes: - - /opt/stereum/lighthouse-40e6f174-062b-45f0-8f61-83b568b96d6f/validator:/opt/app/validator - - /opt/stereum/lighthouse-40e6f174-062b-45f0-8f61-83b568b96d6f/graffitis:/opt/app/graffitis - user: "2000" - autoupdate: true - network: holesky - dependencies: - executionClients: [] - consensusClients: - - service: CharonService - id: 314b878c-0589-f5b1-5a59-46b07066e53a - mevboost: [] - otherServices: [] - become: yes - - - name: Create Charon service config - copy: - dest: "/etc/stereum/services/314b878c-0589-f5b1-5a59-46b07066e53a.yaml" - owner: "root" - group: "root" - mode: 0644 - content: | - service: CharonService - id: 314b878c-0589-f5b1-5a59-46b07066e53a - configVersion: 1 - command: - - run - - --beacon-node-endpoints= - - --log-level=info - - --log-format=console - - --p2p-relays=https://0.relay.obol.tech - - --p2p-tcp-address=0.0.0.0:3610 - - --validator-api-address=0.0.0.0:3600 - - --monitoring-address=0.0.0.0:3620 - entrypoint: - - /usr/local/bin/charon - env: {} - image: obolnetwork/charon:v0.19.2 - ports: - - 0.0.0.0:3610:3610/tcp - volumes: - - /opt/stereum/charon-314b878c-0589-f5b1-5a59-46b07066e53a:/opt/charon - user: "2000" - autoupdate: true - network: holesky - dependencies: - executionClients: [] - consensusClients: [] - mevboost: [] - otherServices: [] - become: yes -# EOF diff --git a/controls/roles/update-changes/molecule/214/verify.yml b/controls/roles/update-changes/molecule/214/verify.yml deleted file mode 100644 index 1804ffda7..000000000 --- a/controls/roles/update-changes/molecule/214/verify.yml +++ /dev/null @@ -1,44 +0,0 @@ ---- -- name: Verify - hosts: all - gather_facts: false - tasks: - # Lodestar - - name: Read Lodestar file - slurp: - src: "/etc/stereum/services/81ce4f8c-1e73-4201-9889-c5c2c8bb53d5.yaml" - register: Lodestar_service_configuration_raw - - # Teku - - name: Read Teku file - slurp: - src: "/etc/stereum/services/9a02c5a4-26f5-4670-93bb-d40ad655b913.yaml" - register: Teku_service_configuration_raw - - # Lighthouse - - name: Read Lighthouse file - slurp: - src: "/etc/stereum/services/40e6f174-062b-45f0-8f61-83b568b96d6f.yaml" - register: Lighthouse_service_configuration_raw - - - name: Parse Besu configurations - set_fact: - Lodestar_service_configuration: "{{ Lodestar_service_configuration_raw['content'] | b64decode | from_yaml }}" - Teku_service_configuration: "{{ Teku_service_configuration_raw['content'] | b64decode | from_yaml }}" - Lighthouse_service_configuration: "{{ Lighthouse_service_configuration_raw['content'] | b64decode | from_yaml }}" - - - debug: - msg: "{{ Lodestar_service_configuration }}" - - - debug: - msg: "{{ Teku_service_configuration }}" - - - debug: - msg: "{{ Lighthouse_service_configuration }}" - - - assert: - that: - - Lodestar_service_configuration.command | select('match', '--useProduceBlockV3=false') | length == 1 - - Teku_service_configuration.command | select('match', '--Xblock-v3-enabled=false') | length == 1 - - Lighthouse_service_configuration.command | select('match', '--produce-block-v3=false') | length == 1 -# EOF diff --git a/controls/roles/update-changes/molecule/215/prepare.yml b/controls/roles/update-changes/molecule/215/prepare.yml index f90e4a21f..0a62595c1 100644 --- a/controls/roles/update-changes/molecule/215/prepare.yml +++ b/controls/roles/update-changes/molecule/215/prepare.yml @@ -100,4 +100,299 @@ mevboost: [] otherServices: [] become: yes + + - name: Create Charon service config + copy: + dest: "/etc/stereum/services/314b878c-0589-f5b1-5a59-46b07066e53a.yaml" + owner: "root" + group: "root" + mode: 0644 + content: | + service: CharonService + id: 314b878c-0589-f5b1-5a59-46b07066e53a + configVersion: 1 + command: + - run + - --beacon-node-endpoints=stereum-4dc79d62-ee8b-47f8-0af8-7514e3a5f445:5051,stereum-4c8bcca8-5464-2790-11f5-07b3d326ca7c:5052 + - --log-level=info + - --log-format=console + - --p2p-relays=https://0.relay.obol.tech + - --p2p-tcp-address=0.0.0.0:3610 + - --validator-api-address=0.0.0.0:3600 + - --monitoring-address=0.0.0.0:3620 + entrypoint: + - /usr/local/bin/charon + env: {} + image: obolnetwork/charon:v1.0.0-rc3 + ports: + - 0.0.0.0:3610:3610/tcp + volumes: + - /opt/stereum/charon-314b878c-0589-f5b1-5a59-46b07066e53a:/opt/charon + user: "2000" + autoupdate: true + network: holesky + dependencies: + executionClients: [] + consensusClients: + - service: TekuBeaconService + id: 4dc79d62-ee8b-47f8-0af8-7514e3a5f445 + - service: NimbusBeaconService + id: 4c8bcca8-5464-2790-11f5-07b3d326ca7c + mevboost: [] + otherServices: [] + become: yes + + - name: Create Lodestar service config + copy: + dest: "/etc/stereum/services/81ce4f8c-1e73-4201-9889-c5c2c8bb53d5.yaml" + owner: "root" + group: "root" + mode: 0644 + content: | + service: LodestarValidatorService + id: 81ce4f8c-1e73-4201-9889-c5c2c8bb53d5 + configVersion: 1 + command: + - validator + - --network=holesky + - --dataDir=/opt/app/validator + - --keymanager=true + - --keymanager.address=0.0.0.0 + - --keymanager.port=5062 + - --metrics=true + - --metrics.port=5064 + - --metrics.address=0.0.0.0 + - --builder + - --suggestedFeeRecipient=0x0000000000000000000000000000000000000000 + - --doppelgangerProtection=true + - --distributed + - --useProduceBlockV3=false + - --beaconNodes=http://stereum-314b878c-0589-f5b1-5a59-46b07066e53a:3600 + entrypoint: + - node + - ./packages/cli/bin/lodestar + env: {} + image: chainsafe/lodestar:v1.18.0 + ports: + - 127.0.0.1:5063:5062/tcp + volumes: + - /opt/stereum/lodestar-81ce4f8c-1e73-4201-9889-c5c2c8bb53d5/validator:/opt/app/validator + user: "2000" + autoupdate: true + network: holesky + dependencies: + executionClients: [] + consensusClients: + - service: CharonService + id: 314b878c-0589-f5b1-5a59-46b07066e53a + mevboost: [] + otherServices: [] + become: yes + + - name: Create Teku service config + copy: + dest: "/etc/stereum/services/9a02c5a4-26f5-4670-93bb-d40ad655b913.yaml" + owner: "root" + group: "root" + mode: 0644 + content: | + service: TekuValidatorService + id: 9a02c5a4-26f5-4670-93bb-d40ad655b913 + configVersion: 1 + command: + - vc + - --network=holesky + - --logging=INFO + - --data-path=/opt/app/data + - --validators-keystore-locking-enabled=true + - --validators-proposer-default-fee-recipient=0x0000000000000000000000000000000000000000 + - --validators-graffiti-file=/opt/app/graffitis/graffitis.yaml + - --log-destination=CONSOLE + - --metrics-enabled=true + - --metrics-port=8008 + - --metrics-interface=0.0.0.0 + - --metrics-host-allowlist=* + - --metrics-publish-interval=10 + - --validator-api-enabled=true + - --validator-api-port=5052 + - --validator-api-host-allowlist=* + - --validator-api-cors-origins=* + - --validator-api-keystore-file=/opt/app/data/teku_api_keystore + - --validator-api-keystore-password-file=/opt/app/data/teku_api_password.txt + - --validators-builder-registration-default-enabled=true + - --validators-proposer-blinded-blocks-enabled=true + - --doppelganger-detection-enabled=true + - --Xblock-v3-enabled=false + - --beacon-node-api-endpoint=http://stereum-314b878c-0589-f5b1-5a59-46b07066e53a:3600 + entrypoint: + - /opt/teku/bin/teku + env: + JAVA_OPTS: -Xmx4g + image: consensys/teku:24.4.0 + ports: [] + volumes: + - /opt/stereum/teku-9a02c5a4-26f5-4670-93bb-d40ad655b913/data:/opt/app/data + - /opt/stereum/teku-9a02c5a4-26f5-4670-93bb-d40ad655b913/graffitis:/opt/app/graffitis + user: "2000" + autoupdate: true + network: holesky + dependencies: + executionClients: [] + consensusClients: + - service: CharonService + id: 314b878c-0589-f5b1-5a59-46b07066e53a + mevboost: [] + otherServices: [] + become: yes + + - name: Create Lighthouse service config + copy: + dest: "/etc/stereum/services/40e6f174-062b-45f0-8f61-83b568b96d6f.yaml" + owner: "root" + group: "root" + mode: 0644 + content: | + service: LighthouseValidatorService + id: 40e6f174-062b-45f0-8f61-83b568b96d6f + configVersion: 1 + command: + - lighthouse + - vc + - --enable-doppelganger-protection + - --debug-level=debug + - --network=holesky + - --suggested-fee-recipient=0x0000000000000000000000000000000000000000 + - --datadir=/opt/app/validator + - --init-slashing-protection + - --graffiti-file=/opt/app/graffitis/graffitis.yaml + - --metrics + - --metrics-address=0.0.0.0 + - --http + - --http-address=0.0.0.0 + - --http-port=5062 + - --unencrypted-http-transport + - --builder-proposals + - --produce-block-v3=false + - --beacon-nodes=http://stereum-314b878c-0589-f5b1-5a59-46b07066e53a:3600 + entrypoint: [] + env: {} + image: sigp/lighthouse:v5.1.3 + ports: + - 127.0.0.1:5062:5062/tcp + volumes: + - /opt/stereum/lighthouse-40e6f174-062b-45f0-8f61-83b568b96d6f/validator:/opt/app/validator + - /opt/stereum/lighthouse-40e6f174-062b-45f0-8f61-83b568b96d6f/graffitis:/opt/app/graffitis + user: "2000" + autoupdate: true + network: holesky + dependencies: + executionClients: [] + consensusClients: + - service: CharonService + id: 314b878c-0589-f5b1-5a59-46b07066e53a + mevboost: [] + otherServices: [] + become: yes + + - name: Create Teku Beacon service config + copy: + dest: "/etc/stereum/services/4dc79d62-ee8b-47f8-0af8-7514e3a5f445.yaml" + owner: "root" + group: "root" + mode: 0644 + content: | + service: TekuBeaconService + id: 4dc79d62-ee8b-47f8-0af8-7514e3a5f445 + configVersion: 2 + command: + - --network=sepolia + - --p2p-enabled=true + - --p2p-port=9001 + - --p2p-advertised-port=9001 + - --ee-endpoint= + - --ee-jwt-secret-file=/engine.jwt + - --metrics-enabled=true + - --metrics-port=8008 + - --metrics-interface=0.0.0.0 + - --metrics-host-allowlist=* + - --metrics-publish-interval=10 + - --data-path=/opt/app/data + - --data-storage-mode=prune + - --rest-api-port=5051 + - --rest-api-host-allowlist=* + - --rest-api-interface=0.0.0.0 + - --rest-api-docs-enabled=true + - --rest-api-enabled=true + - --log-destination=CONSOLE + - --ignore-weak-subjectivity-period-enabled + entrypoint: + - /opt/teku/bin/teku + env: + JAVA_OPTS: -Xmx4g + image: consensys/teku:24.4.0 + ports: + - 0.0.0.0:9001:9001/tcp + - 0.0.0.0:9001:9001/udp + - 127.0.0.1:5053:5052/tcp + - 127.0.0.1:5051:5051/tcp + volumes: + - /opt/stereum/teku-4dc79d62-ee8b-47f8-0af8-7514e3a5f445/data:/opt/app/data + user: "2000" + autoupdate: true + network: sepolia + dependencies: + executionClients: [] + consensusClients: [] + mevboost: [] + otherServices: [] + become: yes + + - name: Create NimbusBeaconService config + copy: + dest: "/etc/stereum/services/4c8bcca8-5464-2790-11f5-07b3d326ca7c.yaml" + owner: "root" + group: "root" + mode: 0644 + content: | + service: NimbusBeaconService + id: 4c8bcca8-5464-2790-11f5-07b3d326ca7c + configVersion: 2 + command: + - --network=sepolia + - --data-dir=/opt/app/beacon + - --web3-url=ws://stereum-070cfe35-9fa3-20ac-388b-3b68edd92e54:8551 + - --metrics + - --metrics-port=8008 + - --metrics-address=0.0.0.0 + - --rest + - --rest-address=0.0.0.0 + - --rest-port=5052 + - --jwt-secret=/engine.jwt + - --history=prune + - --tcp-port=9000 + - --udp-port=9000 + - --enr-auto-update=false + - --trusted-node-url=https://beaconstate-sepolia.chainsafe.io/ + entrypoint: + - /home/user/nimbus_beacon_node + env: {} + image: statusim/nimbus-eth2:multiarch-v24.4.0 + ports: + - 0.0.0.0:9000:9000/tcp + - 0.0.0.0:9000:9000/udp + - 127.0.0.1:5052:5052/tcp + volumes: + - /opt/stereum/nimbus-4c8bcca8-5464-2790-11f5-07b3d326ca7c/beacon:/opt/app/beacon + - /opt/stereum/geth-070cfe35-9fa3-20ac-388b-3b68edd92e54/engine.jwt:/engine.jwt + user: "2000" + autoupdate: true + network: sepolia + dependencies: + executionClients: + - service: GethService + id: 070cfe35-9fa3-20ac-388b-3b68edd92e54 + consensusClients: [] + mevboost: [] + otherServices: [] + become: yes # EOF diff --git a/controls/roles/update-changes/molecule/215/verify.yml b/controls/roles/update-changes/molecule/215/verify.yml index 43ab78117..0a1c4b100 100644 --- a/controls/roles/update-changes/molecule/215/verify.yml +++ b/controls/roles/update-changes/molecule/215/verify.yml @@ -15,10 +15,45 @@ src: "/etc/stereum/services/147c5807-b646-471f-aaa4-5e215ad98b75.yaml" register: Ejector_service_configuration2_raw + # Lodestar + - name: Read Lodestar file + slurp: + src: "/etc/stereum/services/81ce4f8c-1e73-4201-9889-c5c2c8bb53d5.yaml" + register: Lodestar_service_configuration_raw + + # Teku + - name: Read Teku file + slurp: + src: "/etc/stereum/services/9a02c5a4-26f5-4670-93bb-d40ad655b913.yaml" + register: Teku_service_configuration_raw + + # Lighthouse + - name: Read Lighthouse file + slurp: + src: "/etc/stereum/services/40e6f174-062b-45f0-8f61-83b568b96d6f.yaml" + register: Lighthouse_service_configuration_raw + + # Teku Beacon + - name: Read Teku Beacon file + slurp: + src: "/etc/stereum/services/4dc79d62-ee8b-47f8-0af8-7514e3a5f445.yaml" + register: Teku_Beacon_service_configuration_raw + + # Charon + - name: Read Charon file + slurp: + src: "/etc/stereum/services/314b878c-0589-f5b1-5a59-46b07066e53a.yaml" + register: charon_service_configuration_raw + - name: Parse Service configurations set_fact: Ejector_service_configuration: "{{ Ejector_service_configuration_raw['content'] | b64decode | from_yaml }}" Ejector_service_configuration2: "{{ Ejector_service_configuration2_raw['content'] | b64decode | from_yaml }}" + Lodestar_service_configuration: "{{ Lodestar_service_configuration_raw['content'] | b64decode | from_yaml }}" + Teku_service_configuration: "{{ Teku_service_configuration_raw['content'] | b64decode | from_yaml }}" + Lighthouse_service_configuration: "{{ Lighthouse_service_configuration_raw['content'] | b64decode | from_yaml }}" + Teku_Beacon_service_configuration: "{{ Teku_Beacon_service_configuration_raw['content'] | b64decode | from_yaml }}" + charon_service_configuration: "{{ charon_service_configuration_raw['content'] | b64decode | from_yaml }}" - debug: msg: "{{ Ejector_service_configuration }}" @@ -30,8 +65,29 @@ - debug: msg: "{{ Ejector_service_configuration2_raw['content'] | b64decode }}" + - debug: + msg: "{{ Lodestar_service_configuration }}" + + - debug: + msg: "{{ Teku_service_configuration }}" + + - debug: + msg: "{{ Lighthouse_service_configuration }}" + + - debug: + msg: "{{ Teku_Beacon_service_configuration }}" + + - debug: + msg: "{{ charon_service_configuration }}" + - assert: that: - Ejector_service_configuration.env.ORACLE_ADDRESSES_ALLOWLIST | from_yaml | length == 10 - Ejector_service_configuration2.env.ORACLE_ADDRESSES_ALLOWLIST | from_yaml | length == 9 + - Lodestar_service_configuration.command | select('match', '--useProduceBlockV3=false') | length == 0 + - Teku_service_configuration.command | select('match', '--Xblock-v3-enabled=false') | length == 0 + - Lighthouse_service_configuration.command | select('match', '--produce-block-v3=false') | length == 0 + - Teku_Beacon_service_configuration.command | select('match', '--validators-graffiti-client-append-format=DISABLED') | length == 1 + - charon_service_configuration.command | select('match', '--feature-set-enable=json_requests') | length == 1 + - charon_service_configuration.command | select('match', '--builder-api') | length == 1 # EOF diff --git a/controls/roles/update-changes/tasks/2.0-rc23/nethermind_entrypoint.yaml b/controls/roles/update-changes/tasks/2.0-rc23/nethermind_entrypoint.yaml deleted file mode 100644 index 9a78d9245..000000000 --- a/controls/roles/update-changes/tasks/2.0-rc23/nethermind_entrypoint.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -- name: Read service file - slurp: - src: "{{ item.path }}" - register: service_configuration_raw - -- name: Parse service's configuration - set_fact: - service_configuration: "{{ service_configuration_raw['content'] | b64decode | from_yaml }}" - -- name: Update Nethermind Entypoint - block: - - name: Replace Entrypoint - replace: - path: "{{ item.path }}" - regexp: "./Nethermind.Runner" - replace: "./nethermind" - when: service_configuration.service == "NethermindService" diff --git a/controls/roles/update-changes/tasks/2.0-rc23/updates-20-rc23.yaml b/controls/roles/update-changes/tasks/2.0-rc23/updates-20-rc23.yaml deleted file mode 100644 index 16d714751..000000000 --- a/controls/roles/update-changes/tasks/2.0-rc23/updates-20-rc23.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- name: Find service configs - find: - paths: "/etc/stereum/services" - register: service_config_files - -- name: Include Nethermind Entypoint Changes - include_tasks: nethermind_entrypoint.yaml - loop: "{{ service_config_files.files }}" diff --git a/controls/roles/update-changes/tasks/2.0.2/erigon_dbTag.yaml b/controls/roles/update-changes/tasks/2.0.2/erigon_dbTag.yaml deleted file mode 100644 index b61ca15af..000000000 --- a/controls/roles/update-changes/tasks/2.0.2/erigon_dbTag.yaml +++ /dev/null @@ -1,19 +0,0 @@ ---- -- name: Read service file - slurp: - src: "{{ item.path }}" - register: service_configuration_raw - -- name: Parse service's configuration - set_fact: - service_configuration: "{{ service_configuration_raw['content'] | b64decode | from_yaml }}" - service_configuration_text: "{{ service_configuration_raw['content'] | b64decode }}" - -- name: Add DB Tag to Erigon - lineinfile: - path: "{{ item.path }}" - insertafter: "- erigon" - line: "{{ service_configuration_text | split('\n') | select('match', '^\\s*- --datadir.*') | first | split('-') | first }}- --db.size.limit=8TB" - when: - - service_configuration.service == "ErigonService" - - service_configuration.command | select('match', '--db.size.limit.*') | length == 0 diff --git a/controls/roles/update-changes/tasks/2.0.2/updates-202.yaml b/controls/roles/update-changes/tasks/2.0.2/updates-202.yaml deleted file mode 100644 index b1ec10acd..000000000 --- a/controls/roles/update-changes/tasks/2.0.2/updates-202.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- name: Find service configs - find: - paths: "/etc/stereum/services" - register: service_config_files - -- name: Include Erigon DB Tag - include_tasks: erigon_dbTag.yaml - loop: "{{ service_config_files.files }}" diff --git a/controls/roles/update-changes/tasks/2.0.3/nethermind_pruning.yaml b/controls/roles/update-changes/tasks/2.0.3/nethermind_pruning.yaml deleted file mode 100644 index 2096ce29e..000000000 --- a/controls/roles/update-changes/tasks/2.0.3/nethermind_pruning.yaml +++ /dev/null @@ -1,19 +0,0 @@ ---- -- name: Read service file - slurp: - src: "{{ item.path }}" - register: service_configuration_raw - -- name: Parse service's configuration - set_fact: - service_configuration: "{{ service_configuration_raw['content'] | b64decode | from_yaml }}" - service_configuration_text: "{{ service_configuration_raw['content'] | b64decode }}" - -- name: Add Pruning Tag to Nethermind - lineinfile: - path: "{{ item.path }}" - insertafter: "--Pruning.Mode" - line: "{{ service_configuration_text | split('\n') | select('match', '^\\s*- --Pruning.Mode.*') | first | split('-') | first }}- --Pruning.FullPruningTrigger=StateDbSize" - when: - - service_configuration.service == "NethermindService" - - service_configuration.command | select('match', '--Pruning.FullPruningTrigger.*') | length == 0 diff --git a/controls/roles/update-changes/tasks/2.0.3/updates-203.yaml b/controls/roles/update-changes/tasks/2.0.3/updates-203.yaml deleted file mode 100644 index 96e1c32b2..000000000 --- a/controls/roles/update-changes/tasks/2.0.3/updates-203.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- name: Find service configs - find: - paths: "/etc/stereum/services" - register: service_config_files - -- name: Include Nethermind Pruning Tag - include_tasks: nethermind_pruning.yaml - loop: "{{ service_config_files.files }}" diff --git a/controls/roles/update-changes/tasks/2.1.3/besu_changes.yaml b/controls/roles/update-changes/tasks/2.1.3/besu_changes.yaml deleted file mode 100644 index 01bd3a95f..000000000 --- a/controls/roles/update-changes/tasks/2.1.3/besu_changes.yaml +++ /dev/null @@ -1,29 +0,0 @@ ---- -- name: Read service file - slurp: - src: "{{ item.path }}" - register: service_configuration_raw - -- name: Parse service's configuration - set_fact: - service_configuration: "{{ service_configuration_raw['content'] | b64decode | from_yaml }}" - service_configuration_text: "{{ service_configuration_raw['content'] | b64decode }}" - -- name: Besu Changes - block: - - name: Replace X_SNAP with SNAP - replace: - path: "{{ item.path }}" - regexp: "(?i)X_SNAP" - replace: "SNAP" - - name: Replace X_CHECKPOINT with CHECKPOINT - replace: - path: "{{ item.path }}" - regexp: "(?i)X_CHECKPOINT" - replace: "CHECKPOINT" - - name: Remove --engine-jwt-enabled tag - lineinfile: - path: "{{ item.path }}" - regexp: "--engine-jwt-enabled" - state: absent - when: service_configuration.service == "BesuService" diff --git a/controls/roles/update-changes/tasks/2.1.3/updates-213.yaml b/controls/roles/update-changes/tasks/2.1.3/updates-213.yaml deleted file mode 100644 index a9ca15aae..000000000 --- a/controls/roles/update-changes/tasks/2.1.3/updates-213.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- name: Find service configs - find: - paths: "/etc/stereum/services" - register: service_config_files - -- name: Include Besu Changes - include_tasks: besu_changes.yaml - loop: "{{ service_config_files.files }}" diff --git a/controls/roles/update-changes/tasks/2.1.4/sdvt_changes.yaml b/controls/roles/update-changes/tasks/2.1.4/sdvt_changes.yaml deleted file mode 100644 index 1e7b6fe94..000000000 --- a/controls/roles/update-changes/tasks/2.1.4/sdvt_changes.yaml +++ /dev/null @@ -1,37 +0,0 @@ ---- -- name: Read service file - slurp: - src: "{{ item.path }}" - register: service_configuration_raw - -- name: Parse service's configuration - set_fact: - service_configuration: "{{ service_configuration_raw['content'] | b64decode | from_yaml }}" - service_configuration_text: "{{ service_configuration_raw['content'] | b64decode }}" - -- name: Adjust Lodestar Config - lineinfile: - path: "{{ item.path }}" - insertafter: "--distributed" - line: "{{ service_configuration_text | split('\n') | select('match', '^\\s*- --distributed') | first | split('-') | first }}- --useProduceBlockV3=false" - when: - - service_configuration.service == "LodestarValidatorService" - - service_configuration.command | select('match', '--useProduceBlockV3=false') | length == 0 - -- name: Adjust Teku Config - lineinfile: - path: "{{ item.path }}" - insertafter: "--network" - line: "{{ service_configuration_text | split('\n') | select('match', '^\\s*- --network') | first | split('-') | first }}- --Xblock-v3-enabled=false" - when: - - service_configuration.service == "TekuValidatorService" - - service_configuration.command | select('match', '--Xblock-v3-enabled=false') | length == 0 - -- name: Adjust Lighthouse Config - lineinfile: - path: "{{ item.path }}" - insertafter: "--builder-proposals" - line: "{{ service_configuration_text | split('\n') | select('match', '^\\s*- --builder-proposals') | first | split('-') | first }}- --produce-block-v3=false" - when: - - service_configuration.service == "LighthouseValidatorService" - - service_configuration.command | select('match', '--produce-block-v3=false') | length == 0 diff --git a/controls/roles/update-changes/tasks/2.1.4/updates-214.yaml b/controls/roles/update-changes/tasks/2.1.4/updates-214.yaml deleted file mode 100644 index 63d638eeb..000000000 --- a/controls/roles/update-changes/tasks/2.1.4/updates-214.yaml +++ /dev/null @@ -1,33 +0,0 @@ ---- -- name: Check if there is a CharonService - find: - paths: "/etc/stereum/services" - read_whole_file: true - contains: "^service: CharonService" - register: charon_service_files - -- name: Set CharonService id - set_fact: - charon_service_id: "{{ charon_service_files.files[0].path | basename | split('.') | first }}" - when: charon_service_files.matched > 0 - -- debug: - var: charon_service_id - when: charon_service_files.matched > 0 - -- name: Find service configs which contain CharonService ID - find: - paths: "/etc/stereum/services" - read_whole_file: true - contains: "{{ charon_service_id }}" - register: service_config_files - when: charon_service_files.matched > 0 - -- debug: - var: service_config_files - when: charon_service_files.matched > 0 - -- name: Include SDVT Changes - include_tasks: sdvt_changes.yaml - loop: "{{ service_config_files.files }}" - when: charon_service_files.matched > 0 diff --git a/controls/roles/update-changes/tasks/2.1.5/charon_changes.yaml b/controls/roles/update-changes/tasks/2.1.5/charon_changes.yaml new file mode 100644 index 000000000..193b94d62 --- /dev/null +++ b/controls/roles/update-changes/tasks/2.1.5/charon_changes.yaml @@ -0,0 +1,27 @@ +--- +- name: Read service file + slurp: + src: "{{ config_file_path }}" + register: service_configuration_raw + +- name: Parse service's configuration + set_fact: + service_configuration: "{{ service_configuration_raw['content'] | b64decode | from_yaml }}" + service_configuration_text: "{{ service_configuration_raw['content'] | b64decode }}" + +- name: Adjust Charon Config if connected to a NimbusBeaconService + lineinfile: + path: "{{ config_file_path }}" + insertafter: "--beacon-node-endpoints" + line: "{{ service_configuration_text | split('\n') | select('match', '^\\s*- --beacon-node-endpoints') | first | split('-') | first }}- --feature-set-enable=json_requests" + when: + - service_configuration.dependencies.consensusClients | selectattr('service', 'equalto', 'NimbusBeaconService') | list | length > 0 + - service_configuration.command | select('match', '--feature-set-enable=json_requests') | length == 0 + +- name: add builder tag if not present + lineinfile: + path: "{{ config_file_path }}" + insertafter: "--beacon-node-endpoints" + line: "{{ service_configuration_text | split('\n') | select('match', '^\\s*- --beacon-node-endpoints') | first | split('-') | first }}- --builder-api" + when: + - service_configuration.command | select('match', '--builder-api') | length == 0 \ No newline at end of file diff --git a/controls/roles/update-changes/tasks/2.1.5/charon_teku_changes.yaml b/controls/roles/update-changes/tasks/2.1.5/charon_teku_changes.yaml new file mode 100644 index 000000000..845db5559 --- /dev/null +++ b/controls/roles/update-changes/tasks/2.1.5/charon_teku_changes.yaml @@ -0,0 +1,31 @@ +--- +- name: Read service file + slurp: + src: "{{ item.path }}" + register: service_configuration_raw + +- name: Parse service's configuration + set_fact: + service_configuration: "{{ service_configuration_raw['content'] | b64decode | from_yaml }}" + service_configuration_text: "{{ service_configuration_raw['content'] | b64decode }}" + +- name: Check if service_configuration.id is in any of the filtered_files + shell: "grep -lq '{{ service_configuration.id }}' {{ item }}" + loop: "{{ filtered_files }}" + ignore_errors: true + register: grep_result + changed_when: false + +- name: debug grep_result + debug: + var: grep_result + +- name: Adjust Teku Config if charon service connected to it + lineinfile: + path: "{{ item.path }}" + insertafter: "--network" + line: "{{ service_configuration_text | split('\n') | select('match', '^\\s*- --network') | first | split('-') | first }}- --validators-graffiti-client-append-format=DISABLED" + when: + - grep_result.results | selectattr('rc', 'equalto', 0) | list | length > 0 + - service_configuration.service == "TekuBeaconService" + - service_configuration.command | select('match', '--validators-graffiti-client-append-format') | length == 0 \ No newline at end of file diff --git a/controls/roles/update-changes/tasks/2.1.5/sdvt_changes.yaml b/controls/roles/update-changes/tasks/2.1.5/sdvt_changes.yaml new file mode 100644 index 000000000..205658082 --- /dev/null +++ b/controls/roles/update-changes/tasks/2.1.5/sdvt_changes.yaml @@ -0,0 +1,34 @@ +--- +- name: Read service file + slurp: + src: "{{ config_file_path }}" + register: service_configuration_raw + +- name: Parse service's configuration + set_fact: + service_configuration: "{{ service_configuration_raw['content'] | b64decode | from_yaml }}" + service_configuration_text: "{{ service_configuration_raw['content'] | b64decode }}" + +- name: Adjust Lodestar Config + lineinfile: + path: "{{ config_file_path }}" + regexp: "--useProduceBlockV3" + state: absent + when: + - service_configuration.service == "LodestarValidatorService" + +- name: Adjust Teku Config + lineinfile: + path: "{{ config_file_path }}" + regexp: "--Xblock-v3-enabled" + state: absent + when: + - service_configuration.service == "TekuValidatorService" + +- name: Adjust Lighthouse Config + lineinfile: + path: "{{ config_file_path }}" + regexp: "--produce-block-v3" + state: absent + when: + - service_configuration.service == "LighthouseValidatorService" diff --git a/controls/roles/update-changes/tasks/2.1.5/updates-215.yaml b/controls/roles/update-changes/tasks/2.1.5/updates-215.yaml index 9bba3e0df..12974d13b 100644 --- a/controls/roles/update-changes/tasks/2.1.5/updates-215.yaml +++ b/controls/roles/update-changes/tasks/2.1.5/updates-215.yaml @@ -7,3 +7,54 @@ - name: Include ValidatorEjector Changes include_tasks: ValidatorEjector_changes.yaml loop: "{{ service_config_files.files }}" + +- name: Filter files that contain both regex patterns + shell: "grep -rlE 'service: CharonService' /etc/stereum/services | xargs grep -lE 'image: obolnetwork/charon:v1\\.[0-9]+\\.[0-9]+'" + register: grep_result + failed_when: false + changed_when: false + +- name: Set fact for filtered files + set_fact: + filtered_files: "{{ grep_result.stdout_lines }}" + +- name: Extract UUIDs from filtered filenames + set_fact: + uuids: "{{ filtered_files | map('basename') | map('regex_replace', '\\.yaml$', '') | list }}" + when: filtered_files | length > 0 + +- name: Check if files contain any of the extracted UUIDs + shell: "grep -lE '{{ uuids | join('|') }}' {{ item.path }}" + with_items: "{{ service_config_files.files }}" + register: uuid_grep_result + failed_when: false + changed_when: false + when: uuids | length > 0 + +- name: Set fact for files containing UUIDs + set_fact: + files_with_uuids: "{{ uuid_grep_result.results | selectattr('rc', 'eq', 0) | map(attribute='item.path') | list }}" + when: uuids | length > 0 + +- name: Debug files with UUIDs + debug: + var: files_with_uuids + when: files_with_uuids | length > 0 + +- name: Include SDVT Changes + include_tasks: sdvt_changes.yaml + loop: "{{ files_with_uuids }}" + loop_control: + loop_var: config_file_path + when: files_with_uuids | length > 0 + +- name: Include Charon Changes + include_tasks: charon_changes.yaml + loop: "{{ filtered_files }}" + loop_control: + loop_var: config_file_path + when: filtered_files | length > 0 + +- name: Include Charon Teku Changes + include_tasks: charon_teku_changes.yaml + loop: "{{ service_config_files.files }}" \ No newline at end of file diff --git a/controls/roles/update-changes/tasks/main.yml b/controls/roles/update-changes/tasks/main.yml index 7c67cbf83..6c1610374 100644 --- a/controls/roles/update-changes/tasks/main.yml +++ b/controls/roles/update-changes/tasks/main.yml @@ -1,24 +1,3 @@ ---- -- name: Include RC23 Update Scripts - include_tasks: "2.0-rc23/updates-20-rc23.yaml" - ignore_errors: yes - -- name: Include 2.0.2 Update Scripts - include_tasks: "2.0.2/updates-202.yaml" - ignore_errors: yes - -- name: Include 2.0.3 Update Scripts - include_tasks: "2.0.3/updates-203.yaml" - ignore_errors: yes - -- name: Include 2.1.3 Update Scripts - include_tasks: "2.1.3/updates-213.yaml" - ignore_errors: yes - -- name: Include 2.1.4 Update Scripts - include_tasks: "2.1.4/updates-214.yaml" - ignore_errors: yes - - name: Include 2.1.5 Update Scripts include_tasks: "2.1.5/updates-215.yaml" ignore_errors: yes diff --git a/launcher/src/backend/ethereum-services/CharonService.js b/launcher/src/backend/ethereum-services/CharonService.js index 138d08291..08d5ec9d0 100755 --- a/launcher/src/backend/ethereum-services/CharonService.js +++ b/launcher/src/backend/ethereum-services/CharonService.js @@ -43,9 +43,21 @@ export class CharonService extends NodeService { null, // executionClients consensusClients // consensusClients ); + + if (consensusClients.map(s => s.service).includes("NimbusBeaconService")) { + service.command.push("--feature-set-enable=json_requests") + } + + if (consensusClients.map(s => s.service).includes("TekuBeaconService")) { + consensusClients.filter(s => s.service === "TekuBeaconService").forEach(s => { + s.command.push(`--validators-graffiti-client-append-format=DISABLED`) + }) + } + return service; } + static buildByConfiguration(config) { const service = new CharonService();