From e80d48ac1603a59c1f3cf0e7b5bbbafde0ae3c2f Mon Sep 17 00:00:00 2001 From: master Date: Wed, 9 Aug 2023 18:41:40 +0300 Subject: [PATCH 1/9] add moonbeam/moonriver support --- exporters/moonbeam/Dockerfile | 21 + exporters/moonbeam/exporter.py | 246 +++++ exporters/moonbeam/requirements.txt | 5 + grafana/provisioning/dashboards/moonbeam.json | 959 ++++++++++++++++++ moonbeam.env | 2 + moonbeam.yml | 17 + moonriver.env | 2 + moonriver.yml | 17 + 8 files changed, 1269 insertions(+) create mode 100644 exporters/moonbeam/Dockerfile create mode 100755 exporters/moonbeam/exporter.py create mode 100644 exporters/moonbeam/requirements.txt create mode 100644 grafana/provisioning/dashboards/moonbeam.json create mode 100644 moonbeam.env create mode 100644 moonbeam.yml create mode 100644 moonriver.env create mode 100644 moonriver.yml diff --git a/exporters/moonbeam/Dockerfile b/exporters/moonbeam/Dockerfile new file mode 100644 index 0000000..b3cfdb9 --- /dev/null +++ b/exporters/moonbeam/Dockerfile @@ -0,0 +1,21 @@ +FROM alpine/flake8:latest as linter +WORKDIR /apps/ +COPY . /apps/ +## ingore E501 line too long (XX > 79 characters) +RUN flake8 --ignore="E501" *.py + +FROM --platform=linux/amd64 python:3.11-slim-buster + +ARG exporter + +WORKDIR / + +RUN apt-get update && apt-get install -y gcc g++ +COPY requirements.txt requirements.txt +RUN pip3 install -r requirements.txt --no-cache-dir +RUN groupadd -r exporter && useradd -r -g exporter exporter + +COPY --from=linter /apps/${exporter}.py app.py + +USER exporter +CMD ["python3", "app.py"] diff --git a/exporters/moonbeam/exporter.py b/exporters/moonbeam/exporter.py new file mode 100755 index 0000000..0907dba --- /dev/null +++ b/exporters/moonbeam/exporter.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 + +import threading +import requests +import yaml +import time +import json +import logging +from numpy import median, average, percentile +from decimal import * +from collections import deque +from flask import Flask, request, make_response + +logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %I:%M:%S') +app = Flask(__name__) + +@app.route("/metrics") +def metrics(): + chain = get_config('chain') + metrics = q_metrics[0].copy() + + out = "" + + try: + out += '# HELP moonbeam_currentRound Current round\n' + out += '# TYPE moonbeam_round Current round counter\n' + + out += 'moonbeam_currentRound{chain="%s"} %s\n' % (chain,metrics['common']['current_round']) + except KeyError: + pass + + try: + out += '# HELP moonbeam_activeCollatorsCount Active collators\n' + out += '# TYPE moonbeam_activeCollatorsCount Active collators counter\n' + + out += 'moonbeam_activeCollatorsCount{chain="%s"} %s\n' % (chain,metrics['common']['active_collators']) + except KeyError: + pass + + try: + out += '# HELP moonbeam_roundBlocks Round blocks\n' + out += '# TYPE moonbeam_roundBlocks Round blocks counter\n' + + out += 'moonbeam_roundBlocks{chain="%s"} %s\n' % (chain,metrics['common']['rnd_blocks']) + except KeyError: + pass + + try: + out += '# HELP moonbeam_blocksAvg Blocks avarage\n' + out += '# TYPE moonbeam_blocksAvg Blocks avarage counter\n' + + out += 'moonbeam_blocksAvg{chain="%s"} %s\n' % (chain,metrics['common']['average']) + except KeyError: + pass + + try: + out += '# HELP moonbeam_blocksMedian Blocks median\n' + out += '# TYPE moonbeam_blocksMedian Blocks median counter\n' + + out += 'moonbeam_blocksMedian{chain="%s"} %s\n' % (chain,metrics['common']['median']) + except KeyError: + pass + + try: + out += '# HELP moonbeam_blocksP95 Blocks p95\n' + out += '# TYPE moonbeam_blocksP95 Blocks p95 counter\n' + + out += 'moonbeam_blocksP95{chain="%s"} %s\n' % (chain,metrics['common']['p95']) + except KeyError: + pass + + try: + out += '# HELP moonbeam_activeCollators Active collator\n' + out += '# TYPE moonbeam_activeCollators Active collator counter\n' + + for k,v in metrics['collators'].items(): + out += 'moonbeam_activeCollators{node="%s",chain="%s",account="%s"} %s\n' % (v['node'],chain,k,v['is_active']) + except KeyError: + pass + + try: + out += '# HELP moonbeam_blockAuthorship Block authors\n' + out += '# TYPE moonbeam_blockAuthorship Block authors counter\n' + + for k,v in metrics['collators'].items(): + out += 'moonbeam_blockAuthorship{node="%s",chain="%s",account="%s"} %s\n' % (v['node'],chain,k,v['authored_blocks_count']) + except KeyError: + pass + + try: + out += '# HELP moonbeam_activesetPosition Active set position\n' + out += '# TYPE moonbeam_activesetPosition Active set position\n' + + for k,v in metrics['collators'].items(): + out += 'moonbeam_activesetPosition{node="%s",chain="%s",account="%s"} %s\n' % (v['node'],chain,k,v['set_position']) + except KeyError: + pass + + try: + out += '# HELP moonbeam_rooundProgress Round progress\n' + out += '# TYPE moonbeam_roundProgress Round progress counter\n' + + out += 'moonbeam_roundProgress{chain="%s"} %s\n' % (chain,metrics['common']['round_progress']) + except KeyError: + pass + + response = make_response(out, 200) + response.mimetype = "text/plain" + + return response + +def api_request(method = None,args = None): + url = get_config('api_substrate') + + if isinstance(args, list): + for i in range(len(args)): + if isinstance(args[i], str): + args[i] = '"' + args[i] + '"' + elif isinstance(args, str): + args = '"' + args + '"' + elif not args: + args = "" + + data = {'method': method, + 'args': args} + + try: + r = requests.post(url, json=data) + except (ConnectionRefusedError,requests.exceptions.ConnectionError) as e: + logging.critical(e) + return None + + if r.status_code == 200: + return r.json()['result'] + else: + logging.critical('Request to ' + url + ' finished with code ' + str(r.status_code)) + return None + +def get_round_progress(chain): + constants = { + 'moonbeam':{'round_length':1800}, + 'moonriver':{'round_length':600} + } + + round_length = constants[chain]['round_length'] + round_first_block = api_request(method = 'api.query.parachainStaking.round')['first'] + current_block = int(api_request(method = 'api.query.system.number'),16) + round_progress = (int(current_block) - int(round_first_block)) / round_length * 100 + + return round_progress + + +def get_activeset_position(account): + candidate_pool = api_request(method = 'api.query.parachainStaking.candidatePool') + sorted_pool = sorted(candidate_pool, key=lambda i: i['amount'], reverse = True) + set_index = next((index for (index, d) in enumerate(sorted_pool) if d["owner"] == account), None) + 1 + return set_index + +def get_config(part): + with open('./config.yaml') as config_file: + data = yaml.load(config_file, Loader=yaml.FullLoader) + + return data[part] + +def main(): + block = 0 + rnd = 0 + + while True: + try: + current_rnd = api_request(method = 'api.query.parachainStaking.round')['current'] + if current_rnd != rnd: + rnd_blocks_count = 0 + + active_collators = api_request(method = 'api.query.parachainStaking.selectedCandidates') + common = {} + common['active_collators'] = len(active_collators) + common['current_round'] = current_rnd + common['rnd_blocks'] = 0 + + for k,v in collators.items(): + v['authored_blocks_count'] = 0 + v['is_disabled'] = 0 + if k in active_collators: + v['is_active'] = 1 + v['set_position'] = get_activeset_position(k) + else: + v['is_active'] = 0 + v['set_position'] = 0 + + other_collators = {k:0 for k in active_collators if k not in collators.keys()} + + result = {'collators':collators, 'common':common} + logging.info('New round ' + str(current_rnd) + ' has just begun') + + last_block = int(api_request(method = 'api.query.system.number'),16) + + if last_block != block: + logging.info('Processing block ' + str(last_block)) + + block_author = api_request(method = 'api.query.authorInherent.author') + + if block_author in other_collators.keys(): + other_collators[block_author] += 1 + + for addr,params in result['collators'].items(): + if block_author == addr: + params['authored_blocks_count'] += 1 + logging.info('Collator ' + str(addr) + ' has just constructed block ' + str(last_block)) + + result['common']['round_progress'] = int(get_round_progress(get_config('chain'))) + result['common']['rnd_blocks'] += 1 + + result['common']['median'] = int(Decimal(median(list(other_collators.values())))) + result['common']['average'] = int(Decimal(average(list(other_collators.values())))) + result['common']['p95'] = int(Decimal(percentile(list(other_collators.values()),95))) + + q_metrics.clear() + q_metrics.append(result) + + rnd = current_rnd + block = last_block + + except Exception as e: + logging.critical('The main thread been stucked with error "' + str(e) + '"') + time.sleep(5) + continue + + time.sleep(3) + +if __name__ == '__main__': + endpoint_listen = get_config('exporter')['listen'] + endpoint_port = get_config('exporter')['port'] + + collators = {} + + for k,v in get_config('validators').items(): + collators[v['account']] = {'node':k} + + q_metrics = deque([]) + + worker = threading.Thread(target=main) + worker.daemon = True + worker.start() + + app.run(host="0.0.0.0", port=int(endpoint_port)) diff --git a/exporters/moonbeam/requirements.txt b/exporters/moonbeam/requirements.txt new file mode 100644 index 0000000..65365ba --- /dev/null +++ b/exporters/moonbeam/requirements.txt @@ -0,0 +1,5 @@ +pyaml==23.5.9 +requests +flask==2.3.2 +numpy==1.24.3 + diff --git a/grafana/provisioning/dashboards/moonbeam.json b/grafana/provisioning/dashboards/moonbeam.json new file mode 100644 index 0000000..90c12d0 --- /dev/null +++ b/grafana/provisioning/dashboards/moonbeam.json @@ -0,0 +1,959 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 32, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 280 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "custom.width", + "value": 384 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "node" + }, + "properties": [ + { + "id": "custom.width", + "value": 300 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Position" + }, + "properties": [ + { + "id": "custom.width", + "value": 466 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Node" + }, + "properties": [ + { + "id": "custom.width", + "value": 505 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 14, + "x": 0, + "y": 0 + }, + "id": 15, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Position" + } + ] + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max by (node) (moonbeam_activesetPosition{chain=\"$chain\",node=~\"$Node\"}[15s])", + "format": "table", + "instant": true, + "legendFormat": "{{ node }}", + "range": false, + "refId": "A" + } + ], + "title": "Active Set Position", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": {}, + "renameByName": { + "Time": "", + "Value": "Position", + "node": "Node" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 14, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "moonbeam_activeCollatorsCount{chain=\"$chain\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Collators count", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 14, + "y": 5 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "moonbeam_currentRound{chain=\"$chain\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Current round", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "color-background", + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "dark-red", + "value": 0 + }, + { + "color": "dark-yellow", + "value": 95 + }, + { + "color": "dark-green", + "value": 97 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "node" + }, + "properties": [ + { + "id": "custom.width", + "value": 273 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 14, + "x": 0, + "y": 10 + }, + "id": 6, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 1, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "node" + } + ] + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "max by (node) (sum_over_time(moonbeam_blockAuthorship{chain=\"$chain\",node=~\"$Node\"}[$time_period])) / max(sum_over_time(moonbeam_blocksAvg{chain=\"$chain\"}[$time_period])) * 100 and on (node) moonbeam_activeCollators == 1", + "format": "table", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "max by (node) (sum_over_time(moonbeam_blockAuthorship{chain=\"$chain\",node=~\"$Node\"}[$time_period])) / max(sum_over_time(moonbeam_blocksMedian{chain=\"$chain\"}[$time_period])) * 100 and on (node) moonbeam_activeCollators == 1", + "format": "table", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "max by (node) (sum_over_time(moonbeam_blockAuthorship{chain=\"$chain\",node=~\"$Node\"}[$time_period])) / max(sum_over_time(moonbeam_blocksP95{chain=\"$chain\"}[$time_period])) * 100 and on (node) moonbeam_activeCollators == 1", + "format": "table", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "C" + } + ], + "title": "ParaValidator epoch points", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "node" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true + }, + "indexByName": {}, + "renameByName": { + "Value #A": "AVG", + "Value #B": "Median", + "Value #C": "p95", + "Value #D": "GRANDPA precommits", + "node": "" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 14, + "y": 10 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "max_over_time(moonbeam_blocksP95{chain=\"$chain\"}[1m])", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "P95 blocks", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 14, + "y": 13 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "max_over_time(moonbeam_blocksMedian{chain=\"$chain\"}[1m])", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Median blocks", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 14, + "y": 16 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "max_over_time(moonbeam_blocksAvg{chain=\"$chain\"}[1m])", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "AVG blocks", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "moonbeam_blockAuthorship{chain=\"$chain\",node=~\"$Node\"} and on (account) moonbeam_activeCollators == 1", + "interval": "", + "legendFormat": "{{ node }}", + "refId": "A" + } + ], + "title": "Moonbeam block authorship", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "moonbeam_roundBlocks{chain=\"$chain\"}", + "interval": "", + "legendFormat": "{{ node }}", + "refId": "A" + } + ], + "title": "Round blocks", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "moonriver", + "value": "moonriver" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "chain", + "options": [ + { + "selected": false, + "text": "moonbeam", + "value": "moonbeam" + }, + { + "selected": true, + "text": "moonriver", + "value": "moonriver" + } + ], + "query": "moonbeam,moonriver", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(node_boot_time_seconds, job)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "Node", + "options": [], + "query": { + "query": "label_values(node_boot_time_seconds, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "(.+$chain.+|$chain.+)", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "1h", + "value": "1h" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "time_period", + "options": [ + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": true, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "3h", + "value": "3h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "24h", + "value": "24h" + } + ], + "query": "15m,1h,3h,6h,12h,24h", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Moonbeam/Moonriver", + "uid": "UEQXXV4Vk", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/moonbeam.env b/moonbeam.env new file mode 100644 index 0000000..e2e422a --- /dev/null +++ b/moonbeam.env @@ -0,0 +1,2 @@ +WS_ENDPOINT="wss://moonbeam.public.blastapi.io" +WS_ENDPOINTS="wss://moonbeam.public.blastapi.io" diff --git a/moonbeam.yml b/moonbeam.yml new file mode 100644 index 0000000..535fcce --- /dev/null +++ b/moonbeam.yml @@ -0,0 +1,17 @@ +version: '3.4' + +services: + moonbeam_exporter: + build: + context: ./exporters/moonbeam + args: + exporter: "moonbeam_exporter" + environment: + - "LISTEN=0.0.0.0" + - "PORT=9150" + - "CHAIN=moonbeam" + env_file: + - ./moonbeam.env + networks: + - exporters + diff --git a/moonriver.env b/moonriver.env new file mode 100644 index 0000000..319d1d4 --- /dev/null +++ b/moonriver.env @@ -0,0 +1,2 @@ +WS_ENDPOINT="wss://moonriver.public.blastapi.io" +WS_ENDPOINTS="wss://moonriver.public.blastapi.io" diff --git a/moonriver.yml b/moonriver.yml new file mode 100644 index 0000000..58eb415 --- /dev/null +++ b/moonriver.yml @@ -0,0 +1,17 @@ +version: '3.4' + +services: + moonriver_exporter: + build: + context: ./exporters/moonbeam + args: + exporter: "moonriver_exporter" + environment: + - "LISTEN=0.0.0.0" + - "PORT=9150" + - "CHAIN=moonriver" + env_file: + - ./moonriver.env + networks: + - exporters + From 58b274dff07b5bc06cd688b1bb6f39ca82dba13e Mon Sep 17 00:00:00 2001 From: sergeyradchenkop2p Date: Tue, 22 Aug 2023 11:41:19 +0300 Subject: [PATCH 2/9] moonbeam common exporter for MaaS --- exporters/moonbeam/config.yaml | 6 +++ exporters/moonbeam/exporter.py | 74 ++++++++--------------------- exporters/moonbeam/requirements.txt | 7 ++- 3 files changed, 28 insertions(+), 59 deletions(-) create mode 100644 exporters/moonbeam/config.yaml diff --git a/exporters/moonbeam/config.yaml b/exporters/moonbeam/config.yaml new file mode 100644 index 0000000..03f1d54 --- /dev/null +++ b/exporters/moonbeam/config.yaml @@ -0,0 +1,6 @@ +--- +api_substrate: https://moonbeam-api.polka.p2p.world/api +chain: moonbeam +exporter: + listen: 0.0.0.0 + port: 9618 diff --git a/exporters/moonbeam/exporter.py b/exporters/moonbeam/exporter.py index 0907dba..73be592 100755 --- a/exporters/moonbeam/exporter.py +++ b/exporters/moonbeam/exporter.py @@ -57,7 +57,7 @@ def metrics(): out += '# HELP moonbeam_blocksMedian Blocks median\n' out += '# TYPE moonbeam_blocksMedian Blocks median counter\n' - out += 'moonbeam_blocksMedian{chain="%s"} %s\n' % (chain,metrics['common']['median']) + out += 'moonbeam_blocksMedian{chain="%s" } %s\n' % (chain,metrics['common']['median']) except KeyError: pass @@ -69,33 +69,15 @@ def metrics(): except KeyError: pass - try: - out += '# HELP moonbeam_activeCollators Active collator\n' - out += '# TYPE moonbeam_activeCollators Active collator counter\n' - - for k,v in metrics['collators'].items(): - out += 'moonbeam_activeCollators{node="%s",chain="%s",account="%s"} %s\n' % (v['node'],chain,k,v['is_active']) - except KeyError: - pass - try: out += '# HELP moonbeam_blockAuthorship Block authors\n' out += '# TYPE moonbeam_blockAuthorship Block authors counter\n' for k,v in metrics['collators'].items(): - out += 'moonbeam_blockAuthorship{node="%s",chain="%s",account="%s"} %s\n' % (v['node'],chain,k,v['authored_blocks_count']) + out += 'moonbeam_blockAuthorship{chain="%s",account="%s"} %s\n' % (chain,k,v) except KeyError: pass - try: - out += '# HELP moonbeam_activesetPosition Active set position\n' - out += '# TYPE moonbeam_activesetPosition Active set position\n' - - for k,v in metrics['collators'].items(): - out += 'moonbeam_activesetPosition{node="%s",chain="%s",account="%s"} %s\n' % (v['node'],chain,k,v['set_position']) - except KeyError: - pass - try: out += '# HELP moonbeam_rooundProgress Round progress\n' out += '# TYPE moonbeam_roundProgress Round progress counter\n' @@ -149,13 +131,6 @@ def get_round_progress(chain): return round_progress - -def get_activeset_position(account): - candidate_pool = api_request(method = 'api.query.parachainStaking.candidatePool') - sorted_pool = sorted(candidate_pool, key=lambda i: i['amount'], reverse = True) - set_index = next((index for (index, d) in enumerate(sorted_pool) if d["owner"] == account), None) + 1 - return set_index - def get_config(part): with open('./config.yaml') as config_file: data = yaml.load(config_file, Loader=yaml.FullLoader) @@ -177,43 +152,32 @@ def main(): common['active_collators'] = len(active_collators) common['current_round'] = current_rnd common['rnd_blocks'] = 0 - - for k,v in collators.items(): - v['authored_blocks_count'] = 0 - v['is_disabled'] = 0 - if k in active_collators: - v['is_active'] = 1 - v['set_position'] = get_activeset_position(k) - else: - v['is_active'] = 0 - v['set_position'] = 0 - - other_collators = {k:0 for k in active_collators if k not in collators.keys()} - - result = {'collators':collators, 'common':common} + authored_blocks_count = 0 + all_collators = {k:authored_blocks_count for k in active_collators} + result = {'collators':all_collators, 'common':common} logging.info('New round ' + str(current_rnd) + ' has just begun') - + last_block = int(api_request(method = 'api.query.system.number'),16) - - if last_block != block: + + if last_block != block: logging.info('Processing block ' + str(last_block)) - + block_author = api_request(method = 'api.query.authorInherent.author') - if block_author in other_collators.keys(): - other_collators[block_author] += 1 - for addr,params in result['collators'].items(): + if block_author == addr: - params['authored_blocks_count'] += 1 - logging.info('Collator ' + str(addr) + ' has just constructed block ' + str(last_block)) + result['collators'][addr] += 1 + + + logging.info('Collator ' + str(addr) + ' has just constructed block ' + str(last_block)) result['common']['round_progress'] = int(get_round_progress(get_config('chain'))) result['common']['rnd_blocks'] += 1 - result['common']['median'] = int(Decimal(median(list(other_collators.values())))) - result['common']['average'] = int(Decimal(average(list(other_collators.values())))) - result['common']['p95'] = int(Decimal(percentile(list(other_collators.values()),95))) + result['common']['median'] = int(Decimal(median(list(all_collators.values())))) + result['common']['average'] = int(Decimal(average(list(all_collators.values())))) + result['common']['p95'] = int(Decimal(percentile(list(all_collators.values()),95))) q_metrics.clear() q_metrics.append(result) @@ -234,8 +198,8 @@ def main(): collators = {} - for k,v in get_config('validators').items(): - collators[v['account']] = {'node':k} + # for k,v in get_config('validators').items(): + # collators[v['account']] = {'node':k} q_metrics = deque([]) diff --git a/exporters/moonbeam/requirements.txt b/exporters/moonbeam/requirements.txt index 65365ba..f363289 100644 --- a/exporters/moonbeam/requirements.txt +++ b/exporters/moonbeam/requirements.txt @@ -1,5 +1,4 @@ -pyaml==23.5.9 +numpy +flask requests -flask==2.3.2 -numpy==1.24.3 - +pyyaml From c275225ea87a6254ed87d4a561aa79466d0e82c9 Mon Sep 17 00:00:00 2001 From: sergeyradchenkop2p Date: Thu, 14 Dec 2023 13:22:46 +0300 Subject: [PATCH 3/9] astar & acala --- exporters/acala/Dockerfile | 10 ++ exporters/acala/config.yaml | 9 ++ exporters/acala/exporter.py | 165 +++++++++++++++++++ exporters/acala/requirements.txt | 3 + exporters/astar/Dockerfile | 10 ++ exporters/astar/config.yaml | 10 ++ exporters/astar/exporter.py | 263 +++++++++++++++++++++++++++++++ exporters/astar/requirements.txt | 4 + 8 files changed, 474 insertions(+) create mode 100644 exporters/acala/Dockerfile create mode 100644 exporters/acala/config.yaml create mode 100644 exporters/acala/exporter.py create mode 100644 exporters/acala/requirements.txt create mode 100644 exporters/astar/Dockerfile create mode 100644 exporters/astar/config.yaml create mode 100644 exporters/astar/exporter.py create mode 100644 exporters/astar/requirements.txt diff --git a/exporters/acala/Dockerfile b/exporters/acala/Dockerfile new file mode 100644 index 0000000..9d01651 --- /dev/null +++ b/exporters/acala/Dockerfile @@ -0,0 +1,10 @@ +FROM --platform=linux/amd64 python:3.9.10-slim-buster + +WORKDIR / + +COPY requirements.txt requirements.txt +RUN pip3 install -r requirements.txt + +COPY exporter.py app.py + +CMD ["python3", "app.py"] diff --git a/exporters/acala/config.yaml b/exporters/acala/config.yaml new file mode 100644 index 0000000..517b43e --- /dev/null +++ b/exporters/acala/config.yaml @@ -0,0 +1,9 @@ +--- +api_substrate: http://127.0.0.1:3000/api +chain: acala +exporter: + listen: 0.0.0.0 + port: 9618 +validators: + scaleway-acala-collator1: + account: 21nrT9Jf6KmboAEJnyE4As3H18pZ8xbJYkyWpCEJEshPXGzS diff --git a/exporters/acala/exporter.py b/exporters/acala/exporter.py new file mode 100644 index 0000000..adb46be --- /dev/null +++ b/exporters/acala/exporter.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 + +import threading +import requests +import yaml +import time +import json +import logging +from collections import deque +from flask import Flask, request, make_response + +logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %I:%M:%S') +app = Flask(__name__) + +@app.route("/metrics") +def metrics(): + chain = get_config('chain') + metrics = q_metrics[0].copy() + + out = "" + + try: + out += '# HELP acala_session_common Common metrics\n' + out += '# TYPE acala_session_common Common metrics counter\n' + + for k,v in metrics['common'].items(): + out += 'acala_session_common{name="%s",chain="%s"} %s\n' % (k,chain,v) + except KeyError: + pass + + try: + out += '# HELP acala_session_active_validators Active validators\n' + out += '# TYPE acala_session_active_validators Active validators counter\n' + + for k,v in metrics['validators'].items(): + out += 'acala_session_active_validators{node="%s",chain="%s",account_addr="%s"} %s\n' % (v['node'],chain,k,v['is_active']) + except KeyError: + pass + + try: + out += '# HELP acala_session_disabled_validators Disabled validators\n' + out += '# TYPE acala_session_disabled_validators Disabled validators counter\n' + + for k,v in metrics['validators'].items(): + out += 'acala_session_disabled_validators{node="%s",chain="%s",account_addr="%s"} %s\n' % (v['node'],chain,k,v['is_disabled']) + except KeyError: + pass + + try: + out += '# HELP acala_rewards_validator Points earned\n' + out += '# TYPE acala_rewards_validator Points earned counter\n' + + for k,v in metrics['validators'].items(): + out += 'acala_rewards_validator{node="%s",chain="%s",account_addr="%s"} %s\n' % (v['node'],chain,k,v['points']) + except KeyError: + pass + + response = make_response(out, 200) + response.mimetype = "text/plain" + + return response + +def api_request(endpoint = "api_substrate",method = None,args = None): + if endpoint == 'api_substrate': + url = get_config('api_substrate') + + if isinstance(args, list): + for i in range(len(args)): + if isinstance(args[i], str): + args[i] = '"' + args[i] + '"' + elif isinstance(args, str): + args = '"' + args + '"' + elif not args: + args = "" + + data = {'method': method, + 'args': args} + + elif endpoint == 'api_registry': + url = get_config('api_registry') + + data = {'method': method} + + try: + r = requests.post(url, json=data) + except (ConnectionRefusedError,requests.exceptions.ConnectionError): + logging.critical('Coulnd not get data from ' + endpoint) + + return None + + if r.status_code == 200: + return r.json()['result'] + else: + logging.critical('Request to ' + endpoint + ' finished with code ' + str(r.status_code)) + return None + +def get_config(part): + with open('./config.yaml') as config_file: + data = yaml.load(config_file, Loader=yaml.FullLoader) + + return data[part] + +def main(): + block = 0 + session = 0 + + while True: + try: + last_block = int(api_request(method = 'api.query.system.number'),16) + + if last_block != block: + result = {'validators':validators,'common':{}} + active_validators = api_request(method = 'api.query.session.validators') + current_session = int(api_request(method = 'api.query.session.currentIndex'),16) + disabled_validators = api_request(method = 'api.query.session.disabledValidators') + + result['common'] = {} + result['common']['active_validators_count'] = len(active_validators) + result['common']['current_session'] = current_session + + for addr,params in result['validators'].items(): + if addr in active_validators: + validator_idx = active_validators.index(addr) + points = int(api_request(method = 'api.query.collatorSelection.sessionPoints', args = addr),16) + params['points'] = points + params['is_active'] = 1 + params['is_disabled'] = 0 + else: + validator_idx = None + params['is_active'] = 0 + params['points'] = 0 + + if validator_idx in disabled_validators: + params['is_disabled'] = 1 + params['is_active'] = 0 + + q_metrics.clear() + q_metrics.append(result) + + session = current_session + block = last_block + + except Exception as e: + logging.critical(e) + time.sleep(3) + continue + + time.sleep(3) + +if __name__ == '__main__': + endpoint_listen = get_config('exporter')['listen'] + endpoint_port = get_config('exporter')['port'] + + validators = {} + + for k,v in get_config('validators').items(): + validators[v['account']] = {'node':k} + + q_metrics = deque([]) + + worker = threading.Thread(target=main) + worker.daemon = True + worker.start() + + app.run(host="0.0.0.0", port=int(endpoint_port)) diff --git a/exporters/acala/requirements.txt b/exporters/acala/requirements.txt new file mode 100644 index 0000000..09a313b --- /dev/null +++ b/exporters/acala/requirements.txt @@ -0,0 +1,3 @@ +flask +requests +pyyaml diff --git a/exporters/astar/Dockerfile b/exporters/astar/Dockerfile new file mode 100644 index 0000000..9d01651 --- /dev/null +++ b/exporters/astar/Dockerfile @@ -0,0 +1,10 @@ +FROM --platform=linux/amd64 python:3.9.10-slim-buster + +WORKDIR / + +COPY requirements.txt requirements.txt +RUN pip3 install -r requirements.txt + +COPY exporter.py app.py + +CMD ["python3", "app.py"] diff --git a/exporters/astar/config.yaml b/exporters/astar/config.yaml new file mode 100644 index 0000000..6418622 --- /dev/null +++ b/exporters/astar/config.yaml @@ -0,0 +1,10 @@ +--- +api_substrate: https://shiden-api.polka.p2p.world/api +api_registry: https://registry-para-api.polka.p2p.world/api +chain: shiden +exporter: + listen: 0.0.0.0 + port: 9618 +validators: + scaleway-shiden-collator1: + account: 'ZPsVRDiDTsWr9wENxE5iyBFCd2ySckF9mn8PtGSevnKjMwU' diff --git a/exporters/astar/exporter.py b/exporters/astar/exporter.py new file mode 100644 index 0000000..b0ec44b --- /dev/null +++ b/exporters/astar/exporter.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 + +import threading +import requests +import yaml +import time +import json +import logging +from numpy import median, average, percentile +from decimal import * +from collections import deque +from flask import Flask, request, make_response + +logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %I:%M:%S') +app = Flask(__name__) + +@app.route("/metrics") +def metrics(): + chain = get_config('chain') + metrics = q_metrics[0].copy() + + out = "" + + try: + out += '# HELP astar_currentEra Current era\n' + out += '# TYPE astar_currentEra Current era counter\n' + + out += 'astar_currentEra{chain="%s"} %s\n' % (chain,metrics['common']['current_era']) + except KeyError: + pass + + try: + out += '# HELP astar_currentSession Current session\n' + out += '# TYPE astar_currentSession Current session counter\n' + + out += 'astar_currentSession{chain="%s"} %s\n' % (chain,metrics['common']['current_session']) + except KeyError: + pass + + try: + out += '# HELP astar_activeCollatorsCount Active collators\n' + out += '# TYPE astar_activeCollatorsCount Active collators counter\n' + + out += 'astar_activeCollatorsCount{chain="%s"} %s\n' % (chain,metrics['common']['active_collators_count']) + except KeyError: + pass + + try: + out += '# HELP astar_sessionBlocks Session blocks\n' + out += '# TYPE astar_sessionBlocks Session blocks counter\n' + + out += 'astar_sessionBlocks{chain="%s"} %s\n' % (chain,metrics['common']['session_blocks']) + except KeyError: + pass + + try: + out += '# HELP astar_blocksAvg Blocks avarage\n' + out += '# TYPE astar_blocksAvg Blocks avarage counter\n' + + out += 'astar_blocksAvg{chain="%s"} %s\n' % (chain,metrics['common']['average']) + except KeyError: + pass + + try: + out += '# HELP astar_blocksMedian Blocks median\n' + out += '# TYPE astar_blocksMedian Blocks median counter\n' + + out += 'astar_blocksMedian{chain="%s"} %s\n' % (chain,metrics['common']['median']) + except KeyError: + pass + + try: + out += '# HELP astar_blocksP95 Blocks p95\n' + out += '# TYPE astar_blocksP95 Blocks p95 counter\n' + + out += 'astar_blocksP95{chain="%s"} %s\n' % (chain,metrics['common']['p95']) + except KeyError: + pass + + try: + out += '# HELP astar_activeCollators Active collators\n' + out += '# TYPE astar_activeCollators Active collators counter\n' + + for k,v in metrics['collators'].items(): + + out += 'astar_activeCollators{node="%s",chain="%s",account="%s"} %s\n' % (v['node'],chain,k,v['is_active']) + + except KeyError: + pass + + try: + out += '# HELP astar_blockAuthorship Blocks authored\n' + out += '# TYPE astar_blockAuthorship Blocks authored counter\n' + + for k,v in metrics['collators'].items(): + out += 'astar_blockAuthorship{node="%s",chain="%s",account="%s"} %s\n' % (v['node'],chain,k,v['authored_blocks_count']) + except KeyError: + pass + + + response = make_response(out, 200) + response.mimetype = "text/plain" + + return response + +def api_request(method = None,args = None): + url = get_config('api_substrate') + + if isinstance(args, list): + for i in range(len(args)): + if isinstance(args[i], str): + args[i] = '"' + args[i] + '"' + elif isinstance(args, str): + args = '"' + args + '"' + elif not args: + args = "" + + data = {'method': method, + 'args': args} + + try: + r = requests.post(url, json=data) + except (ConnectionRefusedError,requests.exceptions.ConnectionError) as e: + logging.critical(e) + return None + + if r.status_code == 200: + return r.json()['result'] + else: + logging.critical('Request to ' + url + ' finished with code ' + str(r.status_code)) + return None + +def get_config(part): + with open('./config.yaml') as config_file: + data = yaml.load(config_file, Loader=yaml.FullLoader) + + return data[part] + +def get_block_count(addr): + try: + authored_block = int(api_request(method = 'api.query.collatorSelection.lastAuthoredBlock', args = addr),16) + + for i in q_collators: + idx = q_collators.index(i) + if addr in i.keys(): + if authored_block not in q_collators[idx][addr]: + q_collators[idx][addr].append(authored_block) + + except Exception as e: + logging.critical('The collators thread been stucked with error "' + str(e) + '"') + +def main(): + block = 0 + era = 0 + session = 0 + + while True: + try: + current_era = int(api_request(method = 'api.query.dappsStaking.currentEra')[2:],16) + current_session = int(api_request(method = 'api.query.session.currentIndex')[2:],16) + + if era != current_era: + logging.info('New era ' + str(current_era) + ' has just begun') + + if session != current_session: + processed = [] + q_collators.clear() + active_collators = api_request(method = 'api.query.session.validators') + disabled_collators = api_request(method = 'api.query.session.disabledValidators') + + for addr in active_collators: + if addr not in collators.keys() and addr not in processed: + q_collators.append({addr:[]}) + processed.append(addr) + + for k,v in collators.items(): + if v in active_collators: + collator_idx = active_collators.index(k) + v['authored_blocks_count'] = 0 + v['is_active'] = 1 + result = {'collators':collators, 'common':{}} + else: + result = {'collators':{}, 'common':{}} + v['is_active'] = 0 + break + + result['common']['current_era'] = current_era + result['common']['current_session'] = current_session + result['common']['session_blocks'] = 0 + logging.info('New session ' + str(current_session) + ' has just begun') + + last_block = int(api_request(method = 'api.query.system.number'),16) + + if last_block != block: + logging.info('Processing block ' + str(last_block)) + + blocks_per_collator = [] + if result['collators'].items(): + for addr,params in result['collators'].items(): + + if addr in active_collators: + authored_block = api_request(method = 'api.query.collatorSelection.lastAuthoredBlock', args = addr) + authored_block = int(authored_block, 16) + + if 'last_authored_block' not in params: + params['last_authored_block'] = authored_block + + if params['last_authored_block'] != authored_block: + params['authored_blocks_count'] += 1 + logging.info('Collator ' + str(addr) + ' has just constructed block ' + str(authored_block)) + + params['last_authored_block'] = authored_block + + threads = [] + + for addr in active_collators: + if addr not in collators.keys(): + th = threading.Thread(target=get_block_count,args=(addr,)) + threads.append(th) + + for t in threads: + t.start() + t.join() + + for c in q_collators: + blocks_per_collator.append(len(list(c.values())[0])) + + result['common']['session_blocks'] += 1 + result['common']['median'] = int(Decimal(median(blocks_per_collator))) + result['common']['average'] = int(Decimal(average(blocks_per_collator))) + result['common']['p95'] = int(Decimal(percentile(blocks_per_collator,95))) + + q_metrics.clear() + q_metrics.append(result) + + era = current_era + session = current_session + block = last_block + + except Exception as e: + logging.critical('The main thread been stucked with error "' + str(e) + '"') + time.sleep(10) + continue + + time.sleep(3) + +if __name__ == '__main__': + endpoint_listen = get_config('exporter')['listen'] + endpoint_port = get_config('exporter')['port'] + + collators = {} + + for k,v in get_config('validators').items(): + collators[v['account']] = {'node':k} + + q_metrics = deque([]) + q_collators = deque([]) + + worker = threading.Thread(target=main) + worker.daemon = True + worker.start() + + app.run(host="0.0.0.0", port=int(endpoint_port)) diff --git a/exporters/astar/requirements.txt b/exporters/astar/requirements.txt new file mode 100644 index 0000000..f363289 --- /dev/null +++ b/exporters/astar/requirements.txt @@ -0,0 +1,4 @@ +numpy +flask +requests +pyyaml From f56094ace64a31467b204c7689bd338e3ae6a491 Mon Sep 17 00:00:00 2001 From: sergeyradchenkop2p Date: Thu, 14 Dec 2023 20:16:43 +0300 Subject: [PATCH 4/9] acala --- exporters/acala/config.yaml | 5 +---- exporters/acala/exporter.py | 30 ++++++++++-------------------- 2 files changed, 11 insertions(+), 24 deletions(-) diff --git a/exporters/acala/config.yaml b/exporters/acala/config.yaml index 517b43e..5e5c3e8 100644 --- a/exporters/acala/config.yaml +++ b/exporters/acala/config.yaml @@ -1,9 +1,6 @@ --- -api_substrate: http://127.0.0.1:3000/api +api_substrate: https://acala-api.polka.p2p.world/api chain: acala exporter: listen: 0.0.0.0 port: 9618 -validators: - scaleway-acala-collator1: - account: 21nrT9Jf6KmboAEJnyE4As3H18pZ8xbJYkyWpCEJEshPXGzS diff --git a/exporters/acala/exporter.py b/exporters/acala/exporter.py index adb46be..60ec637 100644 --- a/exporters/acala/exporter.py +++ b/exporters/acala/exporter.py @@ -109,27 +109,22 @@ def main(): last_block = int(api_request(method = 'api.query.system.number'),16) if last_block != block: - result = {'validators':validators,'common':{}} - active_validators = api_request(method = 'api.query.session.validators') + validators = api_request(method = 'api.query.session.validators') + + active_validators = {k:points for k in validators} current_session = int(api_request(method = 'api.query.session.currentIndex'),16) disabled_validators = api_request(method = 'api.query.session.disabledValidators') - + result = {'validators':active_validators,'common':{}} result['common'] = {} result['common']['active_validators_count'] = len(active_validators) result['common']['current_session'] = current_session - for addr,params in result['validators'].items(): - if addr in active_validators: - validator_idx = active_validators.index(addr) - points = int(api_request(method = 'api.query.collatorSelection.sessionPoints', args = addr),16) - params['points'] = points - params['is_active'] = 1 - params['is_disabled'] = 0 - else: - validator_idx = None - params['is_active'] = 0 - params['points'] = 0 - + validator_idx = active_validators.index(addr) + points = int(api_request(method = 'api.query.collatorSelection.sessionPoints', args = addr),16) + params['points'] = points + params['is_active'] = 1 + params['is_disabled'] = 0 + if validator_idx in disabled_validators: params['is_disabled'] = 1 params['is_active'] = 0 @@ -151,11 +146,6 @@ def main(): endpoint_listen = get_config('exporter')['listen'] endpoint_port = get_config('exporter')['port'] - validators = {} - - for k,v in get_config('validators').items(): - validators[v['account']] = {'node':k} - q_metrics = deque([]) worker = threading.Thread(target=main) From eac7ad36c3e15680bbedbe7d586fce707973ca2a Mon Sep 17 00:00:00 2001 From: sergeyradchenkop2p Date: Sun, 17 Dec 2023 18:10:06 +0300 Subject: [PATCH 5/9] acala exporter --- exporters/acala/.exporter.py.swp | Bin 0 -> 1024 bytes exporters/acala/exporter.py | 35 +++++++++---------------------- 2 files changed, 10 insertions(+), 25 deletions(-) create mode 100644 exporters/acala/.exporter.py.swp diff --git a/exporters/acala/.exporter.py.swp b/exporters/acala/.exporter.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..34dd0dee5f9ca632ecd5c5986e7dbc59ee47e24a GIT binary patch literal 1024 zcmYc?$V<%2S1{Ex(_=se>|zWI#YKrJ$r-76+4;yanC#rd;*!)NOmTFP)QW=qBA`mW Vf=YC$QT}KMjD`U1Lm(903IM;S4%Pqw literal 0 HcmV?d00001 diff --git a/exporters/acala/exporter.py b/exporters/acala/exporter.py index 60ec637..5f340e1 100644 --- a/exporters/acala/exporter.py +++ b/exporters/acala/exporter.py @@ -24,7 +24,7 @@ def metrics(): out += '# TYPE acala_session_common Common metrics counter\n' for k,v in metrics['common'].items(): - out += 'acala_session_common{name="%s",chain="%s"} %s\n' % (k,chain,v) + out += 'acala_session_common{name="%s",chain="%s"} %s\n' % (k,chain) except KeyError: pass @@ -33,16 +33,7 @@ def metrics(): out += '# TYPE acala_session_active_validators Active validators counter\n' for k,v in metrics['validators'].items(): - out += 'acala_session_active_validators{node="%s",chain="%s",account_addr="%s"} %s\n' % (v['node'],chain,k,v['is_active']) - except KeyError: - pass - - try: - out += '# HELP acala_session_disabled_validators Disabled validators\n' - out += '# TYPE acala_session_disabled_validators Disabled validators counter\n' - - for k,v in metrics['validators'].items(): - out += 'acala_session_disabled_validators{node="%s",chain="%s",account_addr="%s"} %s\n' % (v['node'],chain,k,v['is_disabled']) + out += 'acala_session_active_validators{node="%s",chain="%s",account_addr="%s"} %s\n' % (chain,k) except KeyError: pass @@ -51,7 +42,7 @@ def metrics(): out += '# TYPE acala_rewards_validator Points earned counter\n' for k,v in metrics['validators'].items(): - out += 'acala_rewards_validator{node="%s",chain="%s",account_addr="%s"} %s\n' % (v['node'],chain,k,v['points']) + out += 'acala_rewards_validator{node="%s",chain="%s",account_addr="%s"} %s\n' % (chain,k) except KeyError: pass @@ -110,25 +101,19 @@ def main(): if last_block != block: validators = api_request(method = 'api.query.session.validators') - - active_validators = {k:points for k in validators} current_session = int(api_request(method = 'api.query.session.currentIndex'),16) disabled_validators = api_request(method = 'api.query.session.disabledValidators') - result = {'validators':active_validators,'common':{}} + result = {'validators':{},'common':{}} result['common'] = {} - result['common']['active_validators_count'] = len(active_validators) + result['common']['active_validators_count'] = len(validators) result['common']['current_session'] = current_session - for addr,params in result['validators'].items(): - validator_idx = active_validators.index(addr) + for addr in validators: points = int(api_request(method = 'api.query.collatorSelection.sessionPoints', args = addr),16) - params['points'] = points - params['is_active'] = 1 - params['is_disabled'] = 0 + validator_points = {k:points for k in validators if k == addr} + result['validators'].update(validator_points) + - if validator_idx in disabled_validators: - params['is_disabled'] = 1 - params['is_active'] = 0 - + print (result) q_metrics.clear() q_metrics.append(result) From 952a7edf0d107621aed79454350f6b48408cac05 Mon Sep 17 00:00:00 2001 From: sergeyradchenkop2p Date: Sun, 17 Dec 2023 18:10:29 +0300 Subject: [PATCH 6/9] acala exporter --- exporters/acala/.exporter.py.swp | Bin 1024 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 exporters/acala/.exporter.py.swp diff --git a/exporters/acala/.exporter.py.swp b/exporters/acala/.exporter.py.swp deleted file mode 100644 index 34dd0dee5f9ca632ecd5c5986e7dbc59ee47e24a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1024 zcmYc?$V<%2S1{Ex(_=se>|zWI#YKrJ$r-76+4;yanC#rd;*!)NOmTFP)QW=qBA`mW Vf=YC$QT}KMjD`U1Lm(903IM;S4%Pqw From f4e9d1546909b071c7ae29f3f96e001d67f7fc24 Mon Sep 17 00:00:00 2001 From: sergeyradchenkop2p Date: Sun, 17 Dec 2023 22:27:32 +0300 Subject: [PATCH 7/9] acala fix --- exporters/acala/exporter.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/exporters/acala/exporter.py b/exporters/acala/exporter.py index 5f340e1..68df88a 100644 --- a/exporters/acala/exporter.py +++ b/exporters/acala/exporter.py @@ -24,7 +24,7 @@ def metrics(): out += '# TYPE acala_session_common Common metrics counter\n' for k,v in metrics['common'].items(): - out += 'acala_session_common{name="%s",chain="%s"} %s\n' % (k,chain) + out += 'acala_session_common{name="%s",chain="%s"} %s\n' % (k,chain,v) except KeyError: pass @@ -32,8 +32,8 @@ def metrics(): out += '# HELP acala_session_active_validators Active validators\n' out += '# TYPE acala_session_active_validators Active validators counter\n' - for k,v in metrics['validators'].items(): - out += 'acala_session_active_validators{node="%s",chain="%s",account_addr="%s"} %s\n' % (chain,k) + for k in metrics['validators'].items(): + out += 'acala_session_active_validators{chain="%s"} %s\n' % (chain,k) except KeyError: pass @@ -41,8 +41,9 @@ def metrics(): out += '# HELP acala_rewards_validator Points earned\n' out += '# TYPE acala_rewards_validator Points earned counter\n' - for k,v in metrics['validators'].items(): - out += 'acala_rewards_validator{node="%s",chain="%s",account_addr="%s"} %s\n' % (chain,k) + for k in metrics: + print (k[validators]) + out += 'acala_rewards_validator{chain="%s" } %s\n' % (chain,k) except KeyError: pass @@ -111,12 +112,9 @@ def main(): points = int(api_request(method = 'api.query.collatorSelection.sessionPoints', args = addr),16) validator_points = {k:points for k in validators if k == addr} result['validators'].update(validator_points) - - - print (result) + q_metrics.clear() q_metrics.append(result) - session = current_session block = last_block From 54db5c9cef4cc8b74e6eaa90abac3f223fecccab Mon Sep 17 00:00:00 2001 From: sergeyradchenkop2p Date: Mon, 18 Dec 2023 12:13:19 +0300 Subject: [PATCH 8/9] temp fix --- exporters/acala/exporter.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/exporters/acala/exporter.py b/exporters/acala/exporter.py index 60ec637..71a18d4 100644 --- a/exporters/acala/exporter.py +++ b/exporters/acala/exporter.py @@ -110,8 +110,8 @@ def main(): if last_block != block: validators = api_request(method = 'api.query.session.validators') - - active_validators = {k:points for k in validators} + point = 0 + active_validators = {k:point for k in validators} current_session = int(api_request(method = 'api.query.session.currentIndex'),16) disabled_validators = api_request(method = 'api.query.session.disabledValidators') result = {'validators':active_validators,'common':{}} @@ -119,15 +119,15 @@ def main(): result['common']['active_validators_count'] = len(active_validators) result['common']['current_session'] = current_session for addr,params in result['validators'].items(): - validator_idx = active_validators.index(addr) + #validator_idx = active_validators.index(addr) points = int(api_request(method = 'api.query.collatorSelection.sessionPoints', args = addr),16) - params['points'] = points + print (points) params['is_active'] = 1 params['is_disabled'] = 0 - if validator_idx in disabled_validators: - params['is_disabled'] = 1 - params['is_active'] = 0 + # if validator_idx in disabled_validators: + # params['is_disabled'] = 1 + # params['is_active'] = 0 q_metrics.clear() q_metrics.append(result) From 0779a914b6f86ea27ffca2cc2231b5857f77f298 Mon Sep 17 00:00:00 2001 From: sergeyradchenkop2p Date: Mon, 18 Dec 2023 12:38:47 +0300 Subject: [PATCH 9/9] acala final --- exporters/acala/exporter.py | 137 ++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 exporters/acala/exporter.py diff --git a/exporters/acala/exporter.py b/exporters/acala/exporter.py new file mode 100644 index 0000000..78680a0 --- /dev/null +++ b/exporters/acala/exporter.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 + +import threading +import requests +import yaml +import time +import json +import logging +from collections import deque +from flask import Flask, request, make_response + +logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %I:%M:%S') +app = Flask(__name__) + +@app.route("/metrics") +def metrics(): + chain = get_config('chain') + metrics = q_metrics[0].copy() + + out = "" + + try: + out += '# HELP acala_session_common Common metrics\n' + out += '# TYPE acala_session_common Common metrics counter\n' + + for k,v in metrics['common'].items(): + out += 'acala_session_common{name="%s",chain="%s"} %s\n' % (k,chain,v) + except KeyError: + pass + + try: + out += '# HELP acala_session_active_validators Active validators\n' + out += '# TYPE acala_session_active_validators Active validators counter\n' + + for k,v in metrics['validators'].items(): + out += 'acala_session_active_validators{chain="%s"} %s\n' % (chain,k) + except KeyError: + pass + + try: + out += '# HELP acala_rewards_validator Points earned\n' + out += '# TYPE acala_rewards_validator Points earned counter\n' + + for k in metrics['validators'].items(): + out += 'acala_rewards_validator{chain="%s" } %s\n' % (chain,k) + except KeyError: + pass + + response = make_response(out, 200) + response.mimetype = "text/plain" + + return response + +def api_request(endpoint = "api_substrate",method = None,args = None): + if endpoint == 'api_substrate': + url = get_config('api_substrate') + + if isinstance(args, list): + for i in range(len(args)): + if isinstance(args[i], str): + args[i] = '"' + args[i] + '"' + elif isinstance(args, str): + args = '"' + args + '"' + elif not args: + args = "" + + data = {'method': method, + 'args': args} + + elif endpoint == 'api_registry': + url = get_config('api_registry') + + data = {'method': method} + + try: + r = requests.post(url, json=data) + except (ConnectionRefusedError,requests.exceptions.ConnectionError): + logging.critical('Coulnd not get data from ' + endpoint) + + return None + + if r.status_code == 200: + return r.json()['result'] + else: + logging.critical('Request to ' + endpoint + ' finished with code ' + str(r.status_code)) + return None + +def get_config(part): + with open('./config.yaml') as config_file: + data = yaml.load(config_file, Loader=yaml.FullLoader) + + return data[part] + +def main(): + block = 0 + session = 0 + + while True: + try: + last_block = int(api_request(method = 'api.query.system.number'),16) + + if last_block != block: + validators = api_request(method = 'api.query.session.validators') + current_session = int(api_request(method = 'api.query.session.currentIndex'),16) + disabled_validators = api_request(method = 'api.query.session.disabledValidators') + result = {'validators':{},'common':{}} + result['common'] = {} + result['common']['active_validators_count'] = len(validators) + result['common']['current_session'] = current_session + for addr in validators: + points = int(api_request(method = 'api.query.collatorSelection.sessionPoints', args = addr),16) + validator_points = {k:points for k in validators if k == addr} + result['validators'].update(validator_points) + + q_metrics.clear() + q_metrics.append(result) + session = current_session + block = last_block + + except Exception as e: + logging.critical(e) + time.sleep(3) + continue + + time.sleep(3) + +if __name__ == '__main__': + endpoint_listen = get_config('exporter')['listen'] + endpoint_port = get_config('exporter')['port'] + + q_metrics = deque([]) + + worker = threading.Thread(target=main) + worker.daemon = True + worker.start() + + app.run(host="0.0.0.0", port=int(endpoint_port))