diff --git a/avail.env b/avail.env new file mode 100644 index 0000000..3e9e210 --- /dev/null +++ b/avail.env @@ -0,0 +1,2 @@ +WS_ENDPOINT="wss://mainnet-rpc.avail.so/ws" +WS_ENDPOINTS="wss://mainnet-rpc.avail.so/ws" diff --git a/avail.yml b/avail.yml new file mode 100644 index 0000000..69750f1 --- /dev/null +++ b/avail.yml @@ -0,0 +1,32 @@ +version: '3.4' + +services: + avail_common_exporter: + build: + context: ./exporters/avail + args: + exporter: "common_exporter" + environment: + - "LISTEN=0.0.0.0" + - "PORT=9150" + - "CHAIN=avail" + env_file: + - ./avail.env + networks: + - exporters + restart: on-failure + + avail_finality_exporter: + build: + context: ./exporters/avail + args: + exporter: "finality_exporter" + environment: + - "LISTEN=0.0.0.0" + - "PORT=9150" + - "CHAIN=avail" + env_file: + - ./avail.env + networks: + - exporters + restart: on-failure diff --git a/bot/alerts_tmpl.yml b/bot/alerts_tmpl.yml index 35bfd77..b20e4d2 100644 --- a/bot/alerts_tmpl.yml +++ b/bot/alerts_tmpl.yml @@ -71,7 +71,7 @@ rules: description: "ParaValidator {{ $labels.account }} earned {{ $value }} points by end of epoch." chain: "{{ $labels.chain }}" account: "{{ $labels.account }}" - bot_description: "Will shout if some of selected validators been as a ParaValidator and earned epoch points less than threshold you selected by end of epoch.\nThreshold value - amount of reward points from 0 to 10000" + bot_description: "Will shout if some of selected validators been as a ParaValidator and earned epoch points less than threshold you selected by end of epoch.\nThreshold value - amount of reward points from 0 to 10000" - alert: "[Polkadot] Offline validators" expr: | @@ -223,12 +223,76 @@ rules: account: "{{ $labels.account }}" bot_description: "Will shout if amount of slashed validators in network will reach or more than threshold value.\nThreshold value - amount of validators." +- alert: "[Avail] Reward points by end of era" + expr: | + avail_staking_eraPoints{chain="[[chain]]", account=~"[[accounts]]"} < [[threshold]] and on(account) avail_session_validators == 1 and on (chain) avail_staking_eraProgress >= 95 + for: "[[interval]]" + labels: + uniqueid: 16 + chat_id: "[[chat_id]]" + chain: "[[chain]]" + labels_source: avail_staking_eraPoints + annotations: + summary: "{{ $labels.chain }}: Validator has earned points less than choosen." + description: "Validator {{ $labels.account }} earned {{ $value }} points by end of era." + chain: "{{ $labels.chain }}" + account: "{{ $labels.account }}" + bot_description: "Will shout if some of selected validators earned era points less than threshold you selected by end of era.\nThreshold value - amount of reward points from 0 to 20000" + +- alert: "[Avail] Finality GRANDPA precommits ratio" + expr: | + (avail_finality_precommits{chain="[[chain]]", account=~"[[accounts]]"} / on (chain) group_left() avail_finality_roundsProcessed * 100) < [[threshold]] and on(account) avail_session_validators == 1 and on (chain) avail_session_sessionProgress >= 3 + for: "[[interval]]" + labels: + uniqueid: 17 + chat_id: "[[chat_id]]" + chain: "[[chain]]" + labels_source: avail_finality_precommits + annotations: + summary: "{{ $labels.chain }}: Validator has problem with block finality participation!!!" + description: "Validator {{ $labels.account }} participate in finality for {{ $value }} percent." + chain: "{{ $labels.chain }}" + account: "{{ $labels.account }}" + bot_description: "Will shout if some of selected validators acts bellow threshold value as a consensus member(Precommits).\nThreshold value - percent of grandpa rounds when validator participated in consensus(0-100)" + +- alert: "[Avail] Unapplied slashes" + expr: | + avail_staking_slashedValidators{chain="[[chain]]", account=~"[[accounts]]"} == 1 + for: "[[interval]]" + labels: + uniqueid: 18 + chat_id: "[[chat_id]]" + chain: "[[chain]]" + labels_source: avail_staking_slashedValidators + annotations: + summary: "{{ $labels.chain }}: Validator has been slashed!!!" + description: "Validator {{ $labels.account }} has been slashed." + chain: "{{ $labels.chain }}" + account: "{{ $labels.account }}" + bot_description: "Will shout if some of selected validators has been slashed.\nData from storage - possible also to catch it from events." + +- alert: "[Avail] Unapplied slashes in a whole network" + expr: | + avail_staking_slashedValidatorsCount{chain="[[chain]]"} > [[threshold]] + for: "[[interval]]" + labels: + uniqueid: 19 + chat_id: "[[chat_id]]" + chain: "[[chain]]" + labels_source: avail_staking_slashedValidatorsCount + annotations: + summary: "{{ $labels.chain }}: Lots of slashed validators in the network!" + description: "Amount of slashed validators {{ $value }}" + chain: "{{ $labels.chain }}" + account: "{{ $labels.account }}" + bot_description: "Will shout if amount of slashed validators in network will reach or more than threshold value.\nThreshold value - amount of validators." + - alert: "[Acala] Offline collators" expr: | rate(acala_activeCollators{chain="acala", account=~"[[accounts]]"}[30m]) > 0 for: "[[interval]]" labels: - uniqueid: 16 + uniqueid: 20 chat_id: "[[chat_id]]" labels_source: acala_session_active_validators annotations: @@ -243,7 +307,7 @@ rules: rate(acala_activeCollators{chain="karura", account=~"[[accounts]]"}[30m]) > 0 for: "[[interval]]" labels: - uniqueid: 17 + uniqueid: 21 chat_id: "[[chat_id]]" labels_source: acala_session_active_validators annotations: @@ -259,7 +323,7 @@ rules: moonbeam_blockAuthorship{chain="moonbeam", account=~"[[accounts]]"} < [[threshold]] and on(account) moonbeam_activeCollators == 1 and on (chain) moonbeam_roundProgress >= 90 for: "[[interval]]" labels: - uniqueid: 18 + uniqueid: 22 chat_id: "[[chat_id]]" labels_source: moonbeam_blockAuthorship annotations: @@ -274,7 +338,7 @@ rules: moonbeam_blockAuthorship{chain="moonriver", account=~"[[accounts]]"} < [[threshold]] and on(account) moonbeam_activeCollators == 1 and on (chain) moonbeam_roundProgress >= 90 for: "[[interval]]" labels: - uniqueid: 19 + uniqueid: 23 chat_id: "[[chat_id]]" labels_source: moonbeam_blockAuthorship annotations: @@ -289,7 +353,7 @@ rules: rate(moonbeam_activeCollators{chain="moonbeam", account=~"[[accounts]]"}[30m]) > 0 for: "[[interval]]" labels: - uniqueid: 20 + uniqueid: 24 chat_id: "[[chat_id]]" labels_source: moonbeam_activeCollators annotations: @@ -304,7 +368,7 @@ rules: rate(moonbeam_activeCollators{chain="moonriver", account=~"[[accounts]]"}[30m]) > 0 for: "[[interval]]" labels: - uniqueid: 21 + uniqueid: 25 chat_id: "[[chat_id]]" labels_source: moonbeam_activeCollators annotations: @@ -319,7 +383,7 @@ rules: rate(astar_activeCollators{chain="astar", account=~"[[accounts]]"}[30m]) > 0 for: "[[interval]]" labels: - uniqueid: 22 + uniqueid: 26 chat_id: "[[chat_id]]" labels_source: astar_activeCollators annotations: @@ -334,7 +398,7 @@ rules: rate(astar_activeCollators{chain="shiden", account=~"[[accounts]]"}[30m]) > 0 for: "[[interval]]" labels: - uniqueid: 23 + uniqueid: 27 chat_id: "[[chat_id]]" labels_source: astar_activeCollators annotations: @@ -348,7 +412,7 @@ rules: expr: vector(1) for: "[[interval]]" labels: - uniqueid: 24 + uniqueid: 28 chat_id: "[[chat_id]]" annotations: summary: "This is a summary of test alert." diff --git a/bot/app/__main__.py b/bot/app/__main__.py index 626290a..003af54 100644 --- a/bot/app/__main__.py +++ b/bot/app/__main__.py @@ -4,6 +4,7 @@ import asyncio from aiohttp import web from aiogram import Bot, Dispatcher, Router +from aiogram.client.default import DefaultBotProperties from aiogram.fsm.storage.memory import MemoryStorage from message_handlers.setup import setup_message_handler from web_apps.setup import setup_web_app @@ -40,7 +41,7 @@ cache = CACHE(redis_host, redis_port, redis_password) web_app = web.Application() db = DB(db_name, db_user, db_pass,db_host,db_port) - bot = Bot(token=tg_token, parse_mode="HTML") + bot = Bot(token=tg_token, default=DefaultBotProperties(parse_mode='HTML')) storage = MemoryStorage() dp = Dispatcher(storage=storage) diff --git a/bot/app/callback_query_handlers/accounts.py b/bot/app/callback_query_handlers/accounts.py index e5c2375..e2fba12 100644 --- a/bot/app/callback_query_handlers/accounts.py +++ b/bot/app/callback_query_handlers/accounts.py @@ -18,7 +18,7 @@ async def acc_menu(query: CallbackQuery): menu = MenuBuilder() - text = "Here you can mange accounts you would like to track.\n\nFor now we are processing over " + str(cache.count()) + " uniq accounts of validators and collators.\n\nNetworks covered:\n🔸Polkadot/Kusama\n🔸Acala/Karura\n🔸Moonbeam/Moonriver\n🔸Astar/Shiden\n\n" + text = "Here you can mange accounts you would like to track.\n\nFor now we are processing over " + str(cache.count()) + " uniq accounts of validators and collators.\n\nNetworks covered:\n🔸Polkadot/Kusama\n🔸Avail/Turing(testnet)\n🔸Acala/Karura\n🔸Moonbeam/Moonriver\n🔸Astar/Shiden\n\n" if not validators: text += "☝️ No accounts in portfolio yet." diff --git a/exporters/avail/Dockerfile b/exporters/avail/Dockerfile new file mode 100644 index 0000000..5378502 --- /dev/null +++ b/exporters/avail/Dockerfile @@ -0,0 +1,22 @@ +FROM alpine/flake8:latest as linter +WORKDIR /apps/ +COPY . /apps/ +## ingore E501 line too long (XX > 79 characters) +RUN flake8 --ignore="E501" *.py + +FROM --platform=linux/amd64 python:3.11-slim-buster + +ARG exporter + +WORKDIR / + +RUN apt-get update && apt-get install -y gcc g++ +COPY requirements.txt requirements.txt +RUN pip3 install -r requirements.txt --no-cache-dir +RUN groupadd -r exporter && useradd -r -g exporter exporter + +COPY --from=linter /apps/${exporter}.py app.py +COPY --from=linter /apps/functions.py functions.py + +USER exporter +CMD ["python3", "app.py"] diff --git a/exporters/avail/common_exporter.py b/exporters/avail/common_exporter.py new file mode 100755 index 0000000..db16604 --- /dev/null +++ b/exporters/avail/common_exporter.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Polkadot validator monitoring services. +# +# Copyright 2023 P2P Validator. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +import threading +import logging +import operator +import traceback +from functions import SUBSTRATE_INTERFACE, get_era_points, get_chain_info +from _thread import interrupt_main +from collections import deque +from flask import Flask, make_response +from numpy import median, average, percentile +from decimal import Decimal + +logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %I:%M:%S') +app = Flask(__name__) + + +@app.route('/metrics', methods=['GET']) +def metrics(): + chain = os.environ['CHAIN'] + + if len(q_metrics) == 0: + response = make_response("", 200) + response.mimetype = "text/plain" + + return response + + metrics = q_metrics[0].copy() + + out = "" + + try: + out += '# HELP avail_staking_currentEra Current era\n' + out += '# TYPE avail_staking_currentEra counter\n' + + out += 'avail_staking_currentEra{chain="%s"} %s\n' % (chain, metrics['common']['currentEra']) + + except KeyError: + pass + + try: + out += '# HELP avail_staking_eraProgress Era progress\n' + out += '# TYPE avail_staking_eraProgress counter\n' + + out += 'avail_staking_eraProgress{chain="%s"} %s\n' % (chain, metrics['common']['eraProgress']) + + except KeyError: + pass + + try: + out += '# HELP avail_staking_totalPoints Total points\n' + out += '# TYPE avail_staking_totalPoints counter\n' + + out += 'avail_staking_totalPoints{chain="%s"} %s\n' % (chain, metrics['totalEraPoints']) + + except KeyError: + pass + + try: + out += "# HELP avail_staking_eraPoints Validator points\n" + out += "# TYPE avail_staking_eraPoints counter\n" + + for k, v in metrics['eraPoints'].items(): + out += 'avail_staking_eraPoints{chain="%s",account="%s"} %s\n' % (chain, k, v) + + except KeyError: + pass + + try: + out += "# HELP avail_staking_validatorsChart Validators position chart\n" + out += "# TYPE avail_staking_validatorsChart counter\n" + + for k, v in metrics['validatorsChart'].items(): + out += 'avail_staking_validatorPositionChart{chain="%s",account="%s"} %s\n' % (chain, k, v) + + except KeyError: + pass + + try: + out += "# HELP avail_staking_slashedValidators Unapplied slashes to exact validators\n" + out += "# TYPE avail_staking_slashedValidators counter\n" + + for k, v in metrics['slashedValidators'].items(): + out += 'avail_staking_slashedValidators{chain="%s", account="%s"} %s\n' % (chain, k, v) + + except KeyError: + pass + + try: + out += "# HELP avail_staking_slashedValidatorsCount Count of slashed validators in excat network\n" + out += "# TYPE avail_staking_slashedValidatorsCount counter\n" + + out += 'avail_staking_slashedValidatorsCount{chain="%s"} %s\n' % (chain, metrics['slashedValidatorsCount']) + + except KeyError: + pass + + try: + out += '# HELP avail_session_currentSession Current session\n' + out += '# TYPE avail_session_currentSession counter\n' + + out += 'avail_session_currentSession{chain="%s"} %s\n' % (chain, metrics['common']['currentSession']) + + except KeyError: + pass + + try: + out += '# HELP avail_session_sessionProgress Session progress\n' + out += '# TYPE avail_session_sessionProgress counter\n' + + out += 'avail_session_sessionProgress{chain="%s"} %s\n' % (chain, metrics['common']['sessionProgress']) + + except KeyError: + pass + + try: + out += "# HELP avail_session_validators Session validators\n" + out += "# TYPE avail_session_validators counter\n" + + for k, v in metrics['sessionValidators'].items(): + out += 'avail_session_validators{chain="%s",account="%s"} %s\n' % (chain, k, v) + + except KeyError: + pass + + response = make_response(out, 200) + response.mimetype = "text/plain" + + return response + + +def get_unapplied_slashes(chain, era): + slashed_validators = [] + + max_era = era + 30 + + while era < max_era: + r = substrate_interface.request('Staking', 'UnappliedSlashes', [era]).value + + for i in r: + slashed_validators.append(i['validator']) + + era = era + 1 + + result = {'slashed_validators_count': 0, 'slashed_validators_list': ['null_validator']} + + for i in slashed_validators: + result['slashed_validators_count'] += 1 + result['slashed_validators_list'].append(i) + + return result + + +def construct_metrics(era, current_session, era_progress, session_progress, session_validators, slashed_validators, era_points): + result = {'common': {}} + + result['sessionValidators'] = {k: 1 for k in session_validators} + result['slashedValidators'] = {k: 1 for k in slashed_validators['slashed_validators_list']} + result['eraPoints'] = {k: v for k, v in era_points['result'].items() if k in session_validators} + result['totalEraPoints'] = era_points['total'] + result['slashedValidatorsCount'] = slashed_validators['slashed_validators_count'] + + result['common']['currentEra'] = era + result['common']['currentSession'] = current_session + result['common']['eraProgress'] = int(Decimal(era_progress)) + result['common']['sessionProgress'] = int(Decimal(session_progress)) + + result['common']['median'] = int(Decimal(median(list(result['eraPoints'].values())))) + result['common']['average'] = int(Decimal(average(list(result['eraPoints'].values())))) + result['common']['p95'] = int(Decimal(percentile(list(result['eraPoints'].values()), 95))) + + result['validatorsChart'] = {k: list(dict(sorted(era_points['result'].items(), key=operator.itemgetter(1), reverse=True)).keys()).index(k) for k in era_points['result'].keys() if k in session_validators} + + return result + + +def main(): + while True: + try: + chain_info = get_chain_info(chain, substrate_interface) + era = chain_info['current_era'] + current_session = chain_info['current_session'] + era_progress = chain_info['era_progress'] + session_progress = chain_info['session_progress'] + + era_points = get_era_points(substrate_interface.request('Staking', 'ErasRewardPoints', [era]).value) + session_validators = substrate_interface.request('Session', 'Validators').value + slashed_validators = get_unapplied_slashes(chain, era) + + metrics = construct_metrics(era, current_session, era_progress, session_progress, session_validators, slashed_validators, era_points) + + q_metrics.clear() + q_metrics.append(metrics) + + except Exception: + traceback.print_exc() + interrupt_main() + + time.sleep(15) + + +if __name__ == '__main__': + endpoint_listen = os.environ['LISTEN'] + endpoint_port = os.environ['PORT'] + ws_endpoint = os.environ['WS_ENDPOINT'] + chain = os.environ['CHAIN'] + + substrate_interface = SUBSTRATE_INTERFACE(ws_endpoint) + + q_state = deque([]) + q_metrics = deque([]) + + thread = threading.Thread(target=main) + thread.daemon = True + thread.start() + + app.run(host=endpoint_listen, port=int(endpoint_port)) diff --git a/exporters/avail/config.yaml b/exporters/avail/config.yaml new file mode 100644 index 0000000..f624900 --- /dev/null +++ b/exporters/avail/config.yaml @@ -0,0 +1,6 @@ +--- +ws_endpoint: ws://scaleway-moonbeam-node1:9934 +chain: moonbeam +exporter: + listen: 0.0.0.0 + port: 9618 diff --git a/exporters/avail/finality_exporter.py b/exporters/avail/finality_exporter.py new file mode 100644 index 0000000..e139d44 --- /dev/null +++ b/exporters/avail/finality_exporter.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Polkadot validator monitoring services. +# +# Copyright 2023 P2P Validator. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +import threading +import logging +import traceback +import random +from functions import SUBSTRATE_INTERFACE, get_chain_info, get_keys, ss58_convert +from _thread import interrupt_main +from collections import deque +from flask import Flask, make_response + +logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %I:%M:%S') +app = Flask(__name__) + + +@app.route("/metrics") +def metrics(): + chain = os.environ['CHAIN'] + + if len(q_metrics) == 0: + response = make_response("", 200) + response.mimetype = "text/plain" + + return response + + metrics = q_metrics[0] + + out = "" + + try: + out += '# HELP avail_finality_roundsProcessed Blocks processed\n' + out += '# TYPE avail_finality_roundsProcessed counter\n' + + out += 'avail_finality_roundsProcessed{chain="%s"} %s\n' % (chain, metrics['roundsProcessed']) + + except KeyError: + pass + + try: + out += "# HELP avail_finality_prevotes Prevotes\n" + out += "# TYPE avail_finality_prevotes counter\n" + + for k, v in metrics['validators'].items(): + out += 'avail_finality_prevotes{chain="%s",account="%s"} %s\n' % (chain, k, v['prevotes']) + + except KeyError: + pass + + try: + out += "# HELP avail_finality_precommits Precommits\n" + out += "# TYPE avail_finality_precommits counter\n" + + for k, v in metrics['validators'].items(): + out += 'avail_finality_precommits{chain="%s",account="%s"} %s\n' % (chain, k, v['precommits']) + + except KeyError: + pass + + response = make_response(out, 200) + response.mimetype = "text/plain" + + return response + + +def get_votes(url, substrate_interface): + rd = 0 + r = {} + + while True: + try: + d = substrate_interface.rpc_request(method='grandpa_roundState') + if url in q_outaged: + q_outaged.remove(url) + d = d['result']['best'] + if d['round'] != rd and len(r) != 0: + q_votes_raw.append(r) + + rd = d['round'] + r = {rd: {}} + + r[rd]['prevotes'] = d['prevotes']['missing'] + r[rd]['precommits'] = d['precommits']['missing'] + + except Exception: + if url not in q_outaged: + q_outaged.append(url) + time.sleep(1) + substrate_interface.connect_websocket() + pass + + +def construct_metrics(active_validators, grandpa_keys, votes_threshold, current_session): + data = q_votes_raw.copy() + result = {} + + try: + metrics = q_metrics[0] + except IndexError: + metrics = {} + metrics['currentSession'] = current_session + metrics['validators'] = {k: {'prevotes': 0, 'precommits': 0} for k in grandpa_keys.keys()} + metrics['roundsProcessed'] = 0 + + try: + if current_session != metrics['currentSession']: + logging.info('New session ' + str(current_session) + ' has just begun.') + metrics['validators'] = {k: {'prevotes': 0, 'precommits': 0} for k in grandpa_keys.keys()} + metrics['roundsProcessed'] = 0 + metrics['currentSession'] = current_session + except KeyError: + pass + + try: + for i in data: + for k, v in i.items(): + if k in q_rounds_processed: + continue + + prevotes = ss58_convert(v['prevotes']) + precommits = ss58_convert(v['precommits']) + + try: + result[k]['count'] += 1 + except KeyError: + result[k] = {'count': 1, 'prevotes': prevotes, 'precommits': precommits} + + if len(prevotes) < len(result[k]['prevotes']): + result[k]['prevotes'] = prevotes + + if len(precommits) < len(result[k]['precommits']): + result[k]['precommits'] = precommits + + if result[k]['count'] >= votes_threshold: + voted_prevotes = [] + voted_precommits = [] + + for account, params in metrics['validators'].items(): + if grandpa_keys[account] not in result[k]['prevotes']: + voted_prevotes.append(account) + params['prevotes'] += 1 + + if grandpa_keys[account] not in result[k]['precommits']: + voted_precommits.append(account) + params['precommits'] += 1 + + if 'roundsProcessed' not in metrics: + metrics['roundsProcessed'] = 1 + else: + metrics['roundsProcessed'] += 1 + + logging.info('Round ' + str(k) + ' has processed. Prevotes: ' + str(len(voted_prevotes)) + '. Precommits: ' + str(len(voted_precommits))) + q_rounds_processed.append(k) + + return metrics + except RuntimeError as e: + logging.info('Construct metrics func finished with error ' + str(e)) + pass + + +def main(): + substrate_interface = SUBSTRATE_INTERFACE(random.choice(rpc_endpoints), chain) + while True: + try: + for i in q_outaged: + logging.critical('RPC enpdoint ' + i + ' marked as outaged') + + chain_info = get_chain_info(chain, substrate_interface) + current_session = chain_info['current_session'] + votes_threshold = (rpc_count * thread_count) - (len(q_outaged) * thread_count) + active_validators = substrate_interface.request('Session', 'Validators').value + all_keys = substrate_interface.request('Session', 'QueuedKeys').value + grandpa_keys = get_keys(active_validators, all_keys) + + metrics = construct_metrics(active_validators, grandpa_keys, votes_threshold, current_session) + + q_metrics.clear() + q_metrics.append(metrics) + + except Exception: + traceback.print_exc() + interrupt_main() + + time.sleep(1) + + +if __name__ == '__main__': + endpoint_listen = os.environ['LISTEN'] + endpoint_port = os.environ['PORT'] + chain = os.environ['CHAIN'] + + rpc_endpoints = os.environ['WS_ENDPOINTS'].split(',') + + threads = [] + + rpc_count = len(rpc_endpoints) + thread_count = 3 + + q_metrics = deque([]) + q_votes_raw = deque([], maxlen=rpc_count * thread_count * 15) + q_rounds_processed = deque([], maxlen=100) + q_outaged = deque([], maxlen=30) + + for url in rpc_endpoints: + for i in range(thread_count): + th = threading.Thread(target=get_votes, args=(url, SUBSTRATE_INTERFACE(url))) + th.daemon = True + th.start() + time.sleep(0.2) + + worker = threading.Thread(target=main) + worker.daemon = True + worker.start() + + app.run(host=endpoint_listen, port=int(endpoint_port)) diff --git a/exporters/avail/functions.py b/exporters/avail/functions.py new file mode 100644 index 0000000..356a275 --- /dev/null +++ b/exporters/avail/functions.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Polkadot validator monitoring services. +# +# Copyright 2023 P2P Validator. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from substrateinterface import SubstrateInterface, Keypair +from substrateinterface.exceptions import SubstrateRequestException +from websocket._exceptions import WebSocketConnectionClosedException + + +class SUBSTRATE_INTERFACE: + def __init__(self, ws_endpoint): + self.substrate = SubstrateInterface( + url=ws_endpoint, + ss58_format=42) + + def request(self, module: str, function: str, params: str = None): + try: + r = self.substrate.query( + module=module, + storage_function=function, + params=params) + + return r + except (WebSocketConnectionClosedException, ConnectionRefusedError, SubstrateRequestException) as e: + self.substrate.connect_websocket() + logging.critical('The substrate api call failed with error ' + str(e)) + r = None + + def rpc_request(self, method: str, params: str = None): + try: + return self.substrate.rpc_request(method=method, params=params) + except Exception as e: + self.substrate.connect_websocket() + logging.critical('The substrate api request failed with error ' + str(e)) + + +def get_era_points(data): + result = {} + + for i in data['individual']: + result[i[0]] = i[1] + + return {'result': result, 'total': data['total']} + + +def get_chain_info(chain, substrate_interface): + session_length = 720 + era_length = 4320 + + current_era = substrate_interface.request('Staking', 'ActiveEra').value['index'] + current_session = substrate_interface.request('Session', 'CurrentIndex').value + + eras_start_session_index = substrate_interface.request('Staking', 'ErasStartSessionIndex', [current_era]).value + + genesis_slot = substrate_interface.request('Babe', 'GenesisSlot').value + current_slot = substrate_interface.request('Babe', 'CurrentSlot').value + + session_start_slot = int(current_session) * int(session_length) + int(genesis_slot) + session_progress = int(current_slot) - int(session_start_slot) + + era_session_index = int(current_session) - int(eras_start_session_index) + era_progress = int(era_session_index) * int(session_length) + int(session_progress) + + return {'current_era': current_era, 'eras_start_session_index': eras_start_session_index, 'current_session': current_session, 'era_progress': era_progress / era_length * 100, 'session_progress': session_progress / session_length * 100} + + +def ss58_convert(data): + r = [] + + for key in data: + pubkey = Keypair(ss58_address=key).public_key.hex() + r.append('0x' + pubkey) + + return r + + +def get_keys(validators, keys): + result = {i: None for i in validators} + + for i in keys: + if str(i[0]) in result.keys(): + result[str(i[0])] = str(i[1]['grandpa']) + + return result diff --git a/exporters/avail/requirements.txt b/exporters/avail/requirements.txt new file mode 100644 index 0000000..6188367 --- /dev/null +++ b/exporters/avail/requirements.txt @@ -0,0 +1,4 @@ +pyaml==23.5.9 +substrate-interface==1.7.1 +flask==2.3.2 +numpy==1.24.3 diff --git a/exporters/common/finality_exporter.py b/exporters/common/finality_exporter.py index f9eea51..cd27b1c 100755 --- a/exporters/common/finality_exporter.py +++ b/exporters/common/finality_exporter.py @@ -22,6 +22,7 @@ import threading import logging import traceback +import random from functions import SUBSTRATE_INTERFACE, get_chain_info, get_keys, ss58_convert from _thread import interrupt_main from collections import deque @@ -87,6 +88,8 @@ def get_votes(url, substrate_interface): while True: try: d = substrate_interface.rpc_request(method='grandpa_roundState') + if url in q_outaged: + q_outaged.remove(url) d = d['result']['best'] if d['round'] != rd and len(r) != 0: q_votes_raw.append(r) @@ -97,8 +100,6 @@ def get_votes(url, substrate_interface): r[rd]['prevotes'] = d['prevotes']['missing'] r[rd]['precommits'] = d['precommits']['missing'] - if url in q_outaged: - q_outaged.remove(url) except Exception: if url not in q_outaged: q_outaged.append(url) @@ -106,7 +107,6 @@ def get_votes(url, substrate_interface): substrate_interface.connect_websocket() pass - def construct_metrics(active_validators, grandpa_keys, votes_threshold, current_session): data = q_votes_raw.copy() result = {} @@ -176,6 +176,7 @@ def construct_metrics(active_validators, grandpa_keys, votes_threshold, current_ def main(): + substrate_interface = SUBSTRATE_INTERFACE(random.choice(rpc_endpoints), chain) while True: try: for i in q_outaged: @@ -212,8 +213,6 @@ def main(): rpc_count = len(rpc_endpoints) thread_count = 3 - substrate_interface = SUBSTRATE_INTERFACE(rpc_endpoints[0], chain) - q_metrics = deque([]) q_votes_raw = deque([], maxlen=rpc_count * thread_count * 15) q_rounds_processed = deque([], maxlen=100) @@ -221,10 +220,7 @@ def main(): for url in rpc_endpoints: for i in range(thread_count): - if url == rpc_endpoints[0]: - th = threading.Thread(target=get_votes, args=(url, substrate_interface)) - else: - th = threading.Thread(target=get_votes, args=(url, SUBSTRATE_INTERFACE(url, chain))) + th = threading.Thread(target=get_votes, args=(url, SUBSTRATE_INTERFACE(url, chain))) th.daemon = True th.start() time.sleep(0.2) diff --git a/exporters/common/functions.py b/exporters/common/functions.py index 1c7d3ac..f27668e 100644 --- a/exporters/common/functions.py +++ b/exporters/common/functions.py @@ -32,19 +32,20 @@ def __init__(self, ws_endpoint, chain): def request(self, module: str, function: str, params: str = None): try: - r = self.substrate.query( + return self.substrate.query( module=module, storage_function=function, params=params) - - return r except (WebSocketConnectionClosedException, ConnectionRefusedError, SubstrateRequestException) as e: self.substrate.connect_websocket() logging.critical('The substrate api call failed with error ' + str(e)) - r = None def rpc_request(self, method: str, params: str = None): - return self.substrate.rpc_request(method=method, params=params) + try: + return self.substrate.rpc_request(method=method, params=params) + except Exception as e: + self.substrate.connect_websocket() + logging.critical('The substrate api request failed with error ' + str(e)) def get_era_points(data): diff --git a/exporters/events/cmd/events-exporter/reader.go b/exporters/events/cmd/events-exporter/reader.go index 1dd4ffd..804f58b 100644 --- a/exporters/events/cmd/events-exporter/reader.go +++ b/exporters/events/cmd/events-exporter/reader.go @@ -13,6 +13,7 @@ import ( "sync/atomic" "time" + "github.com/avast/retry-go/v4" "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" ) @@ -25,6 +26,7 @@ type Config struct { NewHeadBy string `json:"NEW_HEAD_BY" envconfig:"NEW_HEAD_BY" default:"poll"` ExposeIdentities bool `json:"EXPOSE_ID" envconfig:"EXPOSE_ID"` KnownValidatorCfg []string `json:"VALIDATORS_CFG" envconfig:"VALIDATORS_CFG" default:""` + ExposeParaVotes bool `json:"EXPOSE_PARAVOTES" envconfig:"EXPOSE_PARAVOTES" default:"true"` } /* @@ -47,6 +49,7 @@ type HeadReader struct { wsclient *client.RPCClient decoder *decoder.Decoder log *logrus.Logger + rw *sync.Mutex paraSessionValidators sync.Map //map[uint32][]string sessionValidators sync.Map //map[uint32][]string cfg Config @@ -76,6 +79,7 @@ func NewHeadReader(l *logrus.Logger, cfg Config, ctx context.Context) (*HeadRead identities: make(map[string]string), log: l, cfg: cfg, + rw: &sync.Mutex{}, } for _, path := range cfg.KnownValidatorCfg { l.Infof("loading config from %s", path) @@ -149,12 +153,13 @@ func (reader *HeadReader) ProcessBlockParaVotes(ctx context.Context, hash string votedValidators = append(votedValidators, groupVotes.MustInt("col1")) // can go deeper and check backable (explicit)/seconded (implicit) statements if need be } - missingVotes := getMissingValidatorsFrom(validatorGroups, votedValidators) for _, vv := range votedValidators { if reader.registry == nil || reader.GetValidatorsHostname(psValidators[vv]) != "" { + reader.mon.ProcessEvent(MetricBackingVotesMissedCount, 0, reader.LabelValues(psValidators[vv])...) reader.mon.ProcessEvent(MetricBackingVotesExpectedCount, 1, reader.LabelValues(psValidators[vv])...) } } + missingVotes := getMissingValidatorsFrom(validatorGroups, votedValidators) if len(missingVotes) > 0 { for _, mv := range missingVotes { if reader.registry == nil || reader.GetValidatorsHostname(psValidators[mv]) != "" { @@ -284,22 +289,47 @@ func (reader *HeadReader) Read(ctx context.Context) error { // handle input hashes g.Go(func() error { for { - callCtx, cancel := context.WithTimeout(ctx, 120*time.Second) - defer cancel() select { case h := <-headHashes: - if err := reader.ProcessBlockEvents(callCtx, h); err != nil { - return err + blockctx, cancel := context.WithTimeout(ctx, 120*time.Second) + start := time.Now() + g, ctx := errgroup.WithContext(blockctx) + g.Go(func() error { + return retry.Do( + func() error { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + return reader.ProcessBlockEvents(ctx, h) + }, + retry.Context(blockctx), + retry.Delay(time.Second), + ) + }) + if reader.cfg.ExposeParaVotes { + g.Go(func() error { + return retry.Do( + func() error { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + return reader.ProcessBlockParaVotes(ctx, h) + }, + retry.Context(blockctx), + retry.Delay(time.Second), + ) + }) } - if err := reader.ProcessBlockParaVotes(callCtx, h); err != nil { + err := g.Wait() + cancel() + duration := time.Since(start).Seconds() + if duration > 5.0 { + reader.log.Infof("slow block %s processing took: %.2f sec", h, time.Since(start).Seconds()) + } + if err != nil { return err } case <-ctx.Done(): return ctx.Err() - case <-callCtx.Done(): - return callCtx.Err() } - cancel() } }) if err := g.Wait(); err != nil { @@ -381,6 +411,8 @@ func (reader *HeadReader) HandleAllEvents(ctx context.Context, hash string, even // TODO: cleanup func (reader *HeadReader) GetParaSessionValidators(ctx context.Context, hash string, session uint32) []string { + reader.rw.Lock() + defer reader.rw.Unlock() if _, ok := reader.paraSessionValidators.Load(session); !ok { req, _ := reader.decoder.NewStorageRequest("paraSessionInfo", "accountKeys", session) resp, err := reader.wsclient.StateGetStorage(ctx, req, hash) @@ -413,6 +445,8 @@ func (reader *HeadReader) GetParaSessionValidatorBy(ctx context.Context, hash st // TODO: cleanup func (reader *HeadReader) GetSessionValidators(ctx context.Context, hash string, session uint32) []string { + reader.rw.Lock() + defer reader.rw.Unlock() if session == 0 { reader.sessionValidators.Range(func(key, value any) bool { i := key.(uint32) diff --git a/exporters/events/go.mod b/exporters/events/go.mod index e6cb296..90507cf 100644 --- a/exporters/events/go.mod +++ b/exporters/events/go.mod @@ -16,6 +16,7 @@ require ( require github.com/kr/text v0.2.0 // indirect require ( + github.com/avast/retry-go/v4 v4.6.0 github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/decred/base58 v1.0.5 // indirect diff --git a/exporters/events/go.sum b/exporters/events/go.sum index 5264568..fff49c1 100644 --- a/exporters/events/go.sum +++ b/exporters/events/go.sum @@ -1,5 +1,7 @@ github.com/ChainSafe/go-schnorrkel v1.0.0 h1:3aDA67lAykLaG1y3AOjs88dMxC88PgUuHRrLeDnvGIM= github.com/ChainSafe/go-schnorrkel v1.0.0/go.mod h1:dpzHYVxLZcp8pjlV+O+UR8K0Hp/z7vcchBSbMBEhCw4= +github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA= +github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= @@ -60,8 +62,8 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/vedhavyas/go-subkey/v2 v2.0.0 h1:LemDIsrVtRSOkp0FA8HxP6ynfKjeOj3BY2U9UNfeDMA= github.com/vedhavyas/go-subkey/v2 v2.0.0/go.mod h1:95aZ+XDCWAUUynjlmi7BtPExjXgXxByE0WfBwbmIRH4= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= diff --git a/exporters/github_release/projects.yml b/exporters/github_release/projects.yml index f8dc736..c76815f 100644 --- a/exporters/github_release/projects.yml +++ b/exporters/github_release/projects.yml @@ -5,3 +5,4 @@ projects: Moonbeam: https://api.github.com/repos/PureStake/moonbeam/releases/latest Acala: https://api.github.com/repos/AcalaNetwork/Acala/releases/latest Astar: https://api.github.com/repos/PlasmNetwork/Astar/releases/latest + Avail: https://api.github.com/repos/availproject/avail/releases/latest diff --git a/grafana/provisioning/dashboards/avail.json b/grafana/provisioning/dashboards/avail.json new file mode 100644 index 0000000..90df8be --- /dev/null +++ b/grafana/provisioning/dashboards/avail.json @@ -0,0 +1,1497 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 6, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 119, + "panels": [], + "title": "Basic", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Current era. ", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "hideTimeOverride": true, + "id": 2, + "interval": "", + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max_over_time(avail_staking_currentEra{chain=~\"$chain\"}[15s])", + "hide": false, + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "timeFrom": "1m", + "title": "Era", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Session idx. ", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "hideTimeOverride": true, + "id": 108, + "interval": "", + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max_over_time(avail_session_currentSession{chain=\"$chain\"}[15s])", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "timeFrom": "1m", + "title": "Session(epoch)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total points.", + "fieldConfig": { + "defaults": { + "decimals": 0, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "hideTimeOverride": true, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max_over_time(avail_staking_totalPoints{chain=\"$chain\"}[15s])", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": "1m", + "title": "Total points", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Era progress.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-yellow", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 5 + }, + "hideTimeOverride": true, + "id": 106, + "interval": "", + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max_over_time(avail_staking_eraProgress{chain=\"$chain\"}[15s])", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "timeFrom": "1m", + "title": "Era progress percents", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Session progress.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-yellow", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 5 + }, + "hideTimeOverride": true, + "id": 107, + "interval": "", + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max_over_time(avail_session_sessionProgress{chain=\"$chain\"}[15s])", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "timeFrom": "1m", + "title": "Session progress percents", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total validators. ", + "fieldConfig": { + "defaults": { + "decimals": 0, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 5 + }, + "hideTimeOverride": true, + "id": 76, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(sum_over_time(avail_session_validators{chain=\"$chain\"}[15s]))", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": "1m", + "title": "Total validators", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total points yearned by whole network. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avail_staking_totalPoints{chain=\"$chain\"}", + "interval": "", + "legendFormat": "{{ chain }}", + "range": true, + "refId": "A" + } + ], + "title": "Total era points", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 123, + "panels": [], + "title": "Finality", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Finality ratio. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "dark-red", + "value": 0 + }, + { + "color": "dark-yellow", + "value": 80 + }, + { + "color": "dark-green", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Prevotes" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "dark-red", + "value": 0 + }, + { + "color": "dark-yellow", + "value": 95 + }, + { + "color": "dark-green", + "value": 98 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 114, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "precommits" + } + ] + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max by (account) (max_over_time(avail_finality_precommits{chain=\"$chain\",account=~\"$account\"}[1m])) / on () group_left() max(max_over_time(avail_finality_roundsProcessed{chain=\"$chain\"}[1m])) * 100 and on(account) avail_session_validators == 1", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max by (account) (max_over_time(avail_finality_prevotes{chain=\"$chain\",account=~\"$account\"}[1m])) / on () group_left() max(max_over_time(avail_finality_roundsProcessed{chain=\"$chain\"}[1m])) * 100 and on(account) avail_session_validators == 1", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "GRANPDA ratio", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "account", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time 1": true, + "Time 2": true + }, + "indexByName": {}, + "renameByName": { + "Time 2": "", + "Value #A": "prevotes", + "Value #B": "precommits" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Prevotes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 103, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avail_finality_prevotes{chain=\"$chain\",account=~\"$account\"} > 0", + "hide": false, + "interval": "", + "legendFormat": "{{ account }}", + "range": true, + "refId": "B" + } + ], + "title": "Block finality (prevotes)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Precommits.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 117, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avail_finality_precommits{chain=\"$chain\",account=~\"$account\"} > 0", + "hide": false, + "interval": "", + "legendFormat": "{{ account }}", + "range": true, + "refId": "B" + } + ], + "title": "Block finality (precommits)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 125, + "panels": [], + "title": "Era validator", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Earned points by selected validators.", + "fieldConfig": { + "defaults": { + "decimals": 0, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 53 + }, + "hideTimeOverride": true, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(sum_over_time(avail_staking_eraPoints{chain=\"$chain\",account=~\"$account\"}[30s]))", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": "1m", + "title": "Earned points", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "List of active validators.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Node" + }, + "properties": [ + { + "id": "custom.width", + "value": 268 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 4, + "y": 53 + }, + "hideTimeOverride": true, + "id": 63, + "interval": "", + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "node" + } + ] + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max_over_time(avail_session_validators{chain=\"$chain\",account=~\"$account\"}[15s]) == 1", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{ validator }} - {{ account_addr }}", + "refId": "A" + } + ], + "timeFrom": "1m", + "title": "Active validators", + "transformations": [ + { + "id": "reduce", + "options": { + "includeTimeField": false, + "mode": "reduceFields", + "reducers": [ + "last" + ] + } + }, + { + "id": "labelsToFields", + "options": {} + }, + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "account_addr": false, + "chain": true, + "endpoint": true, + "instance": true, + "job": true, + "namespace": true, + "pod": true, + "prometheus": true, + "service": true, + "validator": false + }, + "indexByName": { + "Value": 1, + "__name__": 2, + "account": 6, + "chain": 3, + "endpoint": 7, + "instance": 4, + "job": 5, + "namespace": 8, + "node": 0, + "pod": 9, + "prometheus": 10, + "service": 11 + }, + "renameByName": { + "Field": " Node", + "Time": "", + "__name__": "", + "account_addr": "Account address", + "validator": "Node" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Select validators position chart.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 9, + "y": 53 + }, + "hideTimeOverride": true, + "id": 67, + "interval": "", + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Position" + } + ] + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max_over_time(avail_staking_validatorPositionChart{chain=\"$chain\",account=~\"$account\"}[15s]) + 1", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{ account }} ", + "refId": "A" + } + ], + "timeFrom": "1m", + "title": "Validators position chart", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "max" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "account", + "Max": "position" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Active selected validators.", + "fieldConfig": { + "defaults": { + "decimals": 0, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 57 + }, + "hideTimeOverride": true, + "id": 48, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(sum_over_time(avail_session_validators{chain=\"$chain\",account=~\"$account\"}[15s]) > 0)", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": "1m", + "title": "Active validators", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Selected validators era points.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 61 + }, + "hiddenSeries": false, + "id": 77, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avail_staking_eraPoints{chain=\"$chain\",account=~\"$account\"} > 0", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{ account }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Era points", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1095", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1096", + "format": "none", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "turing", + "value": "turing" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "chain", + "options": [ + { + "selected": true, + "text": "turing", + "value": "turing" + }, + { + "selected": false, + "text": "avail", + "value": "avail" + } + ], + "query": "turing,avail", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": "(.*)", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(avail_finality_prevotes{chain=\"$chain\"},account)", + "hide": 0, + "includeAll": true, + "label": "account", + "multi": true, + "name": "account", + "options": [], + "query": { + "query": "label_values(avail_finality_prevotes{chain=\"$chain\"},account)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "P2P.ORG Avail dashboard", + "uid": "qiKLINXIz", + "version": 7, + "weekStart": "" + } \ No newline at end of file diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index a71c4df..c475089 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -84,6 +84,30 @@ scrape_configs: - targets: - 'shiden_exporter:9150' +- job_name: turing_common_exporter + metrics_path: /metrics + static_configs: + - targets: + - 'turing_common_exporter:9150' + +- job_name: avail_common_exporter + metrics_path: /metrics + static_configs: + - targets: + - 'avail_common_exporter:9150' + +- job_name: turing_finality_exporter + metrics_path: /metrics + static_configs: + - targets: + - 'turing_finality_exporter:9150' + +- job_name: avail_finality_exporter + metrics_path: /metrics + static_configs: + - targets: + - 'avail_finality_exporter:9150' + alerting: alertmanagers: - scheme: http diff --git a/turing.env b/turing.env new file mode 100644 index 0000000..65f5061 --- /dev/null +++ b/turing.env @@ -0,0 +1,2 @@ +WS_ENDPOINT="wss://turing-rpc.avail.so/ws" +WS_ENDPOINTS="wss://turing-rpc.avail.so/ws" diff --git a/turing.yml b/turing.yml new file mode 100644 index 0000000..34f157c --- /dev/null +++ b/turing.yml @@ -0,0 +1,32 @@ +version: '3.4' + +services: + turing_common_exporter: + build: + context: ./exporters/avail + args: + exporter: "common_exporter" + environment: + - "LISTEN=0.0.0.0" + - "PORT=9150" + - "CHAIN=turing" + env_file: + - ./turing.env + networks: + - exporters + restart: on-failure + + turing_finality_exporter: + build: + context: ./exporters/avail + args: + exporter: "finality_exporter" + environment: + - "LISTEN=0.0.0.0" + - "PORT=9150" + - "CHAIN=turing" + env_file: + - ./turing.env + networks: + - exporters + restart: on-failure