update migalabs endpoint (#43)

_config.yml

@@ -1,6 +1,6 @@
 # Read about configuration here: https://jekyllrb.com/docs/configuration/options/
 title: Client Diversity | Ethereum
-description: Improve Etheruem's resilience and protect the network by using a minority client.
+description: Improve Ethereum's resilience and protect the network by using a minority client.
 keywords: ethereum, client, diversity, eth, beacon, chain
 url: https://clientdiversity.org
 permalink: pretty # Do not change this, it will break all links
@@ -37,7 +37,7 @@ notification_msg: Join the community! <br><a href=""></a>.
 notification_expiration: 3000000000 # epoch time in seconds
 
 # Toast
-enable_toast: true
+enable_toast: false
 toast_msg_id: 4 # must increment when creating a new message
 toast_title: Ether Alpha's Gitcoin grant is live!
 toast_msg: This round runs through August 29th. If you enjoy this or <a href='https://etheralpha.org/' target='_blank'>other tools</a> created, please support to help maintain existing projects and fund new ones. # Best when under 200 characters

_data/clients-consensus.yml

@@ -27,7 +27,7 @@
   status: stable
   support: Linux, Win, macOS, ARM
   lang: Rust
-  donate: https://gitcoin.co/grants/25/lighthouse-ethereum-20-client
+  donate: https://protocol-guild.readthedocs.io/en/latest/index.html
   opensource: true
 - name: Lodestar
   link: https://lodestar.chainsafe.io/
@@ -38,7 +38,7 @@
   status: stable
   support: Linux, Win, macOS # windows theoretically works but not tested yet https://github.com/ChainSafe/lodestar/issues/3519
   lang: TypeScript
-  donate: https://gitcoin.co/grants/6034/lodestar-typescript-ethereum-consensus-client
+  donate: https://protocol-guild.readthedocs.io/en/latest/index.html
   opensource: true
 - name: Nimbus
   link: https://nimbus.team/
@@ -48,7 +48,7 @@
   status: stable
   support: Linux, Win, macOS, ARM
   lang: Nim
-  donate: https://gitcoin.co/grants/137/nimbus-2
+  donate: 
   opensource: true
 - name: Prysm
   link: https://prysmaticlabs.com/
@@ -58,7 +58,8 @@
   status: stable
   support: Linux, Win, macOS, ARM
   lang: Golang
-  donate: funded # Supported by Offchain Labs
+  # donate: funded # Supported by Offchain Labs
+  donate: https://protocol-guild.readthedocs.io/en/latest/index.html
   opensource: true
 - name: Teku
   link: https://consensys.net/knowledge-base/ethereum-2/teku/
@@ -68,6 +69,7 @@
   status: stable
   support: Linux, Win, macOS
   lang: Java
-  donate: funded # funded by ConsenSys
+  # donate: funded # funded by ConsenSys
+  donate: https://protocol-guild.readthedocs.io/en/latest/index.html
   opensource: true
 

_data/clients-execution.yml

@@ -57,7 +57,7 @@
   status: stable
   support: Linux, Win, macOS, ARM
   lang: Golang
-  donate: https://gitcoin.co/grants/6128/go-ethereum-geth 
+  donate: https://protocol-guild.readthedocs.io/en/latest/index.html
   opensource: true
 - name: Nethermind
   link: http://nethermind.io/
@@ -67,7 +67,7 @@
   status: stable
   support: Linux, Win, macOS, ARM
   lang: .NET
-  donate: https://gitcoin.co/grants/142/nethermind
+  donate: https://protocol-guild.readthedocs.io/en/latest/index.html
   opensource: true
 - name: Nimbus
   link: https://nimbus.team/
@@ -77,7 +77,7 @@
   status: pre-alpha
   support: 
   lang: Nim
-  donate: https://gitcoin.co/grants/137/nimbus-2
+  donate: 
   opensource: true
 # - name: OpenEthereum
 #   link: https://openethereum.org/

_includes/partials/content/distribution.html

@@ -27,7 +27,7 @@ <h3 class="p-2 mb-4">Consensus Clients</h3>
               {%- assign blockprint = site.data.blockprint | last -%}
               {%- include partials/components/progress-bar.html data=blockprint.data.distribution -%}
               <div class="text-center small">
-                Data provided by <a href="https://github.com/sigp/blockprint/blob/main/docs/api.md" target="_blank">Sigma Prime's Blockprint</a> — updated daily. <br>
+                Data provided by <a href="https://blockprint.sigp.io/" target="_blank">Sigma Prime's Blockprint</a> — updated daily. <br>
                 Data may not be 100% accurate. (<a href="/methodology">Read more</a>)
               </div>
             </div>

_scripts/collect_data.py

@@ -18,11 +18,12 @@
 print_fetch_data = False
 print_processed_data = True
 pretty_print = True
-exit_on_fetch_error = True
+exit_on_fetch_error = False
 exit_on_save_error = True
 exit_on_report_error = False
 
 rated_token = os.environ.get("RATED_API_KEY")
+migalabs_token = os.environ.get("MIGALABS_API_KEY")
 google_form_error_report_url = os.environ.get("ERROR_REPORT_ENDPOINT")
 
 # enter values for local testing
@@ -800,26 +801,53 @@ def ethernodes_marketshare():
 
 def get_migalabs_marketshare_data():
   if use_test_data:
-    response = {'status': 200, 'attempts': 1, 'data': {'erigon': 53, 'grandine': 232, 'lighthouse': 3914, 'lodestar': 188, 'nimbus': 719, 'prysm': 4038, 'teku': 1476, 'unknown': 15}}
+    response = {'status': 200, 'attempts': 1, 'data': [{"timestamp":"2023-10-03T04:37:09Z","data":[{"client_name":"lighthouse","node_count":2867},{"client_name":"prysm","node_count":2206},{"client_name":"teku","node_count":1303},{"client_name":"nimbus","node_count":836},{"client_name":"lodestar","node_count":252},{"client_name":"grandine","node_count":213},{"client_name":"unknown","node_count":28}]}]}
     print_data("fetch", response)
     return response
   else:
-    url = "https://migalabs.es/api/v1/client-distribution"
-    response = fetch_json(url)
+    url = "https://monitoreth.io/data-api/api/eth/v1/nodes/consensus/validators/client_diversity"
+    payload = {}
+    headers = {
+      'X-Api-Key': migalabs_token
+    }
+    response = fetch_json(url, "GET", payload, headers)
     return response
 
 def process_migalabs_marketshare_data(raw_data):
   # example migalabs raw data:
-  # raw_data = {'status': 200, 'attempts': 1, 'data': {
-  #   'erigon': 53, 
-  #   'grandine': 232, 
-  #   'lighthouse': 3914, 
-  #   'lodestar': 188, 
-  #   'nimbus': 719, 
-  #   'prysm': 4038, 
-  #   'teku': 1476, 
-  #   'unknown': 15
-  #   }}
+    # raw_data = {'status': 200, 'attempts': 1, 'data': [{
+    #   "timestamp": "2023-10-03T04:37:09Z",
+    #   "data": [
+    #     {
+    #       "client_name": "lighthouse",
+    #       "node_count": 2867
+    #     },
+    #     {
+    #       "client_name": "prysm",
+    #       "node_count": 2206
+    #     },
+    #     {
+    #       "client_name": "teku",
+    #       "node_count": 1303
+    #     },
+    #     {
+    #       "client_name": "nimbus",
+    #       "node_count": 836
+    #     },
+    #     {
+    #       "client_name": "lodestar",
+    #       "node_count": 252
+    #     },
+    #     {
+    #       "client_name": "grandine",
+    #       "node_count": 213
+    #     },
+    #     {
+    #       "client_name": "unknown",
+    #       "node_count": 28
+    #     }
+    #   ]
+    # }]}
 
   main_clients = ["lighthouse", "nimbus", "teku", "prysm", "lodestar", "erigon", "grandine"]
   threshold_percentage = 0.5 # represented as a percent, not a decimal
@@ -831,9 +859,9 @@ def process_migalabs_marketshare_data(raw_data):
   final_data = {}
 
   # reformat data into a list of dicts
-  for key, value in raw_data["data"].items():
-    reformatted_data.append({"name": key.lower(), "value": value})
-    sample_size += value
+  for item in raw_data["data"][0]["data"]:
+    reformatted_data.append({"name": item["client_name"].lower(), "value": item["node_count"]})
+    sample_size += item["node_count"]
   # pprint(["reformatted_data", reformatted_data])
   # pprint(["sample_size", sample_size])
 

methodology.md

@@ -15,7 +15,7 @@ There's no inherent way to know exactly what client a validator is running. Rese
 
 ## Consensus Client Data
 
-[Blockprint](https://github.com/sigp/blockprint) - Developed by Sigma Prime's Michael Sproul, Blockprint  analyzes each client's block proposal style as described in [this Twitter thread](https://twitter.com/sproulM_/status/1440512518242197516) ([Nitter](https://nitter.snopyta.org/sproulM_/status/1440512518242197516)).
+[Blockprint](https://blockprint.sigp.io/) - Developed by Sigma Prime's Michael Sproul, Blockprint  analyzes each client's block proposal style as described in [this Twitter thread](https://twitter.com/sproulM_/status/1440512518242197516) ([Nitter](https://nitter.snopyta.org/sproulM_/status/1440512518242197516)).
 
 [Miga Labs](https://migalabs.io/) - A crawler is used to count beacon nodes and their self-reported identity. However, this means that validators sharing a node are counted only once and nodes with fewer validators have a greater influence on the estimate.