From 04b558ff435b8afb5f3ef642e77ff30b996e73d1 Mon Sep 17 00:00:00 2001 From: Juan Munoz Date: Mon, 23 Sep 2024 16:28:01 -0300 Subject: [PATCH 1/4] add more metrics --- core/node/da_dispatcher/src/da_dispatcher.rs | 20 ++++++++++++++++++++ core/node/da_dispatcher/src/metrics.rs | 8 +++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/core/node/da_dispatcher/src/da_dispatcher.rs b/core/node/da_dispatcher/src/da_dispatcher.rs index f8e6f6b3172..a476add4a70 100644 --- a/core/node/da_dispatcher/src/da_dispatcher.rs +++ b/core/node/da_dispatcher/src/da_dispatcher.rs @@ -53,6 +53,12 @@ impl DataAvailabilityDispatcher { }, ); + let subtasks = futures::future::join(subtasks, async { + if let Err(err) = self.update_metrics().await { + tracing::error!("update_metrics error {err:?}"); + } + }); + tokio::select! { _ = subtasks => {}, _ = stop_receiver.changed() => { @@ -113,6 +119,7 @@ impl DataAvailabilityDispatcher { .last_dispatched_l1_batch .set(batch.l1_batch_number.0 as usize); METRICS.blob_size.observe(batch.pubdata.len()); + METRICS.blobs_dispatched.inc_by(1); tracing::info!( "Dispatched a DA for batch_number: {}, pubdata_size: {}, dispatch_latency: {dispatch_latency_duration:?}", batch.l1_batch_number, @@ -172,6 +179,7 @@ impl DataAvailabilityDispatcher { METRICS .last_included_l1_batch .set(blob_info.l1_batch_number.0 as usize); + METRICS.blobs_included.inc_by(1); tracing::info!( "Received an inclusion data for a batch_number: {}, inclusion_latency_seconds: {}", @@ -181,6 +189,18 @@ impl DataAvailabilityDispatcher { Ok(()) } + + async fn update_metrics(&self) -> anyhow::Result<()> { + let mut conn = self.pool.connection_tagged("da_dispatcher").await?; + let batches = conn + .data_availability_dal() + .get_ready_for_da_dispatch_l1_batches(self.config.max_rows_to_dispatch() as usize) + .await?; + drop(conn); + METRICS.blobs_pending_dispatch.set(batches.len()); + + Ok(()) + } } async fn retry( diff --git a/core/node/da_dispatcher/src/metrics.rs b/core/node/da_dispatcher/src/metrics.rs index 67ac5ed6822..6fe0b848267 100644 --- a/core/node/da_dispatcher/src/metrics.rs +++ b/core/node/da_dispatcher/src/metrics.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use vise::{Buckets, Gauge, Histogram, Metrics, Unit}; +use vise::{Buckets, Counter, Gauge, Histogram, Metrics, Unit}; /// Buckets for `blob_dispatch_latency` (from 0.1 to 120 seconds). const DISPATCH_LATENCIES: Buckets = @@ -19,6 +19,12 @@ pub(super) struct DataAvailabilityDispatcherMetrics { /// Buckets are bytes ranging from 1 KB to 16 MB, which has to satisfy all blob size values. #[metrics(buckets = Buckets::exponential(1_024.0..=16.0 * 1_024.0 * 1_024.0, 2.0), unit = Unit::Bytes)] pub blob_size: Histogram, + /// Amount of pending blobs to be dispatched. + pub blobs_pending_dispatch: Gauge, + /// Total number of blobs dispatched. + pub blobs_dispatched: Gauge, + /// Total number of blobs included. + pub blobs_included: Gauge, /// Number of transactions resent by the DA dispatcher. #[metrics(buckets = Buckets::linear(0.0..=10.0, 1.0))] From eb19b65b04ec39a6ece1b6ff1b54da1ec4bed25f Mon Sep 17 00:00:00 2001 From: Juan Munoz Date: Mon, 23 Sep 2024 17:07:40 -0300 Subject: [PATCH 2/4] eigenda integration docs --- eigenda-integration.md | 74 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 eigenda-integration.md diff --git a/eigenda-integration.md b/eigenda-integration.md new file mode 100644 index 00000000000..1a61336f163 --- /dev/null +++ b/eigenda-integration.md @@ -0,0 +1,74 @@ +# Zksync-era <> EigenDA Integration + +## Local Setup + +1. Install `zk_inception` & `zk_supervisor` + +```bash +./bin/zkt +``` + +2. Start containers + +```bash +zk_inception containers --observability false +``` + +Leave the obserbability flag as `false` you can enable it in the `init` step. + +3. Create `eigen_da` chain + +```bash +zk_inception chain create \ + --chain-name eigen_da \ + --chain-id sequential \ + --prover-mode no-proofs \ + --wallet-creation localhost \ + --l1-batch-commit-data-generator-mode validium \ + --base-token-address 0x0000000000000000000000000000000000000001 \ + --base-token-price-nominator 1 \ + --base-token-price-denominator 1 \ + --set-as-default false +``` + +4. Initialize created ecosystem + +```bash +zk_inception ecosystem init \ + --deploy-paymaster true \ + --deploy-erc20 true \ + --deploy-ecosystem true \ + --l1-rpc-url http://127.0.0.1:8545 \ + --server-db-url=postgres://postgres:notsecurepassword@localhost:5432 \ + --server-db-name=zksync_server_localhost_eigen_da \ + --prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \ + --prover-db-name=zksync_prover_localhost_eigen_da \ + --chain eigen_da \ + --verbose +``` + +You may enable observability here if you want to. + +5. Start the server + +```bash +zk_inception server --chain eigen_da +``` + +### Testing + +Modify the following flag in `core/lib/config/src/configs/da_dispatcher.rs` + +```rs +pub const DEFAULT_USE_DUMMY_INCLUSION_DATA: bool = true; +``` + +And with the server running on one terminal, you can run the server integration tests on a separate terminal with the following command: + +```bash +zk_supervisor test integration --chain eigen_da +``` + +### Metrics + +Access Grafana at [http://localhost:3000/](http://localhost:3000/), go to dashboards and select `EigenDA`. From c832608cbfd36ae932435cd44ecebcd3848c26e3 Mon Sep 17 00:00:00 2001 From: Juan Munoz Date: Mon, 23 Sep 2024 18:09:55 -0300 Subject: [PATCH 3/4] changes to integration doc --- eigenda-integration.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/eigenda-integration.md b/eigenda-integration.md index 1a61336f163..4d02fa6a824 100644 --- a/eigenda-integration.md +++ b/eigenda-integration.md @@ -11,11 +11,9 @@ 2. Start containers ```bash -zk_inception containers --observability false +zk_inception containers --observability true ``` -Leave the obserbability flag as `false` you can enable it in the `init` step. - 3. Create `eigen_da` chain ```bash @@ -57,13 +55,14 @@ zk_inception server --chain eigen_da ### Testing -Modify the following flag in `core/lib/config/src/configs/da_dispatcher.rs` +Modify the following flag in `core/lib/config/src/configs/da_dispatcher.rs` (then restart the server) ```rs pub const DEFAULT_USE_DUMMY_INCLUSION_DATA: bool = true; ``` -And with the server running on one terminal, you can run the server integration tests on a separate terminal with the following command: +And with the server running on one terminal, you can run the server integration tests on a separate terminal with the +following command: ```bash zk_supervisor test integration --chain eigen_da From 62865a07a6cbee755ffd0aa6f94f206063aaa956 Mon Sep 17 00:00:00 2001 From: Juan Munoz Date: Mon, 23 Sep 2024 18:18:54 -0300 Subject: [PATCH 4/4] remove blob retriever --- .dockerignore | 1 - Cargo.lock | 123 +-------- Cargo.toml | 1 - EigenDA.json | 334 ------------------------- core/node/da_dispatcher/src/metrics.rs | 2 +- docker-compose.yml | 11 - docker/retrieve-blobs/Dockerfile | 35 --- retrieve-blobs/.gitignore | 1 - retrieve-blobs/Cargo.toml | 13 - retrieve-blobs/src/main.rs | 114 --------- 10 files changed, 3 insertions(+), 632 deletions(-) delete mode 100644 EigenDA.json delete mode 100644 docker/retrieve-blobs/Dockerfile delete mode 100644 retrieve-blobs/.gitignore delete mode 100644 retrieve-blobs/Cargo.toml delete mode 100644 retrieve-blobs/src/main.rs diff --git a/.dockerignore b/.dockerignore index aa2384a4af8..39efdabca19 100644 --- a/.dockerignore +++ b/.dockerignore @@ -17,7 +17,6 @@ keys/setup !db/ !backups/ !core/ -!retrieve-blobs/ !prover/ !yarn.lock !package.json diff --git a/Cargo.lock b/Cargo.lock index 4a97f7f3db7..7a032bea5e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2293,12 +2293,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "fallible-iterator" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" - [[package]] name = "fastrand" version = "2.0.1" @@ -4890,24 +4884,6 @@ dependencies = [ "indexmap 2.1.0", ] -[[package]] -name = "phf" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" -dependencies = [ - "phf_shared", -] - -[[package]] -name = "phf_shared" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" -dependencies = [ - "siphasher 0.3.11", -] - [[package]] name = "pin-project" version = "1.1.3" @@ -5054,35 +5030,6 @@ dependencies = [ "universal-hash", ] -[[package]] -name = "postgres-protocol" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acda0ebdebc28befa84bee35e651e4c5f09073d668c7aed4cf7e23c3cda84b23" -dependencies = [ - "base64 0.22.1", - "byteorder", - "bytes", - "fallible-iterator", - "hmac 0.12.1", - "md-5", - "memchr", - "rand 0.8.5", - "sha2 0.10.8", - "stringprep", -] - -[[package]] -name = "postgres-types" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f66ea23a2d0e5734297357705193335e0a957696f34bed2f2faefacb2fec336f" -dependencies = [ - "bytes", - "fallible-iterator", - "postgres-protocol", -] - [[package]] name = "powerfmt" version = "0.2.0" @@ -5214,21 +5161,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "prometheus" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" -dependencies = [ - "cfg-if", - "fnv", - "lazy_static", - "memchr", - "parking_lot", - "protobuf", - "thiserror", -] - [[package]] name = "prometheus-client" version = "0.22.2" @@ -5345,12 +5277,6 @@ dependencies = [ "prost 0.12.1", ] -[[package]] -name = "protobuf" -version = "2.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" - [[package]] name = "protox" version = "0.5.1" @@ -5756,18 +5682,6 @@ dependencies = [ "typemap_rev", ] -[[package]] -name = "retrieve-blobs" -version = "0.1.0" -dependencies = [ - "hyper 0.14.29", - "lazy_static", - "prometheus", - "reqwest 0.12.5", - "tokio", - "tokio-postgres", -] - [[package]] name = "rfc6979" version = "0.3.1" @@ -6775,12 +6689,6 @@ dependencies = [ "time", ] -[[package]] -name = "siphasher" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" - [[package]] name = "siphasher" version = "1.0.1" @@ -6892,7 +6800,7 @@ dependencies = [ "serde_json", "sha2 0.10.8", "sha3 0.10.8", - "siphasher 1.0.1", + "siphasher", "slab", "smallvec", "soketto 0.7.1", @@ -6931,7 +6839,7 @@ dependencies = [ "rand_chacha", "serde", "serde_json", - "siphasher 1.0.1", + "siphasher", "slab", "smol", "smoldot", @@ -7866,32 +7774,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-postgres" -version = "0.7.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b5d3742945bc7d7f210693b0c58ae542c6fd47b17adbbda0885f3dcb34a6bdb" -dependencies = [ - "async-trait", - "byteorder", - "bytes", - "fallible-iterator", - "futures-channel", - "futures-util", - "log", - "parking_lot", - "percent-encoding", - "phf", - "pin-project-lite", - "postgres-protocol", - "postgres-types", - "rand 0.8.5", - "socket2", - "tokio", - "tokio-util", - "whoami", -] - [[package]] name = "tokio-rustls" version = "0.24.1" @@ -8686,7 +8568,6 @@ checksum = "0fec781d48b41f8163426ed18e8fc2864c12937df9ce54c88ede7bd47270893e" dependencies = [ "redox_syscall", "wasite", - "web-sys", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index e03a0f5eeac..8498751a599 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,7 +80,6 @@ members = [ "core/tests/test_account", "core/tests/loadnext", "core/tests/vm-benchmark", - "retrieve-blobs", ] resolver = "2" diff --git a/EigenDA.json b/EigenDA.json deleted file mode 100644 index 9f634f4fb38..00000000000 --- a/EigenDA.json +++ /dev/null @@ -1,334 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - }, - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": false, - "expr": "any", - "hide": false, - "iconColor": "red", - "name": "flux", - "tagKeys": "flux", - "target": { - "refId": "Anno", - "tags": [ - "flux" - ], - "type": "tags" - } - } - ] - }, - "description": "", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 4, - "links": [ - { - "asDropdown": false, - "icon": "external link", - "includeVars": false, - "keepTime": false, - "tags": [ - "Era-Main" - ], - "targetBlank": false, - "title": "", - "tooltip": "", - "type": "dashboards", - "url": "" - }, - { - "asDropdown": true, - "icon": "external link", - "includeVars": false, - "keepTime": false, - "tags": [ - "Era" - ], - "targetBlank": false, - "title": "Other Era Dashboards", - "tooltip": "", - "type": "dashboards", - "url": "" - } - ], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 11035, - "panels": [], - "title": "EigenDA", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 11034, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "blob_retrievals_total", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Blob Total", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Blob Amount", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 11036, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "blob_avg_size", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Blob Avg Size (Bytes)", - "type": "stat" - } - ], - "refresh": "1m", - "revision": 1, - "schemaVersion": 39, - "tags": [ - "Era-Main" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "Prometheus", - "value": "PBFA97CFB590B2093" - }, - "hide": 0, - "includeAll": false, - "multi": false, - "name": "datasource", - "options": [], - "query": "prometheus", - "queryValue": "", - "refresh": 1, - "regex": "^(?!prometheus-1$|prometheus$|vm-mainnet$|vm-gpu-sandbox$|vm-testnets$|vm-infra$|vm-infra-staging$|default$).*", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "isNone": true, - "selected": false, - "text": "None", - "value": "" - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(server_processed_txs{stage=~\"state_keeper\"}, namespace)", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "namespace", - "options": [], - "query": { - "query": "label_values(server_processed_txs{stage=~\"state_keeper\"}, namespace)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "^(?!snapshot_recovery_test$).*", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timeRangeUpdatedDuringEditOrView": false, - "timepicker": {}, - "timezone": "", - "title": "EigenDA", - "uid": "0Y8DUff4j", - "version": 1, - "weekStart": "" -} - diff --git a/core/node/da_dispatcher/src/metrics.rs b/core/node/da_dispatcher/src/metrics.rs index 6fe0b848267..4c21e556abe 100644 --- a/core/node/da_dispatcher/src/metrics.rs +++ b/core/node/da_dispatcher/src/metrics.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use vise::{Buckets, Counter, Gauge, Histogram, Metrics, Unit}; +use vise::{Buckets, Gauge, Histogram, Metrics, Unit}; /// Buckets for `blob_dispatch_latency` (from 0.1 to 120 seconds). const DISPATCH_LATENCIES: Buckets = diff --git a/docker-compose.yml b/docker-compose.yml index 505b250eac3..f2471c6bd17 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,14 +61,3 @@ services: ports: - "4242:4242" command: ./eigenda-proxy --addr 0.0.0.0 --port 4242 --memstore.enabled --eigenda-max-blob-length "2MiB" - - eigenda-metrics: - depends_on: - - eigenda-proxy - - postgres - build: # This is the project root - context: . - dockerfile: ./docker/retrieve-blobs/Dockerfile - ports: - - "7070:7070" - command: ./retrieve-blobs diff --git a/docker/retrieve-blobs/Dockerfile b/docker/retrieve-blobs/Dockerfile deleted file mode 100644 index 6b307c2ea99..00000000000 --- a/docker/retrieve-blobs/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -# Use the official Rust image to build the application -FROM rust:latest AS builder - -# Set the working directory inside the container -WORKDIR /usr/src/zksync - -# Copy the Cargo.toml and Cargo.lock files to the container -COPY ./retrieve-blobs . - -# Fetch and build the project dependencies (this will cache the dependencies) -RUN cargo fetch - -# Build the application in release mode -RUN cargo build --release - -# Use a smaller image for runtime -FROM ubuntu:latest - -# Install necessary dependencies for running the application (like PostgreSQL client) -RUN apt-get update && apt-get install -y \ - libpq-dev \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -# Set the working directory inside the runtime container -WORKDIR /usr/src/zksync - -# Copy the built application from the builder image -COPY --from=builder /usr/src/zksync/target/release/retrieve-blobs . - -# Expose the port for the Prometheus metrics server -EXPOSE 7070 - -# Set the environment variable to run the application with correct permissions -ENV RUST_LOG=info diff --git a/retrieve-blobs/.gitignore b/retrieve-blobs/.gitignore deleted file mode 100644 index ea8c4bf7f35..00000000000 --- a/retrieve-blobs/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/target diff --git a/retrieve-blobs/Cargo.toml b/retrieve-blobs/Cargo.toml deleted file mode 100644 index 172e299ff97..00000000000 --- a/retrieve-blobs/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "retrieve-blobs" -version = "0.1.0" -edition = "2021" -license = "MIT OR Apache-2.0" - -[dependencies] -reqwest = "0.12.1" -tokio = { version = "1", features = ["full"] } -tokio-postgres = "0.7" -prometheus = "0.13" -hyper = { version = "0.14", features = ["full", "server"] } -lazy_static = "1.5" diff --git a/retrieve-blobs/src/main.rs b/retrieve-blobs/src/main.rs deleted file mode 100644 index a9ddeb76276..00000000000 --- a/retrieve-blobs/src/main.rs +++ /dev/null @@ -1,114 +0,0 @@ -use std::{collections::HashMap, convert::Infallible, error::Error, time::Duration}; - -use hyper::{ - service::{make_service_fn, service_fn}, - Body, Request, Response, Server, -}; -use prometheus::{register_counter, register_gauge, Counter, Encoder, Gauge, TextEncoder}; -use tokio::time::sleep; -use tokio_postgres::NoTls; - -// Define Prometheus metrics -lazy_static::lazy_static! { - static ref BLOB_RETRIEVALS: Counter = register_counter!( - "blob_retrievals_total", - "Total number of blobs successfully retrieved" - ).unwrap(); - - static ref BLOB_AVG_SIZE: Gauge = register_gauge!( - "blob_avg_size", - "Average size of blobs in bytes" - ).unwrap(); -} - -#[tokio::main] -async fn main() { - // Start the metrics HTTP server - tokio::spawn(start_metrics_server()); - - let mut blobs = HashMap::new(); - - loop { - // Perform blob retrievals - match get_blobs(&mut blobs).await { - Ok(_) => println!("Blob retrieval successful"), - Err(e) => eprintln!("Blob retrieval error: {}", e), - }; - sleep(Duration::from_secs(1)).await; - } -} - -async fn get_blobs(blobs: &mut HashMap) -> Result<(), Box> { - // Connect to the PostgreSQL server - let (client, connection) = tokio_postgres::connect( - "host=postgres user=postgres password=notsecurepassword dbname=zksync_server_localhost_eigen_da", - NoTls, - ) - .await?; - - // Spawn a background task to handle the connection - tokio::spawn(async move { - if let Err(e) = connection.await { - eprintln!("Connection error: {}", e); - } - }); - - // Run the SELECT query - let rows = client - .query("SELECT blob_id FROM data_availability", &[]) - .await?; - - for row in rows { - let blob_id: &str = row.get(0); - let blob_id = blob_id.to_string(); - - if !blobs.contains_key(&blob_id) { - let blob = get(blob_id.clone()).await?; - blobs.insert(blob_id.clone(), blob.len()); - - if !blob.is_empty() { - BLOB_RETRIEVALS.inc(); // Increment counter if blob retrieval succeeds - } - } - BLOB_AVG_SIZE.set(blobs.values().sum::() as f64 / blobs.len() as f64); - } - - Ok(()) -} - -async fn get(commitment: String) -> Result, Box> { - let url = format!("http://host.docker.internal:4242/get/0x{commitment}"); - - let client = reqwest::Client::new(); - let response = client.get(url).send().await?; - - if response.status().is_success() { - // Expecting the response body to be binary data - let body = response.bytes().await?; - Ok(body.to_vec()) - } else { - Ok(vec![]) - } -} - -// Start the Prometheus metrics server -async fn start_metrics_server() { - let make_svc = - make_service_fn(|_conn| async { Ok::<_, Infallible>(service_fn(metrics_handler)) }); - - let addr = ([0, 0, 0, 0], 7070).into(); - let server = Server::bind(&addr).serve(make_svc); - - println!("Serving metrics on http://{}", addr); - server.await.unwrap(); -} - -// Handle the /metrics endpoint -async fn metrics_handler(_: Request) -> Result, Infallible> { - let encoder = TextEncoder::new(); - let metric_families = prometheus::gather(); - let mut buffer = Vec::new(); - encoder.encode(&metric_families, &mut buffer).unwrap(); - - Ok(Response::new(Body::from(buffer))) -}