From 2b77e8efacb447480bccb95abaed38ad061200dc Mon Sep 17 00:00:00 2001 From: ltitanb Date: Wed, 24 Jul 2024 14:05:08 +0100 Subject: [PATCH] expand metrics --- crates/pbs/src/constants.rs | 4 ++ crates/pbs/src/error.rs | 29 +++++---- crates/pbs/src/lib.rs | 1 + crates/pbs/src/metrics.rs | 60 +++++++++++-------- crates/pbs/src/mev_boost/get_header.rs | 22 ++++--- .../pbs/src/mev_boost/register_validator.rs | 12 ++-- crates/pbs/src/mev_boost/status.rs | 10 ++-- crates/pbs/src/mev_boost/submit_block.rs | 14 +++-- crates/pbs/src/routes/get_header.rs | 12 +++- crates/pbs/src/routes/register_validator.rs | 11 +++- crates/pbs/src/routes/status.rs | 12 +++- crates/pbs/src/routes/submit_block.rs | 12 +++- crates/pbs/src/service.rs | 3 +- 13 files changed, 134 insertions(+), 68 deletions(-) create mode 100644 crates/pbs/src/constants.rs diff --git a/crates/pbs/src/constants.rs b/crates/pbs/src/constants.rs new file mode 100644 index 0000000..050be53 --- /dev/null +++ b/crates/pbs/src/constants.rs @@ -0,0 +1,4 @@ +pub(crate) const STATUS_ENDPOINT_TAG: &str = "status"; +pub(crate) const REGISTER_VALIDATOR_ENDPOINT_TAG: &str = "register_validator"; +pub(crate) const SUBMIT_BLINDED_BLOCK_ENDPOINT_TAG: &str = "submit_blinded_block"; +pub(crate) const GET_HEADER_ENDPOINT_TAG: &str = "get_header"; diff --git a/crates/pbs/src/error.rs b/crates/pbs/src/error.rs index 34592bc..9b2c2a0 100644 --- a/crates/pbs/src/error.rs +++ b/crates/pbs/src/error.rs @@ -1,5 +1,7 @@ -use alloy::primitives::{B256, U256}; -use alloy::rpc::types::beacon::BlsPublicKey; +use alloy::{ + primitives::{B256, U256}, + rpc::types::beacon::BlsPublicKey, +}; use axum::{http::StatusCode, response::IntoResponse}; use thiserror::Error; @@ -10,19 +12,26 @@ pub enum PbsClientError { NoPayload, } -impl IntoResponse for PbsClientError { - fn into_response(self) -> axum::response::Response { +impl PbsClientError { + pub fn status_code(&self) -> StatusCode { match self { - PbsClientError::NoResponse => { - (StatusCode::SERVICE_UNAVAILABLE, "no response from relays").into_response() - } - PbsClientError::NoPayload => { - (StatusCode::BAD_GATEWAY, "no payload from relays").into_response() - } + PbsClientError::NoResponse => StatusCode::SERVICE_UNAVAILABLE, + PbsClientError::NoPayload => StatusCode::BAD_GATEWAY, } } } +impl IntoResponse for PbsClientError { + fn into_response(self) -> axum::response::Response { + let msg = match self { + PbsClientError::NoResponse => "no response from relays", + PbsClientError::NoPayload => "no payload from relays", + }; + + (self.status_code(), msg).into_response() + } +} + #[derive(Debug, Error)] pub enum PbsError { #[error("axum error: {0}")] diff --git a/crates/pbs/src/lib.rs b/crates/pbs/src/lib.rs index 0d0d08d..6acb002 100644 --- a/crates/pbs/src/lib.rs +++ b/crates/pbs/src/lib.rs @@ -1,6 +1,7 @@ // implements https://github.com/ethereum/builder-specs and multiplexes to multiple builderAPI compatible clients (ie MEV Boost relays) mod boost; +mod constants; mod error; mod metrics; mod mev_boost; diff --git a/crates/pbs/src/metrics.rs b/crates/pbs/src/metrics.rs index ab4ab67..d7f21dd 100644 --- a/crates/pbs/src/metrics.rs +++ b/crates/pbs/src/metrics.rs @@ -1,33 +1,43 @@ +//! Metrics for PBS module +//! We collect two types of metrics within the PBS module: +//! - what PBS receives from relays +//! - what PBS returns to the beacon node + use lazy_static::lazy_static; -use prometheus::{histogram_opts, opts, HistogramVec, IntCounterVec, Registry}; +use prometheus::{ + register_histogram_vec_with_registry, register_int_counter_vec_with_registry, HistogramVec, + IntCounterVec, Registry, +}; lazy_static! { - pub static ref PBS_METRICS_REGISTRY: prometheus::Registry = + pub static ref PBS_METRICS_REGISTRY: Registry = Registry::new_custom(Some("cb_pbs".to_string()), None).unwrap(); - pub static ref REQUESTS_RECEIVED: IntCounterVec = - IntCounterVec::new(opts!("requests_received", "Number of requests received"), &[ - "endpoint", - ]) - .unwrap(); - pub static ref RELAY_RESPONSES: IntCounterVec = - IntCounterVec::new(opts!("relay_response", "Number of requests received"), &[ - "code", "endpoint", "relay_id" - ]) - .unwrap(); - pub static ref RELAY_RESPONSE_TIME: HistogramVec = - HistogramVec::new(histogram_opts!("relay_response_time", "Relay response times"), &[ - "endpoint", "relay_id" - ]) - .unwrap(); -} -// TODO: this can be done with the macros, need to fix the types -pub fn register_default_metrics() { - PBS_METRICS_REGISTRY.register(Box::new(REQUESTS_RECEIVED.clone())).expect("failed to register"); + // FROM RELAYS + /// Status code received by relay by endpoint + pub static ref RELAY_STATUS_CODE: IntCounterVec = register_int_counter_vec_with_registry!( + "relay_status_code", + "HTTP status code received by relay", + &["http_status_code", "endpoint", "relay_id"], + PBS_METRICS_REGISTRY + ) + .unwrap(); - PBS_METRICS_REGISTRY.register(Box::new(RELAY_RESPONSES.clone())).expect("failed to register"); + /// Latency by relay by endpoint + pub static ref RELAY_LATENCY: HistogramVec = register_histogram_vec_with_registry!( + "relay_latency", + "HTTP latency by relay", + &["endpoint", "relay_id"], + PBS_METRICS_REGISTRY + ) + .unwrap(); - PBS_METRICS_REGISTRY - .register(Box::new(RELAY_RESPONSE_TIME.clone())) - .expect("failed to register"); + // TO BEACON NODE + /// Status code returned to beacon node by endpoint + pub static ref BEACON_NODE_STATUS: IntCounterVec = register_int_counter_vec_with_registry!( + "beacon_node_status_code", + "HTTP status code returned to beacon node", + &["http_status_code", "endpoint"], + PBS_METRICS_REGISTRY + ).unwrap(); } diff --git a/crates/pbs/src/mev_boost/get_header.rs b/crates/pbs/src/mev_boost/get_header.rs index bb8894e..10cd87a 100644 --- a/crates/pbs/src/mev_boost/get_header.rs +++ b/crates/pbs/src/mev_boost/get_header.rs @@ -1,7 +1,9 @@ use std::{sync::Arc, time::Duration}; -use alloy::primitives::{B256, U256}; -use alloy::rpc::types::beacon::BlsPublicKey; +use alloy::{ + primitives::{B256, U256}, + rpc::types::beacon::BlsPublicKey, +}; use axum::http::{HeaderMap, HeaderValue}; use cb_common::{ config::PbsConfig, @@ -15,8 +17,9 @@ use reqwest::{header::USER_AGENT, StatusCode}; use tracing::{debug, error}; use crate::{ + constants::GET_HEADER_ENDPOINT_TAG, error::{PbsError, ValidationError}, - metrics::{RELAY_RESPONSES, RELAY_RESPONSE_TIME}, + metrics::{RELAY_LATENCY, RELAY_STATUS_CODE}, state::{BuilderApiState, PbsState}, types::{SignedExecutionPayloadHeader, EMPTY_TX_ROOT_HASH}, GetHeaderParams, GetHeaderReponse, @@ -84,7 +87,8 @@ async fn send_get_header( ) -> Result, PbsError> { let url = relay.get_header_url(slot, parent_hash, validator_pubkey); - let timer = RELAY_RESPONSE_TIME.with_label_values(&["get_header", &relay.id]).start_timer(); + let timer = + RELAY_LATENCY.with_label_values(&[GET_HEADER_ENDPOINT_TAG, &relay.id]).start_timer(); let res = client .get(url) .timeout(Duration::from_millis(config.timeout_get_header_ms)) @@ -94,7 +98,9 @@ async fn send_get_header( timer.observe_duration(); let status = res.status(); - RELAY_RESPONSES.with_label_values(&[&status.to_string(), "get_header", &relay.id]).inc(); + RELAY_STATUS_CODE + .with_label_values(&[status.as_str(), GET_HEADER_ENDPOINT_TAG, &relay.id]) + .inc(); let response_bytes = res.bytes().await?; if !status.is_success() { @@ -184,8 +190,10 @@ fn validate_header( #[cfg(test)] mod tests { - use alloy::primitives::{B256, U256}; - use alloy::rpc::types::beacon::BlsPublicKey; + use alloy::{ + primitives::{B256, U256}, + rpc::types::beacon::BlsPublicKey, + }; use blst::min_pk; use cb_common::{pbs::RelayEntry, signature::sign_builder_message, types::Chain}; diff --git a/crates/pbs/src/mev_boost/register_validator.rs b/crates/pbs/src/mev_boost/register_validator.rs index 36d766c..5ace0af 100644 --- a/crates/pbs/src/mev_boost/register_validator.rs +++ b/crates/pbs/src/mev_boost/register_validator.rs @@ -11,8 +11,9 @@ use reqwest::header::USER_AGENT; use tracing::error; use crate::{ + constants::REGISTER_VALIDATOR_ENDPOINT_TAG, error::PbsError, - metrics::{RELAY_RESPONSES, RELAY_RESPONSE_TIME}, + metrics::{RELAY_LATENCY, RELAY_STATUS_CODE}, state::{BuilderApiState, PbsState}, }; @@ -62,8 +63,9 @@ async fn send_register_validator( ) -> Result<(), PbsError> { let url = relay.register_validator_url(); - let timer = - RELAY_RESPONSE_TIME.with_label_values(&["register_validator", &relay.id]).start_timer(); + let timer = RELAY_LATENCY + .with_label_values(&[REGISTER_VALIDATOR_ENDPOINT_TAG, &relay.id]) + .start_timer(); let res = client .post(url) .timeout(Duration::from_millis(timeout_ms)) @@ -76,7 +78,9 @@ async fn send_register_validator( // TODO: send to relay monitor let status = res.status(); - RELAY_RESPONSES.with_label_values(&[&status.to_string(), "get_header", &relay.id]).inc(); + RELAY_STATUS_CODE + .with_label_values(&[status.as_str(), REGISTER_VALIDATOR_ENDPOINT_TAG, &relay.id]) + .inc(); let response_bytes = res.bytes().await?; if !status.is_success() { diff --git a/crates/pbs/src/mev_boost/status.rs b/crates/pbs/src/mev_boost/status.rs index 42d3fff..240136e 100644 --- a/crates/pbs/src/mev_boost/status.rs +++ b/crates/pbs/src/mev_boost/status.rs @@ -6,13 +6,15 @@ use futures::future::select_ok; use reqwest::header::USER_AGENT; use crate::{ + constants::STATUS_ENDPOINT_TAG, error::PbsError, - metrics::{RELAY_RESPONSES, RELAY_RESPONSE_TIME}, + metrics::{RELAY_LATENCY, RELAY_STATUS_CODE}, state::{BuilderApiState, PbsState}, }; /// Implements https://ethereum.github.io/builder-specs/#/Builder/status -/// Broadcasts a status check to all relays and returns 200 if at least one relay returns 200 +/// Broadcasts a status check to all relays and returns 200 if at least one +/// relay returns 200 pub async fn get_status( req_headers: HeaderMap, state: PbsState, @@ -54,12 +56,12 @@ async fn send_relay_check( ) -> Result<(), PbsError> { let url = relay.get_status_url(); - let timer = RELAY_RESPONSE_TIME.with_label_values(&["get_status", &relay.id]).start_timer(); + let timer = RELAY_LATENCY.with_label_values(&[STATUS_ENDPOINT_TAG, &relay.id]).start_timer(); let res = client.get(url).timeout(Duration::from_secs(30)).headers(headers).send().await?; timer.observe_duration(); let status = res.status(); - RELAY_RESPONSES.with_label_values(&[&status.to_string(), "get_status", &relay.id]).inc(); + RELAY_STATUS_CODE.with_label_values(&[status.as_str(), STATUS_ENDPOINT_TAG, &relay.id]).inc(); let response_bytes = res.bytes().await?; if !status.is_success() { diff --git a/crates/pbs/src/mev_boost/submit_block.rs b/crates/pbs/src/mev_boost/submit_block.rs index fa33049..07c604b 100644 --- a/crates/pbs/src/mev_boost/submit_block.rs +++ b/crates/pbs/src/mev_boost/submit_block.rs @@ -11,8 +11,9 @@ use reqwest::header::USER_AGENT; use tracing::warn; use crate::{ + constants::SUBMIT_BLINDED_BLOCK_ENDPOINT_TAG, error::{PbsError, ValidationError}, - metrics::{RELAY_RESPONSES, RELAY_RESPONSE_TIME}, + metrics::{RELAY_LATENCY, RELAY_STATUS_CODE}, state::{BuilderApiState, PbsState}, types::{SignedBlindedBeaconBlock, SubmitBlindedBlockResponse}, }; @@ -63,7 +64,8 @@ pub async fn submit_block( } } -// submits blinded signed block and expects the execution payload + blobs bundle back +// submits blinded signed block and expects the execution payload + blobs bundle +// back async fn send_submit_block( headers: HeaderMap, relay: RelayEntry, @@ -73,7 +75,9 @@ async fn send_submit_block( ) -> Result { let url = relay.submit_block_url(); - let timer = RELAY_RESPONSE_TIME.with_label_values(&["submit_block", &relay.id]).start_timer(); + let timer = RELAY_LATENCY + .with_label_values(&[SUBMIT_BLINDED_BLOCK_ENDPOINT_TAG, &relay.id]) + .start_timer(); let res = client .post(url) .timeout(Duration::from_millis(config.timeout_get_payload_ms)) @@ -84,7 +88,9 @@ async fn send_submit_block( timer.observe_duration(); let status = res.status(); - RELAY_RESPONSES.with_label_values(&[&status.to_string(), "submit_block", &relay.id]).inc(); + RELAY_STATUS_CODE + .with_label_values(&[status.as_str(), SUBMIT_BLINDED_BLOCK_ENDPOINT_TAG, &relay.id]) + .inc(); let response_bytes = res.bytes().await?; if !status.is_success() { diff --git a/crates/pbs/src/routes/get_header.rs b/crates/pbs/src/routes/get_header.rs index 963fcef..f90c8a6 100644 --- a/crates/pbs/src/routes/get_header.rs +++ b/crates/pbs/src/routes/get_header.rs @@ -10,8 +10,9 @@ use uuid::Uuid; use crate::{ boost::BuilderApi, + constants::GET_HEADER_ENDPOINT_TAG, error::PbsClientError, - metrics::REQUESTS_RECEIVED, + metrics::BEACON_NODE_STATUS, state::{BuilderApiState, PbsState}, BuilderEvent, GetHeaderParams, }; @@ -21,7 +22,6 @@ pub async fn handle_get_header>( req_headers: HeaderMap, Path(params): Path, ) -> Result { - REQUESTS_RECEIVED.with_label_values(&["get_header"]).inc(); state.publish_event(BuilderEvent::GetHeaderRequest(params)); let req_id = Uuid::new_v4(); @@ -37,16 +37,22 @@ pub async fn handle_get_header>( if let Some(max_bid) = res { info!(event ="get_header", %req_id, block_hash =% max_bid.data.message.header.block_hash, "header available for slot"); + BEACON_NODE_STATUS.with_label_values(&["200", GET_HEADER_ENDPOINT_TAG]).inc(); Ok((StatusCode::OK, axum::Json(max_bid)).into_response()) } else { // spec: return 204 if request is valid but no bid available info!(event = "get_header", %req_id, "no header available for slot"); + BEACON_NODE_STATUS.with_label_values(&["204", GET_HEADER_ENDPOINT_TAG]).inc(); Ok(StatusCode::NO_CONTENT.into_response()) } } Err(err) => { error!(event = "get_header", %req_id, ?err, "failed relay get_header"); - Err(PbsClientError::NoPayload) + let err = PbsClientError::NoPayload; + BEACON_NODE_STATUS + .with_label_values(&[err.status_code().as_str(), GET_HEADER_ENDPOINT_TAG]) + .inc(); + Err(err) } } } diff --git a/crates/pbs/src/routes/register_validator.rs b/crates/pbs/src/routes/register_validator.rs index b689eae..21cea09 100644 --- a/crates/pbs/src/routes/register_validator.rs +++ b/crates/pbs/src/routes/register_validator.rs @@ -7,8 +7,9 @@ use uuid::Uuid; use crate::{ boost::BuilderApi, + constants::REGISTER_VALIDATOR_ENDPOINT_TAG, error::PbsClientError, - metrics::REQUESTS_RECEIVED, + metrics::BEACON_NODE_STATUS, state::{BuilderApiState, PbsState}, BuilderEvent, }; @@ -18,7 +19,6 @@ pub async fn handle_register_validator>( req_headers: HeaderMap, Json(registrations): Json>, ) -> Result { - REQUESTS_RECEIVED.with_label_values(&["register_validator"]).inc(); state.publish_event(BuilderEvent::RegisterValidatorRequest(registrations.clone())); let req_id = Uuid::new_v4(); @@ -30,9 +30,14 @@ pub async fn handle_register_validator>( state.publish_event(BuilderEvent::RegisterValidatorResponse); error!(method = "register_validator", %req_id, ?err, "all relays failed register_validator"); - Err(PbsClientError::NoResponse) + let err = PbsClientError::NoResponse; + BEACON_NODE_STATUS + .with_label_values(&[err.status_code().as_str(), REGISTER_VALIDATOR_ENDPOINT_TAG]) + .inc(); + Err(err) } else { info!(event = "register_validator", %req_id, "register validator successful"); + BEACON_NODE_STATUS.with_label_values(&["200", REGISTER_VALIDATOR_ENDPOINT_TAG]).inc(); Ok(StatusCode::OK) } } diff --git a/crates/pbs/src/routes/status.rs b/crates/pbs/src/routes/status.rs index a2b2388..d79ddd2 100644 --- a/crates/pbs/src/routes/status.rs +++ b/crates/pbs/src/routes/status.rs @@ -6,8 +6,9 @@ use uuid::Uuid; use crate::{ boost::BuilderApi, + constants::STATUS_ENDPOINT_TAG, error::PbsClientError, - metrics::REQUESTS_RECEIVED, + metrics::BEACON_NODE_STATUS, state::{BuilderApiState, PbsState}, BuilderEvent, }; @@ -17,7 +18,7 @@ pub async fn handle_get_status>( State(state): State>, ) -> Result { let req_id = Uuid::new_v4(); - REQUESTS_RECEIVED.with_label_values(&["get_status"]).inc(); + state.publish_event(BuilderEvent::GetStatusEvent); let ua = get_user_agent(&req_headers); @@ -28,11 +29,16 @@ pub async fn handle_get_status>( Ok(_) => { state.publish_event(BuilderEvent::GetStatusResponse); info!(method = "get_status", %req_id, "relay check successful"); + BEACON_NODE_STATUS.with_label_values(&["200", STATUS_ENDPOINT_TAG]).inc(); Ok(StatusCode::OK) } Err(err) => { error!(method = "get_status", %req_id, ?err, "all relays failed get_status"); - Err(PbsClientError::NoResponse) + let err = PbsClientError::NoResponse; + BEACON_NODE_STATUS + .with_label_values(&[err.status_code().as_str(), STATUS_ENDPOINT_TAG]) + .inc(); + Err(err) } } } diff --git a/crates/pbs/src/routes/submit_block.rs b/crates/pbs/src/routes/submit_block.rs index d06d33b..65aca13 100644 --- a/crates/pbs/src/routes/submit_block.rs +++ b/crates/pbs/src/routes/submit_block.rs @@ -6,8 +6,9 @@ use uuid::Uuid; use crate::{ boost::BuilderApi, + constants::SUBMIT_BLINDED_BLOCK_ENDPOINT_TAG, error::PbsClientError, - metrics::REQUESTS_RECEIVED, + metrics::BEACON_NODE_STATUS, state::{BuilderApiState, PbsState}, types::SignedBlindedBeaconBlock, BuilderEvent, @@ -20,7 +21,6 @@ pub async fn handle_submit_block>( req_headers: HeaderMap, Json(signed_blinded_block): Json, ) -> Result { - REQUESTS_RECEIVED.with_label_values(&["submit_block"]).inc(); state.publish_event(BuilderEvent::SubmitBlockRequest(Box::new(signed_blinded_block.clone()))); let req_id = Uuid::new_v4(); @@ -42,6 +42,8 @@ pub async fn handle_submit_block>( state.publish_event(BuilderEvent::SubmitBlockResponse(Box::new(res.clone()))); info!(method="submit_block", %req_id, "received unblinded block"); + + BEACON_NODE_STATUS.with_label_values(&["200", SUBMIT_BLINDED_BLOCK_ENDPOINT_TAG]).inc(); Ok((StatusCode::OK, Json(res).into_response())) } @@ -69,7 +71,11 @@ pub async fn handle_submit_block>( }); }; - Err(PbsClientError::NoPayload) + let err = PbsClientError::NoPayload; + BEACON_NODE_STATUS + .with_label_values(&[err.status_code().as_str(), SUBMIT_BLINDED_BLOCK_ENDPOINT_TAG]) + .inc(); + Err(err) } } } diff --git a/crates/pbs/src/service.rs b/crates/pbs/src/service.rs index dca7b2d..7ca965c 100644 --- a/crates/pbs/src/service.rs +++ b/crates/pbs/src/service.rs @@ -7,7 +7,7 @@ use tracing::{error, info}; use crate::{ boost::BuilderApi, - metrics::{register_default_metrics, PBS_METRICS_REGISTRY}, + metrics::PBS_METRICS_REGISTRY, routes::create_app_router, state::{BuilderApiState, PbsState}, }; @@ -39,7 +39,6 @@ impl PbsService { } pub fn init_metrics() { - register_default_metrics(); MetricsProvider::load_and_run(PBS_METRICS_REGISTRY.clone()); }