feat: add cosmos RPC failure / success metrics and alerts (hyperlane-…

…xyz#5523) ### Description add cosmos RPC failure / success metrics and alerts) ### RPC For RPC clients, I added a wrapper function around each call. This works out better because the HttpClient from tendermint's sdk is not very extensible, and most of the methods require us to manually provide the method metric. ### GRPC For GRPC, I added a wrapper around Tonic's Channel, inspired by code from https://docs.rs/tonic_prometheus_layer/0.1.11/tonic_prometheus_layer/ ### Drive-by changes - move a lot of logic `impl WasmProvider for WasmGrpcProvider` into just `WasmGrpcProvider` - this reduces the amount of code inside the closures ### Related issues hyperlane-xyz#5252 ### Backward compatibility ### Testing - add check to e2e to make sure metrics are being exported
Tessellated-io · Feb 28, 2025 · 018f671 · 018f671
1 parent e8851ae
commit 018f671
Show file tree

Hide file tree

Showing 29 changed files with 747 additions and 518 deletions.
diff --git a/rust/main/Cargo.lock b/rust/main/Cargo.lock
diff --git a/rust/main/Cargo.toml b/rust/main/Cargo.toml
@@ -100,6 +100,7 @@ num-traits = "0.2"
 once_cell = "1.18.0"
 parking_lot = "0.12"
 paste = "1.0"
+pin-project = "1.1.9"
 pretty_env_logger = "0.5.0"
 primitive-types = "=0.12.1"
 prometheus = "0.13"
@@ -148,6 +149,7 @@ tokio-metrics = { version = "0.4.0" }
 tokio-test = "0.4"
 toml_edit = "0.19.14"
 tonic = "0.12.3"
+tower = "*"
 tracing = { version = "0.1" }
 tracing-error = "0.2"
 tracing-futures = "0.2"

diff --git a/rust/main/chains/hyperlane-cosmos/Cargo.toml b/rust/main/chains/hyperlane-cosmos/Cargo.toml
@@ -19,17 +19,14 @@ derive-new = { workspace = true }
 futures = { workspace = true }
 hex = { workspace = true }
 http = { workspace = true }
-hyperlane-core = { path = "../../hyperlane-core", features = ["async"] }
-hyperlane-cosmwasm-interface.workspace = true
-hyperlane-operation-verifier = { path = "../../applications/hyperlane-operation-verifier" }
-hyperlane-warp-route = { path = "../../applications/hyperlane-warp-route" }
 hyper = { workspace = true }
 hyper-tls = { workspace = true }
 ibc-proto = { workspace = true }
 injective-protobuf = { workspace = true }
 injective-std = { workspace = true }
 itertools = { workspace = true }
 once_cell = { workspace = true }
+pin-project.workspace = true
 protobuf = { workspace = true }
 ripemd = { workspace = true }
 serde = { workspace = true }
@@ -47,6 +44,13 @@ tonic = { workspace = true, features = [
     "tls-roots",
     "tls-native-roots",
 ] }
+tower.workspace = true
 tracing = { workspace = true }
 tracing-futures = { workspace = true }
 url = { workspace = true }
+
+hyperlane-core = { path = "../../hyperlane-core", features = ["async"] }
+hyperlane-cosmwasm-interface.workspace = true
+hyperlane-metric = { path = "../../hyperlane-metric" }
+hyperlane-operation-verifier = { path = "../../applications/hyperlane-operation-verifier" }
+hyperlane-warp-route = { path = "../../applications/hyperlane-warp-route" }
diff --git a/rust/main/chains/hyperlane-cosmos/src/aggregation_ism.rs b/rust/main/chains/hyperlane-cosmos/src/aggregation_ism.rs
@@ -25,18 +25,7 @@ pub struct CosmosAggregationIsm {
 
 impl CosmosAggregationIsm {
     /// create new Cosmos AggregationIsm agent
-    pub fn new(
-        conf: ConnectionConf,
-        locator: ContractLocator,
-        signer: Option<Signer>,
-    ) -> ChainResult<Self> {
-        let provider = CosmosProvider::new(
-            locator.domain.clone(),
-            conf.clone(),
-            locator.clone(),
-            signer,
-        )?;
-
+    pub fn new(provider: CosmosProvider, locator: ContractLocator) -> ChainResult<Self> {
         Ok(Self {
             domain: locator.domain.clone(),
             address: locator.address,

diff --git a/rust/main/chains/hyperlane-cosmos/src/interchain_gas.rs b/rust/main/chains/hyperlane-cosmos/src/interchain_gas.rs
@@ -48,18 +48,7 @@ impl InterchainGasPaymaster for CosmosInterchainGasPaymaster {}
 
 impl CosmosInterchainGasPaymaster {
     /// create new Cosmos InterchainGasPaymaster agent
-    pub fn new(
-        conf: ConnectionConf,
-        locator: ContractLocator,
-        signer: Option<Signer>,
-    ) -> ChainResult<Self> {
-        let provider = CosmosProvider::new(
-            locator.domain.clone(),
-            conf.clone(),
-            locator.clone(),
-            signer,
-        )?;
-
+    pub fn new(provider: CosmosProvider, locator: ContractLocator) -> ChainResult<Self> {
         Ok(Self {
             domain: locator.domain.clone(),
             address: locator.address,
@@ -94,21 +83,10 @@ pub struct CosmosInterchainGasPaymasterIndexer {
 
 impl CosmosInterchainGasPaymasterIndexer {
     /// The interchain gas payment event type from the CW contract.
-    const INTERCHAIN_GAS_PAYMENT_EVENT_TYPE: &'static str = "igp-core-pay-for-gas";
+    pub const INTERCHAIN_GAS_PAYMENT_EVENT_TYPE: &'static str = "igp-core-pay-for-gas";
 
     /// create new Cosmos InterchainGasPaymasterIndexer agent
-    pub fn new(
-        conf: ConnectionConf,
-        locator: ContractLocator,
-        reorg_period: u32,
-    ) -> ChainResult<Self> {
-        let provider = CosmosWasmRpcProvider::new(
-            conf,
-            locator,
-            Self::INTERCHAIN_GAS_PAYMENT_EVENT_TYPE.into(),
-            reorg_period,
-        )?;
-
+    pub fn new(provider: CosmosWasmRpcProvider) -> ChainResult<Self> {
         Ok(Self {
             provider: Box::new(provider),
         })

diff --git a/rust/main/chains/hyperlane-cosmos/src/interchain_security_module.rs b/rust/main/chains/hyperlane-cosmos/src/interchain_security_module.rs
@@ -30,18 +30,7 @@ pub struct CosmosInterchainSecurityModule {
 /// The Cosmos Interchain Security Module Implementation.
 impl CosmosInterchainSecurityModule {
     /// Creates a new Cosmos Interchain Security Module.
-    pub fn new(
-        conf: &ConnectionConf,
-        locator: ContractLocator,
-        signer: Option<Signer>,
-    ) -> ChainResult<Self> {
-        let provider = CosmosProvider::new(
-            locator.domain.clone(),
-            conf.clone(),
-            locator.clone(),
-            signer,
-        )?;
-
+    pub fn new(provider: CosmosProvider, locator: ContractLocator) -> ChainResult<Self> {
         Ok(Self {
             domain: locator.domain.clone(),
             address: locator.address,

diff --git a/rust/main/chains/hyperlane-cosmos/src/lib.rs b/rust/main/chains/hyperlane-cosmos/src/lib.rs
@@ -17,6 +17,7 @@ mod mailbox;
 mod merkle_tree_hook;
 mod multisig_ism;
 mod payloads;
+mod prometheus;
 mod providers;
 mod routing_ism;
 mod rpc_clients;

diff --git a/rust/main/chains/hyperlane-cosmos/src/mailbox.rs b/rust/main/chains/hyperlane-cosmos/src/mailbox.rs
@@ -3,5 +3,8 @@ pub use delivery_indexer::CosmosMailboxDeliveryIndexer;
 pub use dispatch_indexer::CosmosMailboxDispatchIndexer;
 
 mod contract;
-mod delivery_indexer;
-mod dispatch_indexer;
+
+/// Cosmos Mailbox Delivery Indexer
+pub mod delivery_indexer;
+/// Cosmos Mailbox Dispatch Indexer
+pub mod dispatch_indexer;
diff --git a/rust/main/chains/hyperlane-cosmos/src/mailbox/contract.rs b/rust/main/chains/hyperlane-cosmos/src/mailbox/contract.rs
@@ -33,17 +33,10 @@ impl CosmosMailbox {
     /// Create a reference to a mailbox at a specific Cosmos address on some
     /// chain
     pub fn new(
+        provider: CosmosProvider,
         conf: ConnectionConf,
         locator: ContractLocator,
-        signer: Option<Signer>,
     ) -> ChainResult<Self> {
-        let provider = CosmosProvider::new(
-            locator.domain.clone(),
-            conf.clone(),
-            locator.clone(),
-            signer,
-        )?;
-
         Ok(Self {
             config: conf,
             domain: locator.domain.clone(),

diff --git a/rust/main/chains/hyperlane-cosmos/src/mailbox/delivery_indexer.rs b/rust/main/chains/hyperlane-cosmos/src/mailbox/delivery_indexer.rs
@@ -21,7 +21,7 @@ use crate::utils::{
 use crate::{ConnectionConf, HyperlaneCosmosError, Signer};
 
 /// The message process event type from the CW contract.
-const MESSAGE_DELIVERY_EVENT_TYPE: &str = "mailbox_process_id";
+pub const MESSAGE_DELIVERY_EVENT_TYPE: &str = "mailbox_process_id";
 const MESSAGE_ID_ATTRIBUTE_KEY: &str = "message_id";
 static MESSAGE_ID_ATTRIBUTE_KEY_BASE64: Lazy<String> =
     Lazy::new(|| BASE64.encode(MESSAGE_ID_ATTRIBUTE_KEY));
@@ -34,21 +34,9 @@ pub struct CosmosMailboxDeliveryIndexer {
 impl CosmosMailboxDeliveryIndexer {
     /// Create a reference to a mailbox at a specific Cosmos address on some
     /// chain
-    pub fn new(
-        conf: ConnectionConf,
-        locator: ContractLocator,
-        signer: Option<Signer>,
-        reorg_period: u32,
-    ) -> ChainResult<Self> {
-        let provider = CosmosWasmRpcProvider::new(
-            conf,
-            locator,
-            MESSAGE_DELIVERY_EVENT_TYPE.to_owned(),
-            reorg_period,
-        )?;
-
+    pub fn new(wasm_provider: CosmosWasmRpcProvider) -> ChainResult<Self> {
         Ok(Self {
-            provider: Box::new(provider),
+            provider: Box::new(wasm_provider),
         })
     }
 

diff --git a/rust/main/chains/hyperlane-cosmos/src/mailbox/dispatch_indexer.rs b/rust/main/chains/hyperlane-cosmos/src/mailbox/dispatch_indexer.rs
@@ -20,7 +20,7 @@ use crate::utils::{
 use crate::{ConnectionConf, CosmosMailbox, HyperlaneCosmosError, Signer};
 
 /// The message dispatch event type from the CW contract.
-const MESSAGE_DISPATCH_EVENT_TYPE: &str = "mailbox_dispatch";
+pub const MESSAGE_DISPATCH_EVENT_TYPE: &str = "mailbox_dispatch";
 const MESSAGE_ATTRIBUTE_KEY: &str = "message";
 static MESSAGE_ATTRIBUTE_KEY_BASE64: Lazy<String> =
     Lazy::new(|| BASE64.encode(MESSAGE_ATTRIBUTE_KEY));
@@ -35,23 +35,10 @@ pub struct CosmosMailboxDispatchIndexer {
 impl CosmosMailboxDispatchIndexer {
     /// Create a reference to a mailbox at a specific Cosmos address on some
     /// chain
-    pub fn new(
-        conf: ConnectionConf,
-        locator: ContractLocator,
-        signer: Option<Signer>,
-        reorg_period: u32,
-    ) -> ChainResult<Self> {
-        let mailbox = CosmosMailbox::new(conf.clone(), locator.clone(), signer.clone())?;
-        let provider = CosmosWasmRpcProvider::new(
-            conf,
-            locator,
-            MESSAGE_DISPATCH_EVENT_TYPE.into(),
-            reorg_period,
-        )?;
-
+    pub fn new(wasm_provider: CosmosWasmRpcProvider, mailbox: CosmosMailbox) -> ChainResult<Self> {
         Ok(Self {
             mailbox,
-            provider: Box::new(provider),
+            provider: Box::new(wasm_provider),
         })
     }
 

diff --git a/rust/main/chains/hyperlane-cosmos/src/merkle_tree_hook.rs b/rust/main/chains/hyperlane-cosmos/src/merkle_tree_hook.rs
@@ -35,18 +35,7 @@ pub struct CosmosMerkleTreeHook {
 
 impl CosmosMerkleTreeHook {
     /// create new Cosmos MerkleTreeHook agent
-    pub fn new(
-        conf: ConnectionConf,
-        locator: ContractLocator,
-        signer: Option<Signer>,
-    ) -> ChainResult<Self> {
-        let provider = CosmosProvider::new(
-            locator.domain.clone(),
-            conf.clone(),
-            locator.clone(),
-            signer,
-        )?;
-
+    pub fn new(provider: CosmosProvider, locator: ContractLocator) -> ChainResult<Self> {
         Ok(Self {
             domain: locator.domain.clone(),
             address: locator.address,
@@ -197,25 +186,17 @@ pub struct CosmosMerkleTreeHookIndexer {
 
 impl CosmosMerkleTreeHookIndexer {
     /// The message dispatch event type from the CW contract.
-    const MERKLE_TREE_INSERTION_EVENT_TYPE: &'static str = "hpl_hook_merkle::post_dispatch";
+    pub const MERKLE_TREE_INSERTION_EVENT_TYPE: &'static str = "hpl_hook_merkle::post_dispatch";
 
     /// create new Cosmos MerkleTreeHookIndexer agent
     pub fn new(
-        conf: ConnectionConf,
+        provider: CosmosProvider,
+        wasm_provider: CosmosWasmRpcProvider,
         locator: ContractLocator,
-        signer: Option<Signer>,
-        reorg_period: u32,
     ) -> ChainResult<Self> {
-        let provider = CosmosWasmRpcProvider::new(
-            conf.clone(),
-            locator.clone(),
-            Self::MERKLE_TREE_INSERTION_EVENT_TYPE.into(),
-            reorg_period,
-        )?;
-
         Ok(Self {
-            merkle_tree_hook: CosmosMerkleTreeHook::new(conf, locator, signer)?,
-            provider: Box::new(provider),
+            merkle_tree_hook: CosmosMerkleTreeHook::new(provider, locator)?,
+            provider: Box::new(wasm_provider),
         })
     }
 

diff --git a/rust/main/chains/hyperlane-cosmos/src/multisig_ism.rs b/rust/main/chains/hyperlane-cosmos/src/multisig_ism.rs
@@ -22,18 +22,7 @@ pub struct CosmosMultisigIsm {
 
 impl CosmosMultisigIsm {
     /// create a new instance of CosmosMultisigIsm
-    pub fn new(
-        conf: ConnectionConf,
-        locator: ContractLocator,
-        signer: Option<Signer>,
-    ) -> ChainResult<Self> {
-        let provider = CosmosProvider::new(
-            locator.domain.clone(),
-            conf.clone(),
-            locator.clone(),
-            signer,
-        )?;
-
+    pub fn new(provider: CosmosProvider, locator: ContractLocator) -> ChainResult<Self> {
         Ok(Self {
             domain: locator.domain.clone(),
             address: locator.address,