diff --git a/.dockerignore b/.dockerignore index b57cc807a9..c4dbfb305f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,7 +8,6 @@ typescript/hyperlane-deploy/.env **/*.swp **/*.swo -rust tmp.env .DS_STORE diff --git a/.github/workflows/rust-docker.yml b/.github/workflows/rust-docker.yml index 534743b82f..bc70a1f867 100644 --- a/.github/workflows/rust-docker.yml +++ b/.github/workflows/rust-docker.yml @@ -65,7 +65,7 @@ jobs: - name: Build and push uses: docker/build-push-action@v5 with: - context: ./rust + context: . file: ./rust/Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 5bf77c0a2c..1e225e68f9 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -4191,6 +4191,7 @@ dependencies = [ name = "hyperlane-base" version = "0.1.0" dependencies = [ + "anyhow", "async-trait", "axum", "backtrace", @@ -4239,6 +4240,7 @@ dependencies = [ "tracing-subscriber", "tracing-test", "url", + "vergen", "walkdir", "warp", "ya-gcp", @@ -7312,6 +7314,7 @@ dependencies = [ name = "run-locally" version = "0.1.0" dependencies = [ + "anyhow", "cosmwasm-schema", "ctrlc", "ethers", @@ -7319,6 +7322,7 @@ dependencies = [ "ethers-core", "eyre", "hex 0.4.3", + "hyperlane-base", "hyperlane-core", "hyperlane-cosmos", "hyperlane-cosmwasm-interface", @@ -7338,6 +7342,7 @@ dependencies = [ "tokio", "toml_edit 0.19.15", "ureq", + "vergen", "which", ] @@ -10859,6 +10864,18 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" +[[package]] +name = "vergen" +version = "8.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2990d9ea5967266ea0ccf413a4aa5c42a93dbcfda9cb49a97de6931726b12566" +dependencies = [ + "anyhow", + "cfg-if", + "rustversion", + "time", +] + [[package]] name = "version_check" version = "0.9.4" diff --git a/rust/Dockerfile b/rust/Dockerfile index 7a5c882608..75be920015 100644 --- a/rust/Dockerfile +++ b/rust/Dockerfile @@ -9,18 +9,24 @@ RUN apt-get update && \ apt-get install -y musl-tools clang && \ rustup target add x86_64-unknown-linux-musl -# Add workspace to workdir -COPY agents ./agents -COPY chains ./chains -COPY hyperlane-base ./hyperlane-base -COPY hyperlane-core ./hyperlane-core -COPY hyperlane-test ./hyperlane-test -COPY ethers-prometheus ./ethers-prometheus -COPY utils ./utils -COPY sealevel ./sealevel - -COPY Cargo.toml . -COPY Cargo.lock . +RUN mkdir rust + +# Add workspace to workdir +COPY rust/agents rust/agents +COPY rust/chains rust/chains +COPY rust/hyperlane-base rust/hyperlane-base +COPY rust/hyperlane-core rust/hyperlane-core +COPY rust/hyperlane-test rust/hyperlane-test +COPY rust/ethers-prometheus rust/ethers-prometheus +COPY rust/utils rust/utils +COPY rust/sealevel rust/sealevel + +COPY rust/Cargo.toml rust/. +COPY rust/Cargo.lock rust/. + +COPY .git .git + +WORKDIR /usr/src/rust # Build binaries RUN \ @@ -29,9 +35,9 @@ RUN \ --mount=id=cargo-home-git,type=cache,sharing=locked,target=/usr/local/cargo/git \ RUSTFLAGS="--cfg tokio_unstable" cargo build --release --bin validator --bin relayer --bin scraper && \ mkdir -p /release && \ - cp /usr/src/target/release/validator /release && \ - cp /usr/src/target/release/relayer /release && \ - cp /usr/src/target/release/scraper /release + cp /usr/src/rust/target/release/validator /release && \ + cp /usr/src/rust/target/release/relayer /release && \ + cp /usr/src/rust/target/release/scraper /release ## 2: Copy the binaries to release image FROM ubuntu:22.04 @@ -43,7 +49,7 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* WORKDIR /app -COPY config ./config +COPY rust/config ./config COPY --from=builder /release/* . RUN chmod 777 /app && \ diff --git a/rust/agents/relayer/src/relayer.rs b/rust/agents/relayer/src/relayer.rs index 3f7c99c8c6..496628d700 100644 --- a/rust/agents/relayer/src/relayer.rs +++ b/rust/agents/relayer/src/relayer.rs @@ -13,8 +13,8 @@ use hyperlane_base::{ db::{HyperlaneRocksDB, DB}, metrics::{AgentMetrics, MetricsUpdater}, settings::ChainConf, - BaseAgent, ChainMetrics, ContractSyncMetrics, ContractSyncer, CoreMetrics, HyperlaneAgentCore, - SyncOptions, + AgentMetadata, BaseAgent, ChainMetrics, ContractSyncMetrics, ContractSyncer, CoreMetrics, + HyperlaneAgentCore, SyncOptions, }; use hyperlane_core::{ HyperlaneDomain, HyperlaneMessage, InterchainGasPayment, MerkleTreeInsertion, QueueOperation, @@ -113,6 +113,7 @@ impl BaseAgent for Relayer { type Settings = RelayerSettings; async fn from_settings( + _agent_metadata: AgentMetadata, settings: Self::Settings, core_metrics: Arc, agent_metrics: AgentMetrics, diff --git a/rust/agents/scraper/src/agent.rs b/rust/agents/scraper/src/agent.rs index b0c787db4b..3febc40927 100644 --- a/rust/agents/scraper/src/agent.rs +++ b/rust/agents/scraper/src/agent.rs @@ -4,8 +4,8 @@ use async_trait::async_trait; use derive_more::AsRef; use futures::future::try_join_all; use hyperlane_base::{ - broadcast::BroadcastMpscSender, metrics::AgentMetrics, settings::IndexSettings, BaseAgent, - ChainMetrics, ContractSyncMetrics, ContractSyncer, CoreMetrics, HyperlaneAgentCore, + broadcast::BroadcastMpscSender, metrics::AgentMetrics, settings::IndexSettings, AgentMetadata, + BaseAgent, ChainMetrics, ContractSyncMetrics, ContractSyncer, CoreMetrics, HyperlaneAgentCore, MetricsUpdater, SyncOptions, }; use hyperlane_core::{Delivery, HyperlaneDomain, HyperlaneMessage, InterchainGasPayment, H512}; @@ -41,6 +41,7 @@ impl BaseAgent for Scraper { type Settings = ScraperSettings; async fn from_settings( + _agent_metadata: AgentMetadata, settings: Self::Settings, metrics: Arc, agent_metrics: AgentMetrics, diff --git a/rust/agents/validator/src/validator.rs b/rust/agents/validator/src/validator.rs index 23e96aeb58..bb7339e6b9 100644 --- a/rust/agents/validator/src/validator.rs +++ b/rust/agents/validator/src/validator.rs @@ -13,8 +13,8 @@ use hyperlane_base::{ db::{HyperlaneRocksDB, DB}, metrics::AgentMetrics, settings::ChainConf, - BaseAgent, ChainMetrics, CheckpointSyncer, ContractSyncMetrics, ContractSyncer, CoreMetrics, - HyperlaneAgentCore, MetricsUpdater, SequencedDataContractSync, + AgentMetadata, BaseAgent, ChainMetrics, CheckpointSyncer, ContractSyncMetrics, ContractSyncer, + CoreMetrics, HyperlaneAgentCore, MetricsUpdater, SequencedDataContractSync, }; use hyperlane_core::{ @@ -50,6 +50,7 @@ pub struct Validator { core_metrics: Arc, agent_metrics: AgentMetrics, chain_metrics: ChainMetrics, + agent_metadata: AgentMetadata, } #[async_trait] @@ -59,6 +60,7 @@ impl BaseAgent for Validator { type Settings = ValidatorSettings; async fn from_settings( + agent_metadata: AgentMetadata, settings: Self::Settings, metrics: Arc, agent_metrics: AgentMetrics, @@ -123,6 +125,7 @@ impl BaseAgent for Validator { agent_metrics, chain_metrics, core_metrics: metrics, + agent_metadata, }) } @@ -169,6 +172,11 @@ impl BaseAgent for Validator { .instrument(info_span!("MetricsUpdater")), ); + // report agent metadata + self.metadata() + .await + .expect("Failed to report agent metadata"); + // announce the validator after spawning the signer task self.announce().await.expect("Failed to announce validator"); @@ -290,6 +298,14 @@ impl Validator { } } + async fn metadata(&self) -> Result<()> { + self.checkpoint_syncer + .write_metadata(&self.agent_metadata) + .await?; + + Ok(()) + } + async fn announce(&self) -> Result<()> { let address = self.signer.eth_address(); let announcement_location = self.checkpoint_syncer.announcement_location(); diff --git a/rust/hyperlane-base/Cargo.toml b/rust/hyperlane-base/Cargo.toml index 0564c06a7e..471815f698 100644 --- a/rust/hyperlane-base/Cargo.toml +++ b/rust/hyperlane-base/Cargo.toml @@ -70,6 +70,10 @@ tempfile.workspace = true tracing-test.workspace = true walkdir.workspace = true +[build-dependencies] +anyhow = { workspace = true } +vergen = { version = "8.3.2", features = ["build", "git", "gitcl"] } + [features] default = ["oneline-errors", "color-eyre"] oneline-eyre = ["backtrace-oneline", "backtrace"] diff --git a/rust/hyperlane-base/build.rs b/rust/hyperlane-base/build.rs new file mode 100644 index 0000000000..ca3892c1db --- /dev/null +++ b/rust/hyperlane-base/build.rs @@ -0,0 +1,8 @@ +use anyhow::Result; +use vergen::EmitBuilder; + +fn main() -> Result<()> { + EmitBuilder::builder().git_sha(false).emit()?; + + Ok(()) +} diff --git a/rust/hyperlane-base/src/agent.rs b/rust/hyperlane-base/src/agent.rs index 153526d584..8bd696ccf5 100644 --- a/rust/hyperlane-base/src/agent.rs +++ b/rust/hyperlane-base/src/agent.rs @@ -1,3 +1,5 @@ +pub use crate::metadata::AgentMetadata; + use std::{env, fmt::Debug, sync::Arc}; use async_trait::async_trait; @@ -40,6 +42,7 @@ pub trait BaseAgent: Send + Sync + Debug { /// Instantiate the agent from the standard settings object async fn from_settings( + agent_metadata: AgentMetadata, settings: Self::Settings, metrics: Arc, agent_metrics: AgentMetrics, @@ -72,6 +75,13 @@ pub async fn agent_main() -> Result<()> { color_eyre::install()?; } + // Latest git commit hash at the time when agent was built. + // If .git was not present at the time of build, + // the variable defaults to "VERGEN_IDEMPOTENT_OUTPUT". + let git_sha = env!("VERGEN_GIT_SHA").to_owned(); + + let agent_metadata = AgentMetadata::new(git_sha); + let settings = A::Settings::load()?; let core_settings: &Settings = settings.as_ref(); @@ -80,6 +90,7 @@ pub async fn agent_main() -> Result<()> { let agent_metrics = create_agent_metrics(&metrics)?; let chain_metrics = create_chain_metrics(&metrics)?; let agent = A::from_settings( + agent_metadata, settings, metrics.clone(), agent_metrics, diff --git a/rust/hyperlane-base/src/lib.rs b/rust/hyperlane-base/src/lib.rs index 7ff1fc2352..7f52c2e5bf 100644 --- a/rust/hyperlane-base/src/lib.rs +++ b/rust/hyperlane-base/src/lib.rs @@ -15,6 +15,8 @@ pub use agent::*; /// The local database used by agents pub mod db; +mod metadata; + pub mod metrics; pub use metrics::*; diff --git a/rust/hyperlane-base/src/metadata.rs b/rust/hyperlane-base/src/metadata.rs new file mode 100644 index 0000000000..3bdfce9af8 --- /dev/null +++ b/rust/hyperlane-base/src/metadata.rs @@ -0,0 +1,9 @@ +use derive_new::new; +use serde::{Deserialize, Serialize}; + +/// Metadata about agent +#[derive(Debug, Deserialize, Serialize, new)] +pub struct AgentMetadata { + /// Contains git commit hash of the agent binary + pub git_sha: String, +} diff --git a/rust/hyperlane-base/src/traits/checkpoint_syncer.rs b/rust/hyperlane-base/src/traits/checkpoint_syncer.rs index abec982c7d..44828bcbea 100644 --- a/rust/hyperlane-base/src/traits/checkpoint_syncer.rs +++ b/rust/hyperlane-base/src/traits/checkpoint_syncer.rs @@ -3,6 +3,7 @@ use std::fmt::Debug; use async_trait::async_trait; use eyre::Result; +use crate::AgentMetadata; use hyperlane_core::{SignedAnnouncement, SignedCheckpointWithMessageId}; /// A generic trait to read/write Checkpoints offchain @@ -27,6 +28,8 @@ pub trait CheckpointSyncer: Debug + Send + Sync { &self, signed_checkpoint: &SignedCheckpointWithMessageId, ) -> Result<()>; + /// Write the agent metadata to this syncer + async fn write_metadata(&self, metadata: &AgentMetadata) -> Result<()>; /// Write the signed announcement to this syncer async fn write_announcement(&self, signed_announcement: &SignedAnnouncement) -> Result<()>; /// Return the announcement storage location for this syncer diff --git a/rust/hyperlane-base/src/types/gcs_storage.rs b/rust/hyperlane-base/src/types/gcs_storage.rs index fbe5c34963..6094ae8c35 100644 --- a/rust/hyperlane-base/src/types/gcs_storage.rs +++ b/rust/hyperlane-base/src/types/gcs_storage.rs @@ -1,4 +1,4 @@ -use crate::CheckpointSyncer; +use crate::{AgentMetadata, CheckpointSyncer}; use async_trait::async_trait; use derive_new::new; use eyre::{bail, Result}; @@ -7,6 +7,7 @@ use std::fmt; use ya_gcp::{storage::StorageClient, AuthFlow, ClientBuilder, ClientBuilderConfig}; const LATEST_INDEX_KEY: &str = "gcsLatestIndexKey"; +const METADATA_KEY: &str = "gcsMetadataKey"; const ANNOUNCEMENT_KEY: &str = "gcsAnnouncementKey"; /// Path to GCS users_secret file pub const GCS_USER_SECRET: &str = "GCS_USER_SECRET"; @@ -174,6 +175,15 @@ impl CheckpointSyncer for GcsStorageClient { Ok(()) } + /// Write the agent metadata to this syncer + async fn write_metadata(&self, metadata: &AgentMetadata) -> Result<()> { + let serialized_metadata = serde_json::to_string_pretty(metadata)?; + self.inner + .insert_object(&self.bucket, METADATA_KEY, serialized_metadata) + .await?; + Ok(()) + } + /// Write the signed announcement to this syncer async fn write_announcement(&self, signed_announcement: &SignedAnnouncement) -> Result<()> { self.inner diff --git a/rust/hyperlane-base/src/types/local_storage.rs b/rust/hyperlane-base/src/types/local_storage.rs index 38397c1bf6..71047f2464 100644 --- a/rust/hyperlane-base/src/types/local_storage.rs +++ b/rust/hyperlane-base/src/types/local_storage.rs @@ -1,12 +1,12 @@ use std::path::PathBuf; +use crate::traits::CheckpointSyncer; +use crate::AgentMetadata; use async_trait::async_trait; use eyre::{Context, Result}; use hyperlane_core::{SignedAnnouncement, SignedCheckpointWithMessageId}; use prometheus::IntGauge; -use crate::traits::CheckpointSyncer; - #[derive(Debug, Clone)] /// Type for reading/write to LocalStorage pub struct LocalStorage { @@ -40,6 +40,10 @@ impl LocalStorage { fn announcement_file_path(&self) -> PathBuf { self.path.join("announcement.json") } + + fn metadata_file_path(&self) -> PathBuf { + self.path.join("metadata_latest.json") + } } #[async_trait] @@ -91,6 +95,15 @@ impl CheckpointSyncer for LocalStorage { Ok(()) } + async fn write_metadata(&self, metadata: &AgentMetadata) -> Result<()> { + let serialized_metadata = serde_json::to_string_pretty(metadata)?; + let path = self.metadata_file_path(); + tokio::fs::write(&path, &serialized_metadata) + .await + .with_context(|| format!("Writing agent metadata to {path:?}"))?; + Ok(()) + } + async fn write_announcement(&self, signed_announcement: &SignedAnnouncement) -> Result<()> { let serialized_announcement = serde_json::to_string_pretty(signed_announcement)?; let path = self.announcement_file_path(); diff --git a/rust/hyperlane-base/src/types/s3_storage.rs b/rust/hyperlane-base/src/types/s3_storage.rs index 4db179ad4d..4d03ff0406 100644 --- a/rust/hyperlane-base/src/types/s3_storage.rs +++ b/rust/hyperlane-base/src/types/s3_storage.rs @@ -14,7 +14,9 @@ use rusoto_s3::{GetObjectError, GetObjectRequest, PutObjectRequest, S3Client, S3 use tokio::time::timeout; use crate::types::utils; -use crate::{settings::aws_credentials::AwsChainCredentialsProvider, CheckpointSyncer}; +use crate::{ + settings::aws_credentials::AwsChainCredentialsProvider, AgentMetadata, CheckpointSyncer, +}; /// The timeout for S3 requests. Rusoto doesn't offer timeout configuration /// out of the box, so S3 requests must be wrapped with a timeout. @@ -136,6 +138,10 @@ impl S3Storage { "checkpoint_latest_index.json".to_owned() } + fn metadata_key() -> String { + "metadata_latest.json".to_owned() + } + fn announcement_key() -> String { "announcement.json".to_owned() } @@ -188,6 +194,13 @@ impl CheckpointSyncer for S3Storage { Ok(()) } + async fn write_metadata(&self, metadata: &AgentMetadata) -> Result<()> { + let serialized_metadata = serde_json::to_string_pretty(metadata)?; + self.write_to_bucket(S3Storage::metadata_key(), &serialized_metadata) + .await?; + Ok(()) + } + async fn write_announcement(&self, signed_announcement: &SignedAnnouncement) -> Result<()> { let serialized_announcement = serde_json::to_string_pretty(signed_announcement)?; self.write_to_bucket(S3Storage::announcement_key(), &serialized_announcement) diff --git a/rust/utils/run-locally/Cargo.toml b/rust/utils/run-locally/Cargo.toml index 99b0e41c9b..06a2b5cdd7 100644 --- a/rust/utils/run-locally/Cargo.toml +++ b/rust/utils/run-locally/Cargo.toml @@ -10,6 +10,7 @@ publish.workspace = true version.workspace = true [dependencies] +hyperlane-base = { path = "../../hyperlane-base" } hyperlane-core = { path = "../../hyperlane-core", features = ["float"]} hyperlane-cosmos = { path = "../../chains/hyperlane-cosmos"} toml_edit.workspace = true @@ -38,5 +39,9 @@ relayer = { path = "../../agents/relayer"} hyperlane-cosmwasm-interface.workspace = true cosmwasm-schema.workspace = true +[build-dependencies] +anyhow = { workspace = true } +vergen = { version = "8.3.2", features = ["build", "git", "gitcl"] } + [features] cosmos = [] \ No newline at end of file diff --git a/rust/utils/run-locally/build.rs b/rust/utils/run-locally/build.rs new file mode 100644 index 0000000000..ca3892c1db --- /dev/null +++ b/rust/utils/run-locally/build.rs @@ -0,0 +1,8 @@ +use anyhow::Result; +use vergen::EmitBuilder; + +fn main() -> Result<()> { + EmitBuilder::builder().git_sha(false).emit()?; + + Ok(()) +} diff --git a/rust/utils/run-locally/src/invariants.rs b/rust/utils/run-locally/src/invariants.rs index 44a6ad4dc5..13fb465b5f 100644 --- a/rust/utils/run-locally/src/invariants.rs +++ b/rust/utils/run-locally/src/invariants.rs @@ -1,213 +1,7 @@ -use std::fs::File; -use std::path::Path; +pub use common::SOL_MESSAGES_EXPECTED; +pub use post_startup_invariants::post_startup_invariants; +pub use termination_invariants::termination_invariants_met; -use crate::config::Config; -use crate::metrics::agent_balance_sum; -use crate::utils::get_matching_lines; -use maplit::hashmap; -use relayer::GAS_EXPENDITURE_LOG_MESSAGE; - -use crate::logging::log; -use crate::solana::solana_termination_invariants_met; -use crate::{fetch_metric, AGENT_LOGGING_DIR, ZERO_MERKLE_INSERTION_KATHY_MESSAGES}; - -// This number should be even, so the messages can be split into two equal halves -// sent before and after the relayer spins up, to avoid rounding errors. -pub const SOL_MESSAGES_EXPECTED: u32 = 20; - -/// Use the metrics to check if the relayer queues are empty and the expected -/// number of messages have been sent. -pub fn termination_invariants_met( - config: &Config, - starting_relayer_balance: f64, - solana_cli_tools_path: Option<&Path>, - solana_config_path: Option<&Path>, -) -> eyre::Result { - let eth_messages_expected = (config.kathy_messages / 2) as u32 * 2; - let sol_messages_expected = if config.sealevel_enabled { - SOL_MESSAGES_EXPECTED - } else { - 0 - }; - let total_messages_expected = eth_messages_expected + sol_messages_expected; - - let lengths = fetch_metric("9092", "hyperlane_submitter_queue_length", &hashmap! {})?; - assert!(!lengths.is_empty(), "Could not find queue length metric"); - if lengths.iter().sum::() != ZERO_MERKLE_INSERTION_KATHY_MESSAGES { - log!("Relayer queues not empty. Lengths: {:?}", lengths); - return Ok(false); - }; - - // Also ensure the counter is as expected (total number of messages), summed - // across all mailboxes. - let msg_processed_count = - fetch_metric("9092", "hyperlane_messages_processed_count", &hashmap! {})? - .iter() - .sum::(); - if msg_processed_count != total_messages_expected { - log!( - "Relayer has {} processed messages, expected {}", - msg_processed_count, - total_messages_expected - ); - return Ok(false); - } - - let gas_payment_events_count = fetch_metric( - "9092", - "hyperlane_contract_sync_stored_events", - &hashmap! {"data_type" => "gas_payments"}, - )? - .iter() - .sum::(); - - let log_file_path = AGENT_LOGGING_DIR.join("RLY-output.log"); - const STORING_NEW_MESSAGE_LOG_MESSAGE: &str = "Storing new message in db"; - const LOOKING_FOR_EVENTS_LOG_MESSAGE: &str = "Looking for events in index range"; - const HYPER_INCOMING_BODY_LOG_MESSAGE: &str = "incoming body completed"; - - const TX_ID_INDEXING_LOG_MESSAGE: &str = "Found log(s) for tx id"; - - let relayer_logfile = File::open(log_file_path)?; - let invariant_logs = &[ - STORING_NEW_MESSAGE_LOG_MESSAGE, - LOOKING_FOR_EVENTS_LOG_MESSAGE, - GAS_EXPENDITURE_LOG_MESSAGE, - HYPER_INCOMING_BODY_LOG_MESSAGE, - TX_ID_INDEXING_LOG_MESSAGE, - ]; - let log_counts = get_matching_lines(&relayer_logfile, invariant_logs); - // Zero insertion messages don't reach `submit` stage where gas is spent, so we only expect these logs for the other messages. - // TODO: Sometimes we find more logs than expected. This may either mean that gas is deducted twice for the same message due to a bug, - // or that submitting the message transaction fails for some messages. Figure out which is the case and convert this check to - // strict equality. - // EDIT: Having had a quick look, it seems like there are some legitimate reverts happening in the confirm step - // (`Transaction attempting to process message either reverted or was reorged`) - // in which case more gas expenditure logs than messages are expected. - assert!( - log_counts.get(GAS_EXPENDITURE_LOG_MESSAGE).unwrap() >= &total_messages_expected, - "Didn't record gas payment for all delivered messages" - ); - // These tests check that we fixed https://github.com/hyperlane-xyz/hyperlane-monorepo/issues/3915, where some logs would not show up - assert!( - log_counts.get(STORING_NEW_MESSAGE_LOG_MESSAGE).unwrap() > &0, - "Didn't find any logs about storing messages in db" - ); - assert!( - log_counts.get(LOOKING_FOR_EVENTS_LOG_MESSAGE).unwrap() > &0, - "Didn't find any logs about looking for events in index range" - ); - let total_tx_id_log_count = log_counts.get(TX_ID_INDEXING_LOG_MESSAGE).unwrap(); - assert!( - // there are 3 txid-indexed events: - // - relayer: merkle insertion and gas payment - // - scraper: gas payment - // some logs are emitted for multiple events, so requiring there to be at least - // `config.kathy_messages` logs is a reasonable approximation, since all three of these events - // are expected to be logged for each message. - *total_tx_id_log_count as u64 >= config.kathy_messages, - "Didn't find as many tx id logs as expected. Found {} and expected {}", - total_tx_id_log_count, - config.kathy_messages - ); - assert!( - log_counts.get(HYPER_INCOMING_BODY_LOG_MESSAGE).is_none(), - "Verbose logs not expected at the log level set in e2e" - ); - - let gas_payment_sealevel_events_count = fetch_metric( - "9092", - "hyperlane_contract_sync_stored_events", - &hashmap! { - "data_type" => "gas_payments", - "chain" => "sealeveltest", - }, - )? - .iter() - .sum::(); - // TestSendReceiver randomly breaks gas payments up into - // two. So we expect at least as many gas payments as messages. - if gas_payment_events_count < total_messages_expected { - log!( - "Relayer has {} gas payment events, expected at least {}", - gas_payment_events_count, - total_messages_expected - ); - return Ok(false); - } - - if let Some((solana_cli_tools_path, solana_config_path)) = - solana_cli_tools_path.zip(solana_config_path) - { - if !solana_termination_invariants_met(solana_cli_tools_path, solana_config_path) { - log!("Solana termination invariants not met"); - return Ok(false); - } - } - - let dispatched_messages_scraped = fetch_metric( - "9093", - "hyperlane_contract_sync_stored_events", - &hashmap! {"data_type" => "message_dispatch"}, - )? - .iter() - .sum::(); - if dispatched_messages_scraped != eth_messages_expected + ZERO_MERKLE_INSERTION_KATHY_MESSAGES { - log!( - "Scraper has scraped {} dispatched messages, expected {}", - dispatched_messages_scraped, - eth_messages_expected - ); - return Ok(false); - } - - let gas_payments_scraped = fetch_metric( - "9093", - "hyperlane_contract_sync_stored_events", - &hashmap! {"data_type" => "gas_payment"}, - )? - .iter() - .sum::(); - // The relayer and scraper should have the same number of gas payments. - // TODO: Sealevel gas payments are not yet included in the event count. - // For now, treat as an exception in the invariants. - let expected_gas_payments = gas_payment_events_count - gas_payment_sealevel_events_count; - if gas_payments_scraped != expected_gas_payments { - log!( - "Scraper has scraped {} gas payments, expected {}", - gas_payments_scraped, - expected_gas_payments - ); - return Ok(false); - } - - let delivered_messages_scraped = fetch_metric( - "9093", - "hyperlane_contract_sync_stored_events", - &hashmap! {"data_type" => "message_delivery"}, - )? - .iter() - .sum::(); - if delivered_messages_scraped != eth_messages_expected { - log!( - "Scraper has scraped {} delivered messages, expected {}", - delivered_messages_scraped, - eth_messages_expected - ); - return Ok(false); - } - - let ending_relayer_balance: f64 = agent_balance_sum(9092).unwrap(); - // Make sure the balance was correctly updated in the metrics. - if starting_relayer_balance <= ending_relayer_balance { - log!( - "Expected starting relayer balance to be greater than ending relayer balance, but got {} <= {}", - starting_relayer_balance, - ending_relayer_balance - ); - return Ok(false); - } - - log!("Termination invariants have been meet"); - Ok(true) -} +mod common; +mod post_startup_invariants; +mod termination_invariants; diff --git a/rust/utils/run-locally/src/invariants/common.rs b/rust/utils/run-locally/src/invariants/common.rs new file mode 100644 index 0000000000..35a0c5eae4 --- /dev/null +++ b/rust/utils/run-locally/src/invariants/common.rs @@ -0,0 +1,3 @@ +// This number should be even, so the messages can be split into two equal halves +// sent before and after the relayer spins up, to avoid rounding errors. +pub const SOL_MESSAGES_EXPECTED: u32 = 20; diff --git a/rust/utils/run-locally/src/invariants/post_startup_invariants.rs b/rust/utils/run-locally/src/invariants/post_startup_invariants.rs new file mode 100644 index 0000000000..2d7053f301 --- /dev/null +++ b/rust/utils/run-locally/src/invariants/post_startup_invariants.rs @@ -0,0 +1,54 @@ +use std::fs::File; +use std::io::BufReader; + +use hyperlane_base::AgentMetadata; + +use crate::DynPath; + +pub fn post_startup_invariants(checkpoints_dirs: &[DynPath]) -> bool { + post_startup_validator_metadata_written(checkpoints_dirs) +} + +fn post_startup_validator_metadata_written(checkpoints_dirs: &[DynPath]) -> bool { + let expected_git_sha = env!("VERGEN_GIT_SHA"); + + let failed_metadata = checkpoints_dirs + .iter() + .map(|path| metadata_file_check(expected_git_sha, path)) + .any(|b| !b); + + !failed_metadata +} + +fn metadata_file_check(expected_git_sha: &str, path: &DynPath) -> bool { + let path = (*path).as_ref().as_ref(); + if !path.exists() { + return false; + } + + let file = path.join("metadata_latest.json"); + if !file.exists() { + return false; + } + + let open = File::open(&file); + let mut reader = if let Ok(file) = open { + BufReader::new(file) + } else { + return false; + }; + + let deserialized = serde_json::from_reader(&mut reader); + + let metadata: AgentMetadata = if let Ok(metadata) = deserialized { + metadata + } else { + return false; + }; + + if metadata.git_sha != expected_git_sha { + return false; + } + + true +} diff --git a/rust/utils/run-locally/src/invariants/termination_invariants.rs b/rust/utils/run-locally/src/invariants/termination_invariants.rs new file mode 100644 index 0000000000..a1c3e63168 --- /dev/null +++ b/rust/utils/run-locally/src/invariants/termination_invariants.rs @@ -0,0 +1,210 @@ +use std::fs::File; +use std::path::Path; + +use crate::config::Config; +use crate::metrics::agent_balance_sum; +use crate::utils::get_matching_lines; +use maplit::hashmap; +use relayer::GAS_EXPENDITURE_LOG_MESSAGE; + +use crate::invariants::SOL_MESSAGES_EXPECTED; +use crate::logging::log; +use crate::solana::solana_termination_invariants_met; +use crate::{fetch_metric, AGENT_LOGGING_DIR, ZERO_MERKLE_INSERTION_KATHY_MESSAGES}; + +/// Use the metrics to check if the relayer queues are empty and the expected +/// number of messages have been sent. +pub fn termination_invariants_met( + config: &Config, + starting_relayer_balance: f64, + solana_cli_tools_path: Option<&Path>, + solana_config_path: Option<&Path>, +) -> eyre::Result { + let eth_messages_expected = (config.kathy_messages / 2) as u32 * 2; + let sol_messages_expected = if config.sealevel_enabled { + SOL_MESSAGES_EXPECTED + } else { + 0 + }; + let total_messages_expected = eth_messages_expected + sol_messages_expected; + + let lengths = fetch_metric("9092", "hyperlane_submitter_queue_length", &hashmap! {})?; + assert!(!lengths.is_empty(), "Could not find queue length metric"); + if lengths.iter().sum::() != ZERO_MERKLE_INSERTION_KATHY_MESSAGES { + log!("Relayer queues not empty. Lengths: {:?}", lengths); + return Ok(false); + }; + + // Also ensure the counter is as expected (total number of messages), summed + // across all mailboxes. + let msg_processed_count = + fetch_metric("9092", "hyperlane_messages_processed_count", &hashmap! {})? + .iter() + .sum::(); + if msg_processed_count != total_messages_expected { + log!( + "Relayer has {} processed messages, expected {}", + msg_processed_count, + total_messages_expected + ); + return Ok(false); + } + + let gas_payment_events_count = fetch_metric( + "9092", + "hyperlane_contract_sync_stored_events", + &hashmap! {"data_type" => "gas_payments"}, + )? + .iter() + .sum::(); + + let log_file_path = AGENT_LOGGING_DIR.join("RLY-output.log"); + const STORING_NEW_MESSAGE_LOG_MESSAGE: &str = "Storing new message in db"; + const LOOKING_FOR_EVENTS_LOG_MESSAGE: &str = "Looking for events in index range"; + const HYPER_INCOMING_BODY_LOG_MESSAGE: &str = "incoming body completed"; + + const TX_ID_INDEXING_LOG_MESSAGE: &str = "Found log(s) for tx id"; + + let relayer_logfile = File::open(log_file_path)?; + let invariant_logs = &[ + STORING_NEW_MESSAGE_LOG_MESSAGE, + LOOKING_FOR_EVENTS_LOG_MESSAGE, + GAS_EXPENDITURE_LOG_MESSAGE, + HYPER_INCOMING_BODY_LOG_MESSAGE, + TX_ID_INDEXING_LOG_MESSAGE, + ]; + let log_counts = get_matching_lines(&relayer_logfile, invariant_logs); + // Zero insertion messages don't reach `submit` stage where gas is spent, so we only expect these logs for the other messages. + // TODO: Sometimes we find more logs than expected. This may either mean that gas is deducted twice for the same message due to a bug, + // or that submitting the message transaction fails for some messages. Figure out which is the case and convert this check to + // strict equality. + // EDIT: Having had a quick look, it seems like there are some legitimate reverts happening in the confirm step + // (`Transaction attempting to process message either reverted or was reorged`) + // in which case more gas expenditure logs than messages are expected. + assert!( + log_counts.get(GAS_EXPENDITURE_LOG_MESSAGE).unwrap() >= &total_messages_expected, + "Didn't record gas payment for all delivered messages" + ); + // These tests check that we fixed https://github.com/hyperlane-xyz/hyperlane-monorepo/issues/3915, where some logs would not show up + assert!( + log_counts.get(STORING_NEW_MESSAGE_LOG_MESSAGE).unwrap() > &0, + "Didn't find any logs about storing messages in db" + ); + assert!( + log_counts.get(LOOKING_FOR_EVENTS_LOG_MESSAGE).unwrap() > &0, + "Didn't find any logs about looking for events in index range" + ); + let total_tx_id_log_count = log_counts.get(TX_ID_INDEXING_LOG_MESSAGE).unwrap(); + assert!( + // there are 3 txid-indexed events: + // - relayer: merkle insertion and gas payment + // - scraper: gas payment + // some logs are emitted for multiple events, so requiring there to be at least + // `config.kathy_messages` logs is a reasonable approximation, since all three of these events + // are expected to be logged for each message. + *total_tx_id_log_count as u64 >= config.kathy_messages, + "Didn't find as many tx id logs as expected. Found {} and expected {}", + total_tx_id_log_count, + config.kathy_messages + ); + assert!( + log_counts.get(HYPER_INCOMING_BODY_LOG_MESSAGE).is_none(), + "Verbose logs not expected at the log level set in e2e" + ); + + let gas_payment_sealevel_events_count = fetch_metric( + "9092", + "hyperlane_contract_sync_stored_events", + &hashmap! { + "data_type" => "gas_payments", + "chain" => "sealeveltest", + }, + )? + .iter() + .sum::(); + // TestSendReceiver randomly breaks gas payments up into + // two. So we expect at least as many gas payments as messages. + if gas_payment_events_count < total_messages_expected { + log!( + "Relayer has {} gas payment events, expected at least {}", + gas_payment_events_count, + total_messages_expected + ); + return Ok(false); + } + + if let Some((solana_cli_tools_path, solana_config_path)) = + solana_cli_tools_path.zip(solana_config_path) + { + if !solana_termination_invariants_met(solana_cli_tools_path, solana_config_path) { + log!("Solana termination invariants not met"); + return Ok(false); + } + } + + let dispatched_messages_scraped = fetch_metric( + "9093", + "hyperlane_contract_sync_stored_events", + &hashmap! {"data_type" => "message_dispatch"}, + )? + .iter() + .sum::(); + if dispatched_messages_scraped != eth_messages_expected + ZERO_MERKLE_INSERTION_KATHY_MESSAGES { + log!( + "Scraper has scraped {} dispatched messages, expected {}", + dispatched_messages_scraped, + eth_messages_expected + ); + return Ok(false); + } + + let gas_payments_scraped = fetch_metric( + "9093", + "hyperlane_contract_sync_stored_events", + &hashmap! {"data_type" => "gas_payment"}, + )? + .iter() + .sum::(); + // The relayer and scraper should have the same number of gas payments. + // TODO: Sealevel gas payments are not yet included in the event count. + // For now, treat as an exception in the invariants. + let expected_gas_payments = gas_payment_events_count - gas_payment_sealevel_events_count; + if gas_payments_scraped != expected_gas_payments { + log!( + "Scraper has scraped {} gas payments, expected {}", + gas_payments_scraped, + expected_gas_payments + ); + return Ok(false); + } + + let delivered_messages_scraped = fetch_metric( + "9093", + "hyperlane_contract_sync_stored_events", + &hashmap! {"data_type" => "message_delivery"}, + )? + .iter() + .sum::(); + if delivered_messages_scraped != eth_messages_expected { + log!( + "Scraper has scraped {} delivered messages, expected {}", + delivered_messages_scraped, + eth_messages_expected + ); + return Ok(false); + } + + let ending_relayer_balance: f64 = agent_balance_sum(9092).unwrap(); + // Make sure the balance was correctly updated in the metrics. + if starting_relayer_balance <= ending_relayer_balance { + log!( + "Expected starting relayer balance to be greater than ending relayer balance, but got {} <= {}", + starting_relayer_balance, + ending_relayer_balance + ); + return Ok(false); + } + + log!("Termination invariants have been meet"); + Ok(true) +} diff --git a/rust/utils/run-locally/src/main.rs b/rust/utils/run-locally/src/main.rs index e5498b5f6e..7cfbce96c3 100644 --- a/rust/utils/run-locally/src/main.rs +++ b/rust/utils/run-locally/src/main.rs @@ -36,7 +36,7 @@ use tempfile::tempdir; use crate::{ config::Config, ethereum::start_anvil, - invariants::{termination_invariants_met, SOL_MESSAGES_EXPECTED}, + invariants::{post_startup_invariants, termination_invariants_met, SOL_MESSAGES_EXPECTED}, metrics::agent_balance_sum, solana::*, utils::{concat_path, make_static, stop_child, AgentHandles, ArbitraryData, TaskHandle}, @@ -454,6 +454,14 @@ fn main() -> ExitCode { let loop_start = Instant::now(); // give things a chance to fully start. sleep(Duration::from_secs(10)); + + if !post_startup_invariants(&checkpoints_dirs) { + log!("Failure: Post startup invariants are not met"); + return report_test_result(true); + } else { + log!("Success: Post startup invariants are met"); + } + let mut failure_occurred = false; let starting_relayer_balance: f64 = agent_balance_sum(9092).unwrap(); while !SHUTDOWN.load(Ordering::Relaxed) { @@ -499,6 +507,10 @@ fn main() -> ExitCode { sleep(Duration::from_secs(5)); } + report_test_result(failure_occurred) +} + +fn report_test_result(failure_occurred: bool) -> ExitCode { if failure_occurred { log!("E2E tests failed"); ExitCode::FAILURE