From 3527fa2a0ad4df56e330c699f4aa944d3007ff9c Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Wed, 25 Sep 2024 17:25:23 +0530 Subject: [PATCH 01/10] feat(`advanced`): foundry-fork-db --- Cargo.toml | 3 + examples/advanced/Cargo.toml | 4 +- examples/advanced/examples/foundry_fork_db.rs | 70 +++++++++++++++++++ 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 examples/advanced/examples/foundry_fork_db.rs diff --git a/Cargo.toml b/Cargo.toml index 3e2e47dd..dcf6b60e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -109,6 +109,9 @@ alloy = { version = "0.3.6", features = [ "signer-yubihsm", ] } +foundry-fork-db = "0.3.1" +revm-primitives = "9.0.2" + # async futures-util = "0.3" tokio = "1.40" diff --git a/examples/advanced/Cargo.toml b/examples/advanced/Cargo.toml index d0b2488e..ef82927c 100644 --- a/examples/advanced/Cargo.toml +++ b/examples/advanced/Cargo.toml @@ -14,8 +14,10 @@ workspace = true [dev-dependencies] alloy.workspace = true +foundry-fork-db.workspace = true +revm-primitives.workspace = true eyre.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } serde = { workspace = true, features = ["derive"] } -serde_json = { workspace = true } \ No newline at end of file +serde_json = { workspace = true } diff --git a/examples/advanced/examples/foundry_fork_db.rs b/examples/advanced/examples/foundry_fork_db.rs new file mode 100644 index 00000000..91d9cf25 --- /dev/null +++ b/examples/advanced/examples/foundry_fork_db.rs @@ -0,0 +1,70 @@ +//! Foundry Fork DB +use std::sync::Arc; + +use eyre::Result; + +use alloy::{ + eips::BlockId, + network::AnyNetwork, + node_bindings::Anvil, + primitives::U256, + providers::{Provider, ProviderBuilder}, + rpc::types::BlockTransactionsKind, +}; +use foundry_fork_db::{cache::BlockchainDbMeta, BlockchainDb, SharedBackend}; +use revm_primitives::{BlockEnv, CfgEnv, Env}; + +// TODO: Add docs and explanation of the workflow. +// TODO: Depict how the backend handler can smartly manages duplicate requests. + +#[tokio::main] +async fn main() -> Result<()> { + let anvil = Anvil::new().spawn(); + let provider = ProviderBuilder::new().network::().on_http(anvil.endpoint_url()); + + let cfg = CfgEnv::default().with_chain_id(31337); + + let block = + provider.get_block(BlockId::latest(), BlockTransactionsKind::Hashes).await?.unwrap(); + + let pin_block = BlockId::number(block.header.number); + let block = BlockEnv { + number: U256::from(block.header.number), + coinbase: block.header.miner, + timestamp: U256::from(block.header.timestamp), + gas_limit: U256::from(block.header.gas_limit), + blob_excess_gas_and_price: None, + difficulty: U256::from(block.header.difficulty), + prevrandao: None, + basefee: U256::from(block.header.base_fee_per_gas.unwrap()), + }; + + let env = Env { cfg, block, ..Default::default() }; + + let meta = BlockchainDbMeta::new(env, anvil.endpoint()); + + let db = BlockchainDb::new(meta, None); + + let shared = SharedBackend::spawn_backend(Arc::new(provider), db, Some(pin_block)).await; + + let start_t = std::time::Instant::now(); + let block_rpc = shared.get_full_block(0).unwrap(); + let time_rpc = start_t.elapsed(); + + // `SharedBackend` are clonable and have the same underlying cache. + let cloned_backend = shared.clone(); + + // Block gets cached in the db + let start_t = std::time::Instant::now(); + let block_cache = cloned_backend.get_full_block(0).unwrap(); + let time_cache = start_t.elapsed(); + + assert_eq!(block_rpc, block_cache); + + assert!(time_cache < time_rpc); + + println!("Time taken for 1st request (via RPC): {:?}", time_rpc); + println!("Time taken for 2nd requst (via Cache): {:?}", time_cache); + + Ok(()) +} From 70e3e5970008cb7bdd6f448c74fd824d796f1df1 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:30:23 +0530 Subject: [PATCH 02/10] patch fork-db --- Cargo.toml | 3 +++ examples/advanced/examples/foundry_fork_db.rs | 21 +++++-------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index dcf6b60e..4f521558 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -120,3 +120,6 @@ tokio = "1.40" eyre = "0.6" serde = "1.0" serde_json = "1.0" + +[patch.crates-io] +foundry-fork-db = { git = "https://github.com/foundry-rs/foundry-fork-db", rev = "4c205d3" } diff --git a/examples/advanced/examples/foundry_fork_db.rs b/examples/advanced/examples/foundry_fork_db.rs index 91d9cf25..473371ee 100644 --- a/examples/advanced/examples/foundry_fork_db.rs +++ b/examples/advanced/examples/foundry_fork_db.rs @@ -22,26 +22,15 @@ async fn main() -> Result<()> { let anvil = Anvil::new().spawn(); let provider = ProviderBuilder::new().network::().on_http(anvil.endpoint_url()); - let cfg = CfgEnv::default().with_chain_id(31337); - let block = provider.get_block(BlockId::latest(), BlockTransactionsKind::Hashes).await?.unwrap(); let pin_block = BlockId::number(block.header.number); - let block = BlockEnv { - number: U256::from(block.header.number), - coinbase: block.header.miner, - timestamp: U256::from(block.header.timestamp), - gas_limit: U256::from(block.header.gas_limit), - blob_excess_gas_and_price: None, - difficulty: U256::from(block.header.difficulty), - prevrandao: None, - basefee: U256::from(block.header.base_fee_per_gas.unwrap()), - }; - - let env = Env { cfg, block, ..Default::default() }; - - let meta = BlockchainDbMeta::new(env, anvil.endpoint()); + + let meta = BlockchainDbMeta::default() + .with_chain_id(31337) + .with_block(&block.inner) + .with_url(&anvil.endpoint()); let db = BlockchainDb::new(meta, None); From 04a8387232e267cec2ba17b48aefe9369cd5af8b Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Thu, 26 Sep 2024 19:59:21 +0530 Subject: [PATCH 03/10] bump foundry-fork-db --- Cargo.toml | 4 ++-- examples/advanced/examples/foundry_fork_db.rs | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4f521558..9a581f47 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -110,7 +110,7 @@ alloy = { version = "0.3.6", features = [ ] } foundry-fork-db = "0.3.1" -revm-primitives = "9.0.2" +revm-primitives = "10.0.0" # async futures-util = "0.3" @@ -122,4 +122,4 @@ serde = "1.0" serde_json = "1.0" [patch.crates-io] -foundry-fork-db = { git = "https://github.com/foundry-rs/foundry-fork-db", rev = "4c205d3" } +foundry-fork-db = { git = "https://github.com/foundry-rs/foundry-fork-db", rev = "7ac8ca4" } diff --git a/examples/advanced/examples/foundry_fork_db.rs b/examples/advanced/examples/foundry_fork_db.rs index 473371ee..879f4947 100644 --- a/examples/advanced/examples/foundry_fork_db.rs +++ b/examples/advanced/examples/foundry_fork_db.rs @@ -7,12 +7,10 @@ use alloy::{ eips::BlockId, network::AnyNetwork, node_bindings::Anvil, - primitives::U256, providers::{Provider, ProviderBuilder}, rpc::types::BlockTransactionsKind, }; use foundry_fork_db::{cache::BlockchainDbMeta, BlockchainDb, SharedBackend}; -use revm_primitives::{BlockEnv, CfgEnv, Env}; // TODO: Add docs and explanation of the workflow. // TODO: Depict how the backend handler can smartly manages duplicate requests. From 33dd38eaac58a3119e3d4532b10d78f0b9ef0a96 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:51:56 +0530 Subject: [PATCH 04/10] transact with revm --- Cargo.toml | 1 + examples/advanced/Cargo.toml | 1 + examples/advanced/examples/foundry_fork_db.rs | 103 ++++++++++++++++-- 3 files changed, 96 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9a581f47..2a4a7f72 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,6 +111,7 @@ alloy = { version = "0.3.6", features = [ foundry-fork-db = "0.3.1" revm-primitives = "10.0.0" +revm = "14.0.3" # async futures-util = "0.3" diff --git a/examples/advanced/Cargo.toml b/examples/advanced/Cargo.toml index ef82927c..233d12d8 100644 --- a/examples/advanced/Cargo.toml +++ b/examples/advanced/Cargo.toml @@ -16,6 +16,7 @@ workspace = true alloy.workspace = true foundry-fork-db.workspace = true revm-primitives.workspace = true +revm.workspace = true eyre.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/examples/advanced/examples/foundry_fork_db.rs b/examples/advanced/examples/foundry_fork_db.rs index 879f4947..f7df575f 100644 --- a/examples/advanced/examples/foundry_fork_db.rs +++ b/examples/advanced/examples/foundry_fork_db.rs @@ -5,15 +5,18 @@ use eyre::Result; use alloy::{ eips::BlockId, - network::AnyNetwork, + network::{AnyNetwork, TransactionBuilder}, node_bindings::Anvil, + primitives::U256, providers::{Provider, ProviderBuilder}, - rpc::types::BlockTransactionsKind, + rpc::types::{ + serde_helpers::WithOtherFields, Block, BlockTransactionsKind, Transaction, + TransactionRequest, + }, }; use foundry_fork_db::{cache::BlockchainDbMeta, BlockchainDb, SharedBackend}; - -// TODO: Add docs and explanation of the workflow. -// TODO: Depict how the backend handler can smartly manages duplicate requests. +use revm::{db::CacheDB, DatabaseRef, Evm}; +use revm_primitives::{BlobExcessGasAndPrice, BlockEnv, TxEnv}; #[tokio::main] async fn main() -> Result<()> { @@ -32,13 +35,14 @@ async fn main() -> Result<()> { let db = BlockchainDb::new(meta, None); - let shared = SharedBackend::spawn_backend(Arc::new(provider), db, Some(pin_block)).await; + let shared = + SharedBackend::spawn_backend(Arc::new(provider.clone()), db, Some(pin_block)).await; let start_t = std::time::Instant::now(); let block_rpc = shared.get_full_block(0).unwrap(); let time_rpc = start_t.elapsed(); - // `SharedBackend` are clonable and have the same underlying cache. + // `SharedBackend` is cloneable and holds the channel to the same `BackendHandler`. let cloned_backend = shared.clone(); // Block gets cached in the db @@ -50,8 +54,89 @@ async fn main() -> Result<()> { assert!(time_cache < time_rpc); - println!("Time taken for 1st request (via RPC): {:?}", time_rpc); - println!("Time taken for 2nd requst (via Cache): {:?}", time_cache); + println!("-------get_full_block--------"); + // The backend handle falls back to the RPC provider if the block is not in the cache. + println!("1st request (via rpc): {:?}", time_rpc); + // The block is cached due to the previous request and can be fetched from db. + println!("2nd request (via fork db): {:?}\n", time_cache); + + let alice = anvil.addresses()[0]; + let bob = anvil.addresses()[1]; + + let basefee = block.header.base_fee_per_gas.unwrap(); + + let tx_req = TransactionRequest::default() + .with_from(alice) + .with_to(bob) + .with_value(U256::from(100)) + .with_max_fee_per_gas(basefee) + .with_max_priority_fee_per_gas(basefee + 1) + .with_gas_limit(21000) + .with_nonce(0); + + let mut evm = configure_evm_env(block, shared.clone(), configure_tx_env(tx_req)); + + // Fetches accounts from the RPC + let start_t = std::time::Instant::now(); + let alice_bal = shared.basic_ref(alice)?.unwrap().balance; + let bob_bal = shared.basic_ref(bob)?.unwrap().balance; + let time_rpc = start_t.elapsed(); + + let res = evm.transact().unwrap(); + + let total_spent = U256::from(res.result.gas_used()) * U256::from(basefee) + U256::from(100); + + shared.data().do_commit(res.state); + + // Fetches accounts from the cache + let start_t = std::time::Instant::now(); + let alice_bal_after = shared.basic_ref(alice)?.unwrap().balance; + let bob_bal_after = shared.basic_ref(bob)?.unwrap().balance; + let time_cache = start_t.elapsed(); + + println!("-------get_account--------"); + println!("1st request (via rpc): {:?}", time_rpc); + println!("2nd request (via fork db): {:?}\n", time_cache); + + assert_eq!(alice_bal_after, alice_bal - total_spent); + assert_eq!(bob_bal_after, bob_bal + U256::from(100)); Ok(()) } + +fn configure_evm_env( + block: WithOtherFields>>, + shared: SharedBackend, + tx_env: TxEnv, +) -> Evm<'static, (), CacheDB> { + let basefee = block.header.base_fee_per_gas.map(U256::from).unwrap_or_default(); + let block_env = BlockEnv { + number: U256::from(block.header.number), + coinbase: block.header.miner, + timestamp: U256::from(block.header.timestamp), + gas_limit: U256::from(block.header.gas_limit), + basefee, + prevrandao: block.header.mix_hash, + difficulty: block.header.difficulty, + blob_excess_gas_and_price: Some(BlobExcessGasAndPrice::new( + block.header.excess_blob_gas.map(|g| g as u64).unwrap_or_default(), + )), + }; + + let db = CacheDB::new(shared); + + let evm = Evm::builder().with_block_env(block_env).with_db(db).with_tx_env(tx_env).build(); + + evm +} + +fn configure_tx_env(tx_req: TransactionRequest) -> TxEnv { + TxEnv { + caller: tx_req.from.unwrap(), + transact_to: tx_req.to.unwrap(), + value: tx_req.value.unwrap(), + gas_price: U256::from(tx_req.max_fee_per_gas.unwrap()), + gas_limit: tx_req.gas.map(|g| g as u64).unwrap_or_default(), + ..Default::default() + } +} From 9b10261b1ada583817a8df0966396eec64698dd9 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 27 Sep 2024 13:11:22 +0530 Subject: [PATCH 05/10] docs --- examples/advanced/examples/foundry_fork_db.rs | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/examples/advanced/examples/foundry_fork_db.rs b/examples/advanced/examples/foundry_fork_db.rs index f7df575f..ab58823d 100644 --- a/examples/advanced/examples/foundry_fork_db.rs +++ b/examples/advanced/examples/foundry_fork_db.rs @@ -1,4 +1,10 @@ -//! Foundry Fork DB +//! This example demonstrates how to use `foundry_fork_db` to build a minimal fork with a db that +//! caches responses from the RPC provider. +//! +//! `foundry_fork_db` is designed out-of-the-box to smartly cache and deduplicate requests to the +//! rpc provider, while fetching data that is missing from it's db instance. +//! +//! `foundry_fork_db` serves as the backend for foundry's forking functionality in anvil and forge. use std::sync::Arc; use eyre::Result; @@ -26,8 +32,8 @@ async fn main() -> Result<()> { let block = provider.get_block(BlockId::latest(), BlockTransactionsKind::Hashes).await?.unwrap(); - let pin_block = BlockId::number(block.header.number); - + // The `BlockchainDbMeta` is used a identifier when the db is flushed to the disk. + // This aids in cases where the disk contains data from multiple forks. let meta = BlockchainDbMeta::default() .with_chain_id(31337) .with_block(&block.inner) @@ -35,8 +41,19 @@ async fn main() -> Result<()> { let db = BlockchainDb::new(meta, None); - let shared = - SharedBackend::spawn_backend(Arc::new(provider.clone()), db, Some(pin_block)).await; + // Spawn the backend with the db instance. + // `SharedBackend` is used to send request to the `BackendHandler` which is responsible for + // filling missing data in the db, and also deduplicate requests that are being sent to the + // RPC provider. + // + // For example, if we send two requests to get_full_block(0) simultaneously, the + // `BackendHandler` is smart enough to only send one request to the RPC provider, and queue the + // other request until the response is received. + // Once the response from RPC provider is received it relays the response to both the requests + // over their respective channels. + // + // The `SharedBackend` and `BackendHandler` communicate over an unbounded channel. + let shared = SharedBackend::spawn_backend(Arc::new(provider.clone()), db, None).await; let start_t = std::time::Instant::now(); let block_rpc = shared.get_full_block(0).unwrap(); @@ -52,8 +69,6 @@ async fn main() -> Result<()> { assert_eq!(block_rpc, block_cache); - assert!(time_cache < time_rpc); - println!("-------get_full_block--------"); // The backend handle falls back to the RPC provider if the block is not in the cache. println!("1st request (via rpc): {:?}", time_rpc); From e696cb81a747485d4043169133780ce62cd9ee8a Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 27 Sep 2024 19:37:22 +0530 Subject: [PATCH 06/10] allow clippy needless return --- clippy.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/clippy.toml b/clippy.toml index 472818ef..b35f94ae 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1 +1,2 @@ msrv = "1.76" +allow = ["clippy::needless_return"] From 4781cbe5da5a61f9804be28f18cc35141eff9224 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 27 Sep 2024 19:43:03 +0530 Subject: [PATCH 07/10] fix --- Cargo.toml | 1 + clippy.toml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 2a4a7f72..c0bf14b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,6 +92,7 @@ non_send_fields_in_send_ty = "allow" redundant_pub_crate = "allow" significant_drop_in_scrutinee = "allow" significant_drop_tightening = "allow" +needless_return = "allow" [workspace.dependencies] alloy = { version = "0.3.6", features = [ diff --git a/clippy.toml b/clippy.toml index b35f94ae..472818ef 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1,2 +1 @@ msrv = "1.76" -allow = ["clippy::needless_return"] From 48a84fa5d3d307c88bcfb72d5f8504c5126fb644 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 27 Sep 2024 19:44:42 +0530 Subject: [PATCH 08/10] clippy --- examples/advanced/examples/foundry_fork_db.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/advanced/examples/foundry_fork_db.rs b/examples/advanced/examples/foundry_fork_db.rs index ab58823d..f6a76f1f 100644 --- a/examples/advanced/examples/foundry_fork_db.rs +++ b/examples/advanced/examples/foundry_fork_db.rs @@ -60,6 +60,7 @@ async fn main() -> Result<()> { let time_rpc = start_t.elapsed(); // `SharedBackend` is cloneable and holds the channel to the same `BackendHandler`. + #[allow(clippy::redundant_clone)] let cloned_backend = shared.clone(); // Block gets cached in the db From 427ed1ac43958414862520c7aa7aa70d5466596d Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Mon, 30 Sep 2024 18:54:26 +0530 Subject: [PATCH 09/10] bump fork-db --- Cargo.toml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c0bf14b4..f317747a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -110,7 +110,7 @@ alloy = { version = "0.3.6", features = [ "signer-yubihsm", ] } -foundry-fork-db = "0.3.1" +foundry-fork-db = "0.4.0" revm-primitives = "10.0.0" revm = "14.0.3" @@ -122,6 +122,3 @@ tokio = "1.40" eyre = "0.6" serde = "1.0" serde_json = "1.0" - -[patch.crates-io] -foundry-fork-db = { git = "https://github.com/foundry-rs/foundry-fork-db", rev = "7ac8ca4" } From 358393ca7d8075f1d4c61385bcd315e2c5eb5aa8 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 1 Oct 2024 18:46:02 +0530 Subject: [PATCH 10/10] bump alloy --- Cargo.toml | 2 +- examples/advanced/examples/foundry_fork_db.rs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f317747a..ea3a79f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -95,7 +95,7 @@ significant_drop_tightening = "allow" needless_return = "allow" [workspace.dependencies] -alloy = { version = "0.3.6", features = [ +alloy = { version = "0.4.0", features = [ "full", "node-bindings", "rpc-types-debug", diff --git a/examples/advanced/examples/foundry_fork_db.rs b/examples/advanced/examples/foundry_fork_db.rs index f6a76f1f..6d775db5 100644 --- a/examples/advanced/examples/foundry_fork_db.rs +++ b/examples/advanced/examples/foundry_fork_db.rs @@ -85,8 +85,8 @@ async fn main() -> Result<()> { .with_from(alice) .with_to(bob) .with_value(U256::from(100)) - .with_max_fee_per_gas(basefee) - .with_max_priority_fee_per_gas(basefee + 1) + .with_max_fee_per_gas(basefee as u128) + .with_max_priority_fee_per_gas(basefee as u128 + 1) .with_gas_limit(21000) .with_nonce(0); @@ -135,7 +135,7 @@ fn configure_evm_env( prevrandao: block.header.mix_hash, difficulty: block.header.difficulty, blob_excess_gas_and_price: Some(BlobExcessGasAndPrice::new( - block.header.excess_blob_gas.map(|g| g as u64).unwrap_or_default(), + block.header.excess_blob_gas.unwrap_or_default(), )), }; @@ -152,7 +152,7 @@ fn configure_tx_env(tx_req: TransactionRequest) -> TxEnv { transact_to: tx_req.to.unwrap(), value: tx_req.value.unwrap(), gas_price: U256::from(tx_req.max_fee_per_gas.unwrap()), - gas_limit: tx_req.gas.map(|g| g as u64).unwrap_or_default(), + gas_limit: tx_req.gas.unwrap_or_default(), ..Default::default() } }