From 39ac5f760f8f0fd187f224de60b15121188338cb Mon Sep 17 00:00:00 2001 From: ZanCorDX <126988525+ZanCorDX@users.noreply.github.com> Date: Thu, 2 Jan 2025 16:42:27 -0300 Subject: [PATCH] Removed sim_value from OrdersWithTimestamp (#334) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 📝 Summary Removed sim_value from OrdersWithTimestamp. ## 💡 Motivation and Context It wasn't being used and had lots of "None"s. ## ✅ I have completed the following steps: * [X] Run `make lint` * [X] Run `make test` * [ ] Added tests (if applicable) --- .../src/backtest/fetch/flashbots_db.rs | 130 ++++-------------- crates/rbuilder/src/backtest/fetch/mempool.rs | 1 - crates/rbuilder/src/backtest/mod.rs | 7 +- crates/rbuilder/src/backtest/store.rs | 22 +-- 4 files changed, 29 insertions(+), 131 deletions(-) diff --git a/crates/rbuilder/src/backtest/fetch/flashbots_db.rs b/crates/rbuilder/src/backtest/fetch/flashbots_db.rs index c1400d7cf..3a069842f 100644 --- a/crates/rbuilder/src/backtest/fetch/flashbots_db.rs +++ b/crates/rbuilder/src/backtest/fetch/flashbots_db.rs @@ -5,7 +5,7 @@ use crate::{ }, primitives::{ serialize::{RawBundle, RawOrder, RawShareBundle, TxEncoding}, - Order, OrderId, SimValue, + Order, OrderId, }, }; use alloy_primitives::{Bytes, B256, I256, U256, U64}; @@ -16,7 +16,7 @@ use bigdecimal::{ }; use eyre::WrapErr; use sqlx::postgres::PgPool; -use std::{collections::HashSet, ops::Mul, str::FromStr}; +use std::{collections::HashSet, str::FromStr}; use time::{OffsetDateTime, PrimitiveDateTime}; use tracing::trace; use uuid::Uuid; @@ -44,10 +44,10 @@ impl RelayDB { ) -> Result, eyre::Error> { let block = i64::try_from(block)?; - let bundles = sqlx::query_as::<_, (OffsetDateTime, String, i64, String, String, Option, Option, Option, Option, Option)>( + let bundles = sqlx::query_as::<_, (OffsetDateTime, String, i64, String, String, Option, Option, Option)>( "SELECT \ inserted_at, bundle_hash, param_block_number, param_signed_txs, param_reverting_tx_hashes, \ - coinbase_diff, total_gas_used, signing_address, replacement_uuid, param_timestamp \ + signing_address, replacement_uuid, param_timestamp \ FROM bundles \ WHERE is_simulated = true and param_block_number = $1\ ORDER BY inserted_at ASC", @@ -65,8 +65,6 @@ impl RelayDB { block, txs, reverting_tx_hashes, - coinbase_diff, - total_gas_used, signing_address, replacement_uuid, min_timestamp, @@ -115,17 +113,6 @@ impl RelayDB { replacement_nonce: replacement_uuid.and(Some(0)), }; - let sim_value = coinbase_diff.zip(total_gas_used).and_then(|(cb, gas)| { - let coinbase_profit = sql_eth_decimal_to_wei(cb)?; - let mev_gas_price = coinbase_profit / U256::try_from(gas).ok()?; - Some(SimValue { - coinbase_profit, - gas_used: gas.try_into().ok()?, - mev_gas_price, - ..Default::default() - }) - }); - let order = RawOrder::Bundle(raw_bundle) .decode(TxEncoding::NoBlobData) .wrap_err_with(|| format!("Failed to parse bundle {}", bundle_hash))?; @@ -134,7 +121,6 @@ impl RelayDB { timestamp_ms: (inserted_at.unix_timestamp_nanos() / 1_000_000) .try_into()?, order, - sim_value, }) }, ) @@ -162,60 +148,44 @@ impl RelayDB { let from_time = block_timestamp - time::Duration::seconds(26 * 12); let to_time = block_timestamp; - let simulated_bundles = sqlx::query_as::< - _, - ( - OffsetDateTime, - Vec, - sqlx::types::JsonValue, - Option, - Option, - ), - >( - "SELECT received_at, hash, body, sim_profit, sim_gas_used \ + let simulated_bundles = + sqlx::query_as::<_, (OffsetDateTime, Vec, sqlx::types::JsonValue)>( + "SELECT received_at, hash, body\ FROM sbundle \ WHERE sim_success = true and cancelled = false and inserted_at between $1 and $2 \ ORDER BY inserted_at ASC", - ) - .bind(from_time) - .bind(to_time) - .fetch_all(&self.pool) - .await?; + ) + .bind(from_time) + .bind(to_time) + .fetch_all(&self.pool) + .await?; // We pull sbundles that builder actually used for the given block because db overwrite may // change block range and timestamp where bundle can be applied after the fact and we miss them. - let used_bundles = sqlx::query_as::< - _, - ( - PrimitiveDateTime, - Vec, - sqlx::types::JsonValue, - Option, - Option, - ), - >( - "WITH used_sbundles AS ( + let used_bundles = + sqlx::query_as::<_, (PrimitiveDateTime, Vec, sqlx::types::JsonValue)>( + "WITH used_sbundles AS ( SELECT DISTINCT ON (sbu.hash) sbu.hash, bb.orders_closed_at as received_at FROM sbundle_builder_used sbu JOIN built_blocks bb ON sbu.block_id = bb.block_id WHERE bb.block_number = $1 ORDER BY sbu.hash, bb.orders_closed_at ASC - ) SELECT us.received_at, s.hash, s.body, s.sim_profit, s.sim_gas_used + ) SELECT us.received_at, s.hash, s.body FROM used_sbundles us JOIN sbundle s ON us.hash = s.hash", - ) - .bind(block as i64) - .fetch_all(&self.pool) - .await?; + ) + .bind(block as i64) + .fetch_all(&self.pool) + .await?; let bundles = simulated_bundles.into_iter().map(|v| (v, false)).chain( used_bundles .into_iter() - .map(|v| ((v.0.assume_utc(), v.1, v.2, v.3, v.4), true)), + .map(|v| ((v.0.assume_utc(), v.1, v.2), true)), ); let bundles = bundles .map( - |((received_at, hash, body, coinbase_diff, total_gas_used), used_sbundle)| - -> eyre::Result<(u64, RawShareBundle, Option, B256)> { + |((received_at, hash, body), used_sbundle)| + -> eyre::Result<(u64, RawShareBundle, B256)> { let hash = (hash.len() == 32).then(|| B256::from_slice(&hash)).ok_or_else(|| eyre::eyre!("Invalid hash length"))?; let mut bundle = serde_json::from_value::(body) .wrap_err_with(|| { @@ -228,22 +198,10 @@ impl RelayDB { bundle.inclusion.max_block = None; } - let sim_value = coinbase_diff.zip(total_gas_used).and_then(|(cb, gas)| { - let coinbase_profit = sql_eth_decimal_to_wei(cb)?; - let mev_gas_price = coinbase_profit / U256::try_from(gas).ok()?; - Some(SimValue { - coinbase_profit, - gas_used: gas.try_into().ok()?, - mev_gas_price, - ..Default::default() - }) - }); - Ok(( (received_at.unix_timestamp_nanos() / 1_000_000) .try_into()?, bundle, - sim_value, hash, )) }, @@ -253,7 +211,7 @@ impl RelayDB { let mut result = Vec::with_capacity(bundles.len()); let mut inserted_bundles: HashSet = HashSet::default(); - for (timestamp_ms, bundle, sim_value, hash) in bundles { + for (timestamp_ms, bundle, hash) in bundles { if inserted_bundles.contains(&hash) { continue; } @@ -277,7 +235,6 @@ impl RelayDB { result.push(OrdersWithTimestamp { timestamp_ms, order, - sim_value, }); inserted_bundles.insert(hash); } @@ -429,24 +386,6 @@ fn sql_wei_decimal_to_wei(val: sqlx::types::BigDecimal) -> Option { Some(cb) } -fn sql_eth_decimal_to_wei(val: sqlx::types::BigDecimal) -> Option { - let (bi, exp) = val.into_bigint_and_exponent(); - let (bi_sign, bi_bytes) = bi.to_bytes_be(); - let bi = BigInt::from_bytes_be(bi_sign, &bi_bytes); - let cb = BigDecimal::new(bi, exp); - - let eth = BigDecimal::new(1u64.into(), -18); - let cb = cb.mul(eth); - let cb = cb.to_bigint()?; - let (sign, bytes) = cb.to_bytes_be(); - if sign == Sign::Minus { - return None; - } - let cb = U256::try_from_be_slice(&bytes)?; - - Some(cb) -} - #[cfg(test)] mod tests { use super::*; @@ -506,25 +445,4 @@ mod tests { tracing::trace!("bundles: {:#?}", bundles[bundles.len() - 1]); } - - #[test] - fn test_sql_decimal_to_wei() { - set_test_debug_tracing_subscriber(); - - let val = sqlx::types::BigDecimal::from_str("0.000000000000000001").unwrap(); - let wei = sql_eth_decimal_to_wei(val); - assert_eq!(wei, Some(U256::from(1u64))); - - let val = sqlx::types::BigDecimal::from_str("0.100000000000000001").unwrap(); - let wei = sql_eth_decimal_to_wei(val); - assert_eq!(wei, Some(U256::from(100000000000000001u64))); - - let val = sqlx::types::BigDecimal::from_str("-1").unwrap(); - let wei = sql_eth_decimal_to_wei(val); - assert_eq!(wei, None); - - let val = sqlx::types::BigDecimal::from_str("0.00000000000000000000001").unwrap(); - let wei = sql_eth_decimal_to_wei(val); - assert_eq!(wei, Some(U256::from(0u64))); - } } diff --git a/crates/rbuilder/src/backtest/fetch/mempool.rs b/crates/rbuilder/src/backtest/fetch/mempool.rs index c75a30bb5..0854b380b 100644 --- a/crates/rbuilder/src/backtest/fetch/mempool.rs +++ b/crates/rbuilder/src/backtest/fetch/mempool.rs @@ -53,7 +53,6 @@ pub fn get_mempool_transactions( Some(OrdersWithTimestamp { timestamp_ms, order, - sim_value: None, }) }) .collect()) diff --git a/crates/rbuilder/src/backtest/mod.rs b/crates/rbuilder/src/backtest/mod.rs index 9f95064d1..7097da50f 100644 --- a/crates/rbuilder/src/backtest/mod.rs +++ b/crates/rbuilder/src/backtest/mod.rs @@ -16,7 +16,7 @@ use crate::{ mev_boost::BuilderBlockReceived, primitives::{ serialize::{RawOrder, RawOrderConvertError, TxEncoding}, - AccountNonce, Order, OrderId, OrderReplacementKey, SimValue, + AccountNonce, Order, OrderId, OrderReplacementKey, }, utils::offset_datetime_to_timestamp_ms, }; @@ -34,8 +34,6 @@ use time::OffsetDateTime; pub struct RawOrdersWithTimestamp { pub timestamp_ms: u64, pub order: RawOrder, - #[serde(skip_serializing_if = "Option::is_none")] - pub sim_value: Option, } impl From for RawOrdersWithTimestamp { @@ -43,7 +41,6 @@ impl From for RawOrdersWithTimestamp { Self { timestamp_ms: orders.timestamp_ms, order: orders.order.into(), - sim_value: orders.sim_value, } } } @@ -53,7 +50,6 @@ impl RawOrdersWithTimestamp { Ok(OrdersWithTimestamp { timestamp_ms: self.timestamp_ms, order: self.order.decode(encoding)?, - sim_value: self.sim_value, }) } } @@ -62,7 +58,6 @@ impl RawOrdersWithTimestamp { pub struct OrdersWithTimestamp { pub timestamp_ms: u64, pub order: Order, - pub sim_value: Option, } /// Historic data for a block. diff --git a/crates/rbuilder/src/backtest/store.rs b/crates/rbuilder/src/backtest/store.rs index ff153e19e..521e94d0c 100644 --- a/crates/rbuilder/src/backtest/store.rs +++ b/crates/rbuilder/src/backtest/store.rs @@ -28,7 +28,7 @@ use std::{ /// Version of the data/format on the DB. /// Since we don't have backwards compatibility every time this is increased we must re-create the DB (manually delete the sqlite) -const VERSION: i64 = 10; +const VERSION: i64 = 11; /// Storage of BlockData. /// It allows us to locally cache (using a SQLite DB) all the info we need for backtesting so we don't have to @@ -243,16 +243,12 @@ impl HistoricalDataStorage { let order_json = compress_data(&serde_json::to_vec(&raw_order)?); sqlx::query( r#" - INSERT INTO orders (block_number, timestamp_ms, order_type, coinbase_profit, gas_used, order_id, order_data) - VALUES (?, ?, ?, ?, ?, ?, ?) + INSERT INTO orders (block_number, timestamp_ms, order_type, order_id, order_data) + VALUES (?, ?, ?, ?, ?) "#, ).bind(block_data.block_number as i64) .bind(order.timestamp_ms as i64) .bind(order_type(&raw_order.order)) - .bind(order.sim_value.clone().map(|v| { - format_ether(v.coinbase_profit) - })) - .bind(order.sim_value.clone().map(|v| v.gas_used as i64)) .bind(order_id) .bind(order_json) .execute(conn.as_mut()) @@ -616,10 +612,7 @@ mod test { use crate::{ backtest::RawOrdersWithTimestamp, mev_boost::BuilderBlockReceived, - primitives::{ - serialize::{RawBundle, RawTx}, - SimValue, - }, + primitives::serialize::{RawBundle, RawTx}, }; use alloy_consensus::TxEnvelope; use alloy_primitives::{hex, Address, PrimitiveSignature, B256, U256, U64}; @@ -642,7 +635,6 @@ mod test { order: RawOrder::Tx(RawTx { tx: tx.clone().into(), }), - sim_value: None, } .decode(TxEncoding::WithBlobData) .unwrap(), @@ -660,12 +652,6 @@ mod test { max_timestamp: Some(100), replacement_nonce: Some(0), }), - sim_value: Some(SimValue { - coinbase_profit: U256::from(42u64), - gas_used: 21000, - mev_gas_price: U256::from(44u64), - ..Default::default() - }), } .decode(TxEncoding::WithBlobData) .unwrap(),