From ce9154d9bdf63f18c7984886708297b1c18a7810 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Fri, 31 May 2024 12:46:46 -0400 Subject: [PATCH 01/11] chore: expose blob_versioned_hashes methods on block (#8530) --- crates/primitives/src/block.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/crates/primitives/src/block.rs b/crates/primitives/src/block.rs index b37d5146fae4..a797b01bb02e 100644 --- a/crates/primitives/src/block.rs +++ b/crates/primitives/src/block.rs @@ -144,6 +144,32 @@ impl Block { self.body.iter().any(|tx| tx.is_eip4844()) } + /// Returns an iterator over all blob transactions of the block + #[inline] + pub fn blob_transactions_iter(&self) -> impl Iterator + '_ { + self.body.iter().filter(|tx| tx.is_eip4844()) + } + + /// Returns only the blob transactions, if any, from the block body. + #[inline] + pub fn blob_transactions(&self) -> Vec<&TransactionSigned> { + self.blob_transactions_iter().collect() + } + + /// Returns an iterator over all blob versioned hashes from the block body. + #[inline] + pub fn blob_versioned_hashes_iter(&self) -> impl Iterator + '_ { + self.blob_transactions_iter() + .filter_map(|tx| tx.as_eip4844().map(|blob_tx| &blob_tx.blob_versioned_hashes)) + .flatten() + } + + /// Returns all blob versioned hashes from the block body. + #[inline] + pub fn blob_versioned_hashes(&self) -> Vec<&B256> { + self.blob_versioned_hashes_iter().collect() + } + /// Calculates a heuristic for the in-memory size of the [Block]. #[inline] pub fn size(&self) -> usize { From b80469d9fdffacd3040c32b5c38ce17ac88e4207 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Fri, 31 May 2024 13:46:50 -0400 Subject: [PATCH 02/11] fix: replace storage action metric recording (#8501) --- .../src/providers/database/metrics.rs | 123 +++++++++++------ etc/grafana/dashboards/overview.json | 125 ------------------ 2 files changed, 85 insertions(+), 163 deletions(-) diff --git a/crates/storage/provider/src/providers/database/metrics.rs b/crates/storage/provider/src/providers/database/metrics.rs index c16ce385c74d..ba43298c36b5 100644 --- a/crates/storage/provider/src/providers/database/metrics.rs +++ b/crates/storage/provider/src/providers/database/metrics.rs @@ -5,13 +5,19 @@ use std::time::{Duration, Instant}; #[derive(Debug)] pub(crate) struct DurationsRecorder { start: Instant, + current_metrics: DatabaseProviderMetrics, pub(crate) actions: Vec<(Action, Duration)>, latest: Option, } impl Default for DurationsRecorder { fn default() -> Self { - Self { start: Instant::now(), actions: Vec::new(), latest: None } + Self { + start: Instant::now(), + actions: Vec::new(), + latest: None, + current_metrics: DatabaseProviderMetrics::default(), + } } } @@ -20,7 +26,7 @@ impl DurationsRecorder { /// `action` label. pub(crate) fn record_duration(&mut self, action: Action, duration: Duration) { self.actions.push((action, duration)); - Metrics::new_with_labels(&[("action", action.as_str())]).duration.record(duration); + self.current_metrics.record_duration(action, duration); self.latest = Some(self.start.elapsed()); } @@ -31,8 +37,7 @@ impl DurationsRecorder { let duration = elapsed - self.latest.unwrap_or_default(); self.actions.push((action, duration)); - Metrics::new_with_labels(&[("action", action.as_str())]).duration.record(duration); - + self.current_metrics.record_duration(action, duration); self.latest = Some(elapsed); } } @@ -59,44 +64,86 @@ pub(crate) enum Action { InsertBlockRequests, InsertBlockBodyIndices, InsertTransactionBlocks, - GetNextTxNum, GetParentTD, } -impl Action { - const fn as_str(&self) -> &'static str { - match self { - Self::InsertStorageHashing => "insert storage hashing", - Self::InsertAccountHashing => "insert account hashing", - Self::InsertMerkleTree => "insert merkle tree", - Self::InsertBlock => "insert block", - Self::InsertState => "insert state", - Self::InsertHashes => "insert hashes", - Self::InsertHistoryIndices => "insert history indices", - Self::UpdatePipelineStages => "update pipeline stages", - Self::InsertCanonicalHeaders => "insert canonical headers", - Self::InsertHeaders => "insert headers", - Self::InsertHeaderNumbers => "insert header numbers", - Self::InsertHeaderTerminalDifficulties => "insert header TD", - Self::InsertBlockOmmers => "insert block ommers", - Self::InsertTransactionSenders => "insert tx senders", - Self::InsertTransactions => "insert transactions", - Self::InsertTransactionHashNumbers => "insert transaction hash numbers", - Self::InsertBlockWithdrawals => "insert block withdrawals", - Self::InsertBlockRequests => "insert block withdrawals", - Self::InsertBlockBodyIndices => "insert block body indices", - Self::InsertTransactionBlocks => "insert transaction blocks", - Self::GetNextTxNum => "get next tx num", - Self::GetParentTD => "get parent TD", - } - } -} - +/// Database provider metrics #[derive(Metrics)] #[metrics(scope = "storage.providers.database")] -/// Database provider metrics -struct Metrics { - /// The time it took to execute an action - duration: Histogram, +struct DatabaseProviderMetrics { + /// Duration of insert storage hashing + insert_storage_hashing: Histogram, + /// Duration of insert account hashing + insert_account_hashing: Histogram, + /// Duration of insert merkle tree + insert_merkle_tree: Histogram, + /// Duration of insert block + insert_block: Histogram, + /// Duration of insert state + insert_state: Histogram, + /// Duration of insert hashes + insert_hashes: Histogram, + /// Duration of insert history indices + insert_history_indices: Histogram, + /// Duration of update pipeline stages + update_pipeline_stages: Histogram, + /// Duration of insert canonical headers + insert_canonical_headers: Histogram, + /// Duration of insert headers + insert_headers: Histogram, + /// Duration of insert header numbers + insert_header_numbers: Histogram, + /// Duration of insert header TD + insert_header_td: Histogram, + /// Duration of insert block ommers + insert_block_ommers: Histogram, + /// Duration of insert tx senders + insert_tx_senders: Histogram, + /// Duration of insert transactions + insert_transactions: Histogram, + /// Duration of insert transaction hash numbers + insert_tx_hash_numbers: Histogram, + /// Duration of insert block withdrawals + insert_block_withdrawals: Histogram, + /// Duration of insert block requests + insert_block_requests: Histogram, + /// Duration of insert block body indices + insert_block_body_indices: Histogram, + /// Duration of insert transaction blocks + insert_tx_blocks: Histogram, + /// Duration of get next tx num + get_next_tx_num: Histogram, + /// Duration of get parent TD + get_parent_td: Histogram, +} + +impl DatabaseProviderMetrics { + /// Records the duration for the given action. + pub(crate) fn record_duration(&self, action: Action, duration: Duration) { + match action { + Action::InsertStorageHashing => self.insert_storage_hashing.record(duration), + Action::InsertAccountHashing => self.insert_account_hashing.record(duration), + Action::InsertMerkleTree => self.insert_merkle_tree.record(duration), + Action::InsertBlock => self.insert_block.record(duration), + Action::InsertState => self.insert_state.record(duration), + Action::InsertHashes => self.insert_hashes.record(duration), + Action::InsertHistoryIndices => self.insert_history_indices.record(duration), + Action::UpdatePipelineStages => self.update_pipeline_stages.record(duration), + Action::InsertCanonicalHeaders => self.insert_canonical_headers.record(duration), + Action::InsertHeaders => self.insert_headers.record(duration), + Action::InsertHeaderNumbers => self.insert_header_numbers.record(duration), + Action::InsertHeaderTerminalDifficulties => self.insert_header_td.record(duration), + Action::InsertBlockOmmers => self.insert_block_ommers.record(duration), + Action::InsertTransactionSenders => self.insert_tx_senders.record(duration), + Action::InsertTransactions => self.insert_transactions.record(duration), + Action::InsertTransactionHashNumbers => self.insert_tx_hash_numbers.record(duration), + Action::InsertBlockWithdrawals => self.insert_block_withdrawals.record(duration), + Action::InsertBlockRequests => self.insert_block_requests.record(duration), + Action::InsertBlockBodyIndices => self.insert_block_body_indices.record(duration), + Action::InsertTransactionBlocks => self.insert_tx_blocks.record(duration), + Action::GetNextTxNum => self.get_next_tx_num.record(duration), + Action::GetParentTD => self.get_parent_td.record(duration), + } + } } diff --git a/etc/grafana/dashboards/overview.json b/etc/grafana/dashboards/overview.json index ddbfbd07ffb6..e56c94b112c9 100644 --- a/etc/grafana/dashboards/overview.json +++ b/etc/grafana/dashboards/overview.json @@ -2105,131 +2105,6 @@ "title": "Freelist", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 61 - }, - "id": 189, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "reth_storage_providers_database_duration{instance=~\"$instance\", action=\"$database_action\", quantile=\"0.5\"}", - "instant": false, - "legendFormat": "{{quantile}} percentile", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "reth_storage_providers_database_duration{instance=~\"$instance\", action=\"$database_action\", quantile=\"0.95\"} ", - "hide": false, - "instant": false, - "legendFormat": "{{quantile}} percentile", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "reth_storage_providers_database_duration{instance=~\"$instance\", action=\"$database_action\", quantile=\"1\"} ", - "hide": false, - "instant": false, - "legendFormat": "{{quantile}} percentile", - "range": true, - "refId": "C" - } - ], - "title": "Database duration per action", - "type": "timeseries" - }, { "collapsed": false, "gridPos": { From fc8dce5ee7d408ba6770f4b33502f06549903024 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Fri, 31 May 2024 19:41:51 -0400 Subject: [PATCH 03/11] chore: bump alloy 61140ec (#8534) --- Cargo.lock | 118 ++++++++++++++++++++++++++--------------------------- Cargo.toml | 28 ++++++------- 2 files changed, 73 insertions(+), 73 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 07e8aa54b70f..59eb3b42a933 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -124,12 +124,12 @@ dependencies = [ [[package]] name = "alloy-consensus" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy?rev=7320d4c#7320d4ca3878bd059a96273ae33e52730618f1a0" +source = "git+https://github.com/alloy-rs/alloy?rev=61140ec#61140ec42988abed6db3d2d38044b2918e08d2a9" dependencies = [ - "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "alloy-primitives", "alloy-rlp", - "alloy-serde 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-serde 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "arbitrary", "c-kzg", "proptest", @@ -171,11 +171,11 @@ dependencies = [ [[package]] name = "alloy-eips" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy?rev=7320d4c#7320d4ca3878bd059a96273ae33e52730618f1a0" +source = "git+https://github.com/alloy-rs/alloy?rev=61140ec#61140ec42988abed6db3d2d38044b2918e08d2a9" dependencies = [ "alloy-primitives", "alloy-rlp", - "alloy-serde 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-serde 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "arbitrary", "c-kzg", "derive_more", @@ -203,10 +203,10 @@ dependencies = [ [[package]] name = "alloy-genesis" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy?rev=7320d4c#7320d4ca3878bd059a96273ae33e52730618f1a0" +source = "git+https://github.com/alloy-rs/alloy?rev=61140ec#61140ec42988abed6db3d2d38044b2918e08d2a9" dependencies = [ "alloy-primitives", - "alloy-serde 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-serde 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "serde", "serde_json", ] @@ -237,7 +237,7 @@ dependencies = [ [[package]] name = "alloy-json-rpc" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy?rev=7320d4c#7320d4ca3878bd059a96273ae33e52730618f1a0" +source = "git+https://github.com/alloy-rs/alloy?rev=61140ec#61140ec42988abed6db3d2d38044b2918e08d2a9" dependencies = [ "alloy-primitives", "serde", @@ -249,13 +249,13 @@ dependencies = [ [[package]] name = "alloy-network" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy?rev=7320d4c#7320d4ca3878bd059a96273ae33e52730618f1a0" +source = "git+https://github.com/alloy-rs/alloy?rev=61140ec#61140ec42988abed6db3d2d38044b2918e08d2a9" dependencies = [ - "alloy-consensus 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", - "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-consensus 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", + "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "alloy-json-rpc", "alloy-primitives", - "alloy-rpc-types 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-rpc-types 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "alloy-signer", "alloy-sol-types", "async-trait", @@ -267,9 +267,9 @@ dependencies = [ [[package]] name = "alloy-node-bindings" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy?rev=7320d4c#7320d4ca3878bd059a96273ae33e52730618f1a0" +source = "git+https://github.com/alloy-rs/alloy?rev=61140ec#61140ec42988abed6db3d2d38044b2918e08d2a9" dependencies = [ - "alloy-genesis 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-genesis 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "alloy-primitives", "k256", "serde_json", @@ -309,16 +309,16 @@ dependencies = [ [[package]] name = "alloy-provider" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy?rev=7320d4c#7320d4ca3878bd059a96273ae33e52730618f1a0" +source = "git+https://github.com/alloy-rs/alloy?rev=61140ec#61140ec42988abed6db3d2d38044b2918e08d2a9" dependencies = [ "alloy-chains", - "alloy-consensus 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", - "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-consensus 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", + "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "alloy-json-rpc", "alloy-network", "alloy-primitives", "alloy-rpc-client", - "alloy-rpc-types 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-rpc-types 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "alloy-rpc-types-trace", "alloy-transport", "alloy-transport-http", @@ -363,7 +363,7 @@ dependencies = [ [[package]] name = "alloy-rpc-client" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy?rev=7320d4c#7320d4ca3878bd059a96273ae33e52730618f1a0" +source = "git+https://github.com/alloy-rs/alloy?rev=61140ec#61140ec42988abed6db3d2d38044b2918e08d2a9" dependencies = [ "alloy-json-rpc", "alloy-transport", @@ -383,14 +383,14 @@ dependencies = [ [[package]] name = "alloy-rpc-types" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy?rev=7320d4c#7320d4ca3878bd059a96273ae33e52730618f1a0" +source = "git+https://github.com/alloy-rs/alloy?rev=61140ec#61140ec42988abed6db3d2d38044b2918e08d2a9" dependencies = [ - "alloy-consensus 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", - "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", - "alloy-genesis 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-consensus 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", + "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", + "alloy-genesis 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "alloy-primitives", "alloy-rlp", - "alloy-serde 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-serde 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "alloy-sol-types", "arbitrary", "itertools 0.12.1", @@ -423,19 +423,19 @@ dependencies = [ [[package]] name = "alloy-rpc-types-anvil" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy?rev=7320d4c#7320d4ca3878bd059a96273ae33e52730618f1a0" +source = "git+https://github.com/alloy-rs/alloy?rev=61140ec#61140ec42988abed6db3d2d38044b2918e08d2a9" dependencies = [ "alloy-primitives", - "alloy-serde 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-serde 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "serde", ] [[package]] name = "alloy-rpc-types-beacon" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy?rev=7320d4c#7320d4ca3878bd059a96273ae33e52730618f1a0" +source = "git+https://github.com/alloy-rs/alloy?rev=61140ec#61140ec42988abed6db3d2d38044b2918e08d2a9" dependencies = [ - "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "alloy-primitives", "alloy-rpc-types-engine", "serde", @@ -446,14 +446,14 @@ dependencies = [ [[package]] name = "alloy-rpc-types-engine" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy?rev=7320d4c#7320d4ca3878bd059a96273ae33e52730618f1a0" +source = "git+https://github.com/alloy-rs/alloy?rev=61140ec#61140ec42988abed6db3d2d38044b2918e08d2a9" dependencies = [ - "alloy-consensus 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", - "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-consensus 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", + "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "alloy-primitives", "alloy-rlp", - "alloy-rpc-types 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", - "alloy-serde 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-rpc-types 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", + "alloy-serde 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "jsonrpsee-types", "jsonwebtoken 9.3.0", "rand 0.8.5", @@ -464,11 +464,11 @@ dependencies = [ [[package]] name = "alloy-rpc-types-trace" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy?rev=7320d4c#7320d4ca3878bd059a96273ae33e52730618f1a0" +source = "git+https://github.com/alloy-rs/alloy?rev=61140ec#61140ec42988abed6db3d2d38044b2918e08d2a9" dependencies = [ "alloy-primitives", - "alloy-rpc-types 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", - "alloy-serde 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-rpc-types 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", + "alloy-serde 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "serde", "serde_json", ] @@ -476,7 +476,7 @@ dependencies = [ [[package]] name = "alloy-serde" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy?rev=7320d4c#7320d4ca3878bd059a96273ae33e52730618f1a0" +source = "git+https://github.com/alloy-rs/alloy?rev=61140ec#61140ec42988abed6db3d2d38044b2918e08d2a9" dependencies = [ "alloy-primitives", "serde", @@ -496,7 +496,7 @@ dependencies = [ [[package]] name = "alloy-signer" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy?rev=7320d4c#7320d4ca3878bd059a96273ae33e52730618f1a0" +source = "git+https://github.com/alloy-rs/alloy?rev=61140ec#61140ec42988abed6db3d2d38044b2918e08d2a9" dependencies = [ "alloy-primitives", "async-trait", @@ -509,9 +509,9 @@ dependencies = [ [[package]] name = "alloy-signer-wallet" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy?rev=7320d4c#7320d4ca3878bd059a96273ae33e52730618f1a0" +source = "git+https://github.com/alloy-rs/alloy?rev=61140ec#61140ec42988abed6db3d2d38044b2918e08d2a9" dependencies = [ - "alloy-consensus 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-consensus 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "alloy-network", "alloy-primitives", "alloy-signer", @@ -598,7 +598,7 @@ dependencies = [ [[package]] name = "alloy-transport" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy?rev=7320d4c#7320d4ca3878bd059a96273ae33e52730618f1a0" +source = "git+https://github.com/alloy-rs/alloy?rev=61140ec#61140ec42988abed6db3d2d38044b2918e08d2a9" dependencies = [ "alloy-json-rpc", "base64 0.22.1", @@ -616,7 +616,7 @@ dependencies = [ [[package]] name = "alloy-transport-http" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy?rev=7320d4c#7320d4ca3878bd059a96273ae33e52730618f1a0" +source = "git+https://github.com/alloy-rs/alloy?rev=61140ec#61140ec42988abed6db3d2d38044b2918e08d2a9" dependencies = [ "alloy-json-rpc", "alloy-transport", @@ -2867,7 +2867,7 @@ dependencies = [ name = "exex-rollup" version = "0.0.0" dependencies = [ - "alloy-consensus 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-consensus 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "alloy-rlp", "alloy-sol-types", "eyre", @@ -6394,9 +6394,9 @@ dependencies = [ name = "reth-codecs" version = "0.2.0-beta.7" dependencies = [ - "alloy-consensus 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", - "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", - "alloy-genesis 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-consensus 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", + "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", + "alloy-genesis 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "alloy-primitives", "arbitrary", "bytes", @@ -6619,9 +6619,9 @@ dependencies = [ name = "reth-e2e-test-utils" version = "0.2.0-beta.7" dependencies = [ - "alloy-consensus 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-consensus 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "alloy-network", - "alloy-rpc-types 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-rpc-types 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "alloy-signer", "alloy-signer-wallet", "eyre", @@ -6837,7 +6837,7 @@ dependencies = [ name = "reth-evm-ethereum" version = "0.2.0-beta.7" dependencies = [ - "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "alloy-sol-types", "reth-ethereum-consensus", "reth-evm", @@ -7412,12 +7412,12 @@ name = "reth-primitives" version = "0.2.0-beta.7" dependencies = [ "alloy-chains", - "alloy-consensus 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", - "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", - "alloy-genesis 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-consensus 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", + "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", + "alloy-genesis 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "alloy-primitives", "alloy-rlp", - "alloy-rpc-types 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-rpc-types 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "alloy-trie", "arbitrary", "assert_matches", @@ -7521,7 +7521,7 @@ dependencies = [ name = "reth-revm" version = "0.2.0-beta.7" dependencies = [ - "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "alloy-rlp", "reth-consensus-common", "reth-execution-errors", @@ -7708,7 +7708,7 @@ name = "reth-rpc-types" version = "0.2.0-beta.7" dependencies = [ "alloy-primitives", - "alloy-rpc-types 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-rpc-types 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "alloy-rpc-types-anvil", "alloy-rpc-types-beacon", "alloy-rpc-types-engine", @@ -7729,7 +7729,7 @@ name = "reth-rpc-types-compat" version = "0.2.0-beta.7" dependencies = [ "alloy-rlp", - "alloy-rpc-types 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-rpc-types 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "reth-primitives", "reth-rpc-types", "serde_json", @@ -7875,7 +7875,7 @@ dependencies = [ name = "reth-testing-utils" version = "0.2.0-beta.7" dependencies = [ - "alloy-genesis 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-genesis 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "rand 0.8.5", "reth-primitives", "secp256k1 0.28.2", @@ -8008,10 +8008,10 @@ dependencies = [ [[package]] name = "revm-inspectors" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/evm-inspectors?rev=0f43cf5#0f43cf5d15fef5f7db5a318a4985758852911c6a" +source = "git+https://github.com/paradigmxyz/evm-inspectors?rev=2b6fff1#2b6fff148a1c07e18156561114c04878618ece6b" dependencies = [ "alloy-primitives", - "alloy-rpc-types 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=7320d4c)", + "alloy-rpc-types 0.1.0 (git+https://github.com/alloy-rs/alloy?rev=61140ec)", "alloy-rpc-types-trace", "alloy-sol-types", "anstyle", diff --git a/Cargo.toml b/Cargo.toml index dfa5db3f0915..5e93ee684dd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -310,7 +310,7 @@ revm = { version = "9.0.0", features = [ revm-primitives = { version = "4.0.0", features = [ "std", ], default-features = false } -revm-inspectors = { git = "https://github.com/paradigmxyz/evm-inspectors", rev = "0f43cf5" } +revm-inspectors = { git = "https://github.com/paradigmxyz/evm-inspectors", rev = "2b6fff1" } # eth alloy-chains = "0.1.15" @@ -319,21 +319,21 @@ alloy-dyn-abi = "0.7.2" alloy-sol-types = "0.7.2" alloy-rlp = "0.3.4" alloy-trie = "0.4" -alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", rev = "7320d4c" } -alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", rev = "7320d4c" } -alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", rev = "7320d4c" } -alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", rev = "7320d4c" } -alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", rev = "7320d4c" } -alloy-genesis = { git = "https://github.com/alloy-rs/alloy", rev = "7320d4c" } -alloy-node-bindings = { git = "https://github.com/alloy-rs/alloy", rev = "7320d4c" } -alloy-provider = { git = "https://github.com/alloy-rs/alloy", rev = "7320d4c", default-features = false, features = [ +alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", rev = "61140ec" } +alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", rev = "61140ec" } +alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", rev = "61140ec" } +alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", rev = "61140ec" } +alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", rev = "61140ec" } +alloy-genesis = { git = "https://github.com/alloy-rs/alloy", rev = "61140ec" } +alloy-node-bindings = { git = "https://github.com/alloy-rs/alloy", rev = "61140ec" } +alloy-provider = { git = "https://github.com/alloy-rs/alloy", rev = "61140ec", default-features = false, features = [ "reqwest", ] } -alloy-eips = { git = "https://github.com/alloy-rs/alloy", default-features = false, rev = "7320d4c" } -alloy-signer = { git = "https://github.com/alloy-rs/alloy", rev = "7320d4c" } -alloy-signer-wallet = { git = "https://github.com/alloy-rs/alloy", rev = "7320d4c" } -alloy-network = { git = "https://github.com/alloy-rs/alloy", rev = "7320d4c" } -alloy-consensus = { git = "https://github.com/alloy-rs/alloy", rev = "7320d4c" } +alloy-eips = { git = "https://github.com/alloy-rs/alloy", default-features = false, rev = "61140ec" } +alloy-signer = { git = "https://github.com/alloy-rs/alloy", rev = "61140ec" } +alloy-signer-wallet = { git = "https://github.com/alloy-rs/alloy", rev = "61140ec" } +alloy-network = { git = "https://github.com/alloy-rs/alloy", rev = "61140ec" } +alloy-consensus = { git = "https://github.com/alloy-rs/alloy", rev = "61140ec" } # misc auto_impl = "1" From 105570ded0bb3cc8119574ac1aba8fd177b39f68 Mon Sep 17 00:00:00 2001 From: publicqi <56060664+publicqi@users.noreply.github.com> Date: Fri, 31 May 2024 19:48:42 -0400 Subject: [PATCH 04/11] fix: pruner config use toml over default (#8533) --- crates/node/builder/src/launch/common.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 0f16369fda44..a37f2fd8c730 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -273,7 +273,7 @@ impl LaunchContextWith> { /// Returns the configured [PruneConfig] pub fn prune_config(&self) -> Option { - self.node_config().prune_config().or_else(|| self.toml_config().prune.clone()) + self.toml_config().prune.clone().or_else(|| self.node_config().prune_config()) } /// Returns the configured [PruneModes] From a4df6bbd62fa86c64197f42270d54026d19a50d4 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Sat, 1 Jun 2024 12:56:13 +0200 Subject: [PATCH 05/11] feat: run `StaticFileProvider::check_consistency` on start up (#8143) --- bin/reth/src/commands/db/mod.rs | 19 +- .../src/commands/db/static_files/headers.rs | 2 +- bin/reth/src/commands/db/static_files/mod.rs | 8 +- .../src/commands/db/static_files/receipts.rs | 2 +- .../commands/db/static_files/transactions.rs | 2 +- bin/reth/src/commands/db/stats.rs | 2 +- .../src/commands/debug_cmd/build_block.rs | 15 +- bin/reth/src/commands/debug_cmd/execution.rs | 20 +- .../commands/debug_cmd/in_memory_merkle.rs | 18 +- bin/reth/src/commands/debug_cmd/merkle.rs | 17 +- .../src/commands/debug_cmd/replay_engine.rs | 17 +- bin/reth/src/commands/import.rs | 12 +- bin/reth/src/commands/import_op.rs | 12 +- bin/reth/src/commands/import_receipts_op.rs | 11 +- bin/reth/src/commands/init_cmd.rs | 8 +- bin/reth/src/commands/init_state.rs | 8 +- bin/reth/src/commands/p2p/mod.rs | 6 +- .../src/commands/recover/storage_tries.rs | 10 +- bin/reth/src/commands/stage/drop.rs | 12 +- bin/reth/src/commands/stage/dump/execution.rs | 8 +- .../commands/stage/dump/hashing_account.rs | 8 +- .../commands/stage/dump/hashing_storage.rs | 8 +- bin/reth/src/commands/stage/dump/merkle.rs | 8 +- bin/reth/src/commands/stage/dump/mod.rs | 9 +- bin/reth/src/commands/stage/run.rs | 14 +- bin/reth/src/commands/stage/unwind.rs | 11 +- crates/blockchain-tree-api/src/error.rs | 1 - crates/config/src/config.rs | 7 + crates/consensus/beacon/src/engine/mod.rs | 76 ++-- crates/evm/execution-errors/src/lib.rs | 5 - crates/evm/src/execute.rs | 2 +- crates/evm/src/lib.rs | 1 + crates/evm/src/noop.rs | 68 ++++ crates/net/downloaders/src/bodies/bodies.rs | 38 +- crates/node/builder/src/launch/common.rs | 84 ++++- crates/node/builder/src/launch/mod.rs | 2 +- crates/primitives/src/stage/mod.rs | 13 +- crates/prune/src/pruner.rs | 9 +- crates/stages/api/src/pipeline/mod.rs | 12 +- crates/stages/stages/src/stages/execution.rs | 5 +- crates/stages/stages/src/stages/mod.rs | 270 +++++++++++++- .../stages/stages/src/test_utils/test_db.rs | 60 ++- crates/static-file-types/src/segment.rs | 5 + crates/storage/db-common/src/init.rs | 13 +- crates/storage/errors/src/provider.rs | 3 + crates/storage/nippy-jar/src/error.rs | 2 + crates/storage/nippy-jar/src/lib.rs | 22 +- crates/storage/nippy-jar/src/writer.rs | 68 +++- crates/storage/provider/src/lib.rs | 2 +- .../provider/src/providers/database/mod.rs | 24 +- crates/storage/provider/src/providers/mod.rs | 4 +- .../src/providers/static_file/manager.rs | 350 +++++++++++++++++- .../provider/src/providers/static_file/mod.rs | 5 +- .../src/providers/static_file/writer.rs | 140 +++++-- crates/storage/provider/src/test_utils/mod.rs | 9 +- examples/db-access/src/main.rs | 10 +- examples/rpc-db/src/main.rs | 11 +- testing/ef-tests/src/cases/blockchain_test.rs | 9 +- 58 files changed, 1335 insertions(+), 262 deletions(-) create mode 100644 crates/evm/src/noop.rs diff --git a/bin/reth/src/commands/db/mod.rs b/bin/reth/src/commands/db/mod.rs index 6eedabcc7714..1aecef717f1e 100644 --- a/bin/reth/src/commands/db/mod.rs +++ b/bin/reth/src/commands/db/mod.rs @@ -14,7 +14,7 @@ use reth_db::{ version::{get_db_version, DatabaseVersionError, DB_VERSION}, }; use reth_primitives::ChainSpec; -use reth_provider::ProviderFactory; +use reth_provider::{providers::StaticFileProvider, ProviderFactory}; use std::{ io::{self, Write}, sync::Arc, @@ -96,7 +96,8 @@ pub enum Subcommands { macro_rules! db_ro_exec { ($chain:expr, $db_path:expr, $db_args:ident, $sfp:ident, $tool:ident, $command:block) => { let db = open_db_read_only($db_path, $db_args)?; - let provider_factory = ProviderFactory::new(db, $chain.clone(), $sfp)?; + let provider_factory = + ProviderFactory::new(db, $chain.clone(), StaticFileProvider::read_only($sfp)?); let $tool = DbTool::new(provider_factory, $chain.clone())?; $command; @@ -156,16 +157,22 @@ impl Command { } let db = open_db(&db_path, db_args)?; - let provider_factory = - ProviderFactory::new(db, self.chain.clone(), static_files_path.clone())?; + let provider_factory = ProviderFactory::new( + db, + self.chain.clone(), + StaticFileProvider::read_write(&static_files_path)?, + ); let tool = DbTool::new(provider_factory, self.chain.clone())?; tool.drop(db_path, static_files_path)?; } Subcommands::Clear(command) => { let db = open_db(&db_path, db_args)?; - let provider_factory = - ProviderFactory::new(db, self.chain.clone(), static_files_path)?; + let provider_factory = ProviderFactory::new( + db, + self.chain.clone(), + StaticFileProvider::read_write(static_files_path)?, + ); command.execute(provider_factory)?; } diff --git a/bin/reth/src/commands/db/static_files/headers.rs b/bin/reth/src/commands/db/static_files/headers.rs index 7584f614c9bf..e9dd8802679e 100644 --- a/bin/reth/src/commands/db/static_files/headers.rs +++ b/bin/reth/src/commands/db/static_files/headers.rs @@ -38,7 +38,7 @@ impl Command { let path: PathBuf = StaticFileSegment::Headers .filename_with_configuration(filters, compression, &block_range) .into(); - let provider = StaticFileProvider::new(PathBuf::default())?; + let provider = StaticFileProvider::read_only(PathBuf::default())?; let jar_provider = provider.get_segment_provider_from_block( StaticFileSegment::Headers, self.from, diff --git a/bin/reth/src/commands/db/static_files/mod.rs b/bin/reth/src/commands/db/static_files/mod.rs index 8f5930e10835..f7532237fbaa 100644 --- a/bin/reth/src/commands/db/static_files/mod.rs +++ b/bin/reth/src/commands/db/static_files/mod.rs @@ -16,7 +16,7 @@ use reth_primitives::{ }, BlockNumber, ChainSpec, StaticFileSegment, }; -use reth_provider::{BlockNumReader, ProviderFactory}; +use reth_provider::{providers::StaticFileProvider, BlockNumReader, ProviderFactory}; use reth_static_file::{segments as static_file_segments, segments::Segment}; use std::{ path::{Path, PathBuf}, @@ -99,7 +99,11 @@ impl Command { data_dir.db().as_path(), db_args.with_max_read_transaction_duration(Some(MaxReadTransactionDuration::Unbounded)), )?; - let provider_factory = Arc::new(ProviderFactory::new(db, chain, data_dir.static_files())?); + let provider_factory = Arc::new(ProviderFactory::new( + db, + chain, + StaticFileProvider::read_only(data_dir.static_files())?, + )); { if !self.only_bench { diff --git a/bin/reth/src/commands/db/static_files/receipts.rs b/bin/reth/src/commands/db/static_files/receipts.rs index 50ebd42caf6b..5c2e8ea7aee9 100644 --- a/bin/reth/src/commands/db/static_files/receipts.rs +++ b/bin/reth/src/commands/db/static_files/receipts.rs @@ -43,7 +43,7 @@ impl Command { .filename_with_configuration(filters, compression, &block_range) .into(); - let provider = StaticFileProvider::new(PathBuf::default())?; + let provider = StaticFileProvider::read_only(PathBuf::default())?; let jar_provider = provider.get_segment_provider_from_block( StaticFileSegment::Receipts, self.from, diff --git a/bin/reth/src/commands/db/static_files/transactions.rs b/bin/reth/src/commands/db/static_files/transactions.rs index a5939c0f41bb..cd8b8811df79 100644 --- a/bin/reth/src/commands/db/static_files/transactions.rs +++ b/bin/reth/src/commands/db/static_files/transactions.rs @@ -42,7 +42,7 @@ impl Command { let path: PathBuf = StaticFileSegment::Transactions .filename_with_configuration(filters, compression, &block_range) .into(); - let provider = StaticFileProvider::new(PathBuf::default())?; + let provider = StaticFileProvider::read_only(PathBuf::default())?; let jar_provider = provider.get_segment_provider_from_block( StaticFileSegment::Transactions, self.from, diff --git a/bin/reth/src/commands/db/stats.rs b/bin/reth/src/commands/db/stats.rs index 8ea473e8fe6b..c3675f387b0b 100644 --- a/bin/reth/src/commands/db/stats.rs +++ b/bin/reth/src/commands/db/stats.rs @@ -168,7 +168,7 @@ impl Command { } let static_files = iter_static_files(data_dir.static_files())?; - let static_file_provider = StaticFileProvider::new(data_dir.static_files())?; + let static_file_provider = StaticFileProvider::read_only(data_dir.static_files())?; let mut total_data_size = 0; let mut total_index_size = 0; diff --git a/bin/reth/src/commands/debug_cmd/build_block.rs b/bin/reth/src/commands/debug_cmd/build_block.rs index 632f11d758d8..f4720d81708d 100644 --- a/bin/reth/src/commands/debug_cmd/build_block.rs +++ b/bin/reth/src/commands/debug_cmd/build_block.rs @@ -35,8 +35,9 @@ use reth_primitives::{ U256, }; use reth_provider::{ - providers::BlockchainProvider, BlockHashReader, BlockReader, BlockWriter, - BundleStateWithReceipts, ProviderFactory, StageCheckpointReader, StateProviderFactory, + providers::{BlockchainProvider, StaticFileProvider}, + BlockHashReader, BlockReader, BlockWriter, BundleStateWithReceipts, ProviderFactory, + StageCheckpointReader, StateProviderFactory, }; use reth_revm::database::StateProviderDatabase; use reth_rpc_types::engine::{BlobsBundleV1, PayloadAttributes}; @@ -113,8 +114,10 @@ impl Command { let factory = ProviderFactory::new( db, self.chain.clone(), - self.datadir.unwrap_or_chain_default(self.chain.chain).static_files(), - )?; + StaticFileProvider::read_only( + self.datadir.unwrap_or_chain_default(self.chain.chain).static_files(), + )?, + ); let provider = factory.provider()?; let best_number = @@ -155,8 +158,8 @@ impl Command { let provider_factory = ProviderFactory::new( Arc::clone(&db), Arc::clone(&self.chain), - data_dir.static_files(), - )?; + StaticFileProvider::read_only(data_dir.static_files())?, + ); let consensus: Arc = Arc::new(EthBeaconConsensus::new(Arc::clone(&self.chain))); diff --git a/bin/reth/src/commands/debug_cmd/execution.rs b/bin/reth/src/commands/debug_cmd/execution.rs index 87e122b4c061..5403f450c1cc 100644 --- a/bin/reth/src/commands/debug_cmd/execution.rs +++ b/bin/reth/src/commands/debug_cmd/execution.rs @@ -31,8 +31,8 @@ use reth_primitives::{ stage::StageId, BlockHashOrNumber, BlockNumber, ChainSpec, PruneModes, B256, }; use reth_provider::{ - BlockExecutionWriter, HeaderSyncMode, ProviderFactory, StageCheckpointReader, - StaticFileProviderFactory, + providers::StaticFileProvider, BlockExecutionWriter, HeaderSyncMode, ProviderFactory, + StageCheckpointReader, StaticFileProviderFactory, }; use reth_stages::{ sets::DefaultStages, @@ -156,6 +156,9 @@ impl Command { default_peers_path: PathBuf, ) -> eyre::Result { let secret_key = get_secret_key(&network_secret_path)?; + let static_files = StaticFileProvider::read_only( + self.datadir.unwrap_or_chain_default(self.chain.chain).static_files(), + )?; let network = self .network .network_config(config, self.chain.clone(), secret_key, default_peers_path) @@ -165,11 +168,7 @@ impl Command { self.network.discovery.addr, self.network.discovery.port, )) - .build(ProviderFactory::new( - db, - self.chain.clone(), - self.datadir.unwrap_or_chain_default(self.chain.chain).static_files(), - )?) + .build(ProviderFactory::new(db, self.chain.clone(), static_files)) .start_network() .await?; info!(target: "reth::cli", peer_id = %network.peer_id(), local_addr = %network.local_addr(), "Connected to P2P network"); @@ -210,8 +209,11 @@ impl Command { fs::create_dir_all(&db_path)?; let db = Arc::new(init_db(db_path, self.db.database_args())?); - let provider_factory = - ProviderFactory::new(db.clone(), self.chain.clone(), data_dir.static_files())?; + let provider_factory = ProviderFactory::new( + db.clone(), + self.chain.clone(), + StaticFileProvider::read_write(data_dir.static_files())?, + ); debug!(target: "reth::cli", chain=%self.chain.chain, genesis=?self.chain.genesis_hash(), "Initializing genesis"); init_genesis(provider_factory.clone())?; diff --git a/bin/reth/src/commands/debug_cmd/in_memory_merkle.rs b/bin/reth/src/commands/debug_cmd/in_memory_merkle.rs index 4a38bec01f8a..3e1d984686c3 100644 --- a/bin/reth/src/commands/debug_cmd/in_memory_merkle.rs +++ b/bin/reth/src/commands/debug_cmd/in_memory_merkle.rs @@ -22,9 +22,9 @@ use reth_network::NetworkHandle; use reth_network_api::NetworkInfo; use reth_primitives::{stage::StageId, BlockHashOrNumber, ChainSpec, Receipts}; use reth_provider::{ - AccountExtReader, BundleStateWithReceipts, HashingWriter, HeaderProvider, - LatestStateProviderRef, OriginalValuesKnown, ProviderFactory, StageCheckpointReader, - StateWriter, StaticFileProviderFactory, StorageReader, + providers::StaticFileProvider, AccountExtReader, BundleStateWithReceipts, HashingWriter, + HeaderProvider, LatestStateProviderRef, OriginalValuesKnown, ProviderFactory, + StageCheckpointReader, StateWriter, StaticFileProviderFactory, StorageReader, }; use reth_revm::database::StateProviderDatabase; use reth_tasks::TaskExecutor; @@ -97,8 +97,10 @@ impl Command { .build(ProviderFactory::new( db, self.chain.clone(), - self.datadir.unwrap_or_chain_default(self.chain.chain).static_files(), - )?) + StaticFileProvider::read_only( + self.datadir.unwrap_or_chain_default(self.chain.chain).static_files(), + )?, + )) .start_network() .await?; info!(target: "reth::cli", peer_id = %network.peer_id(), local_addr = %network.local_addr(), "Connected to P2P network"); @@ -117,7 +119,11 @@ impl Command { // initialize the database let db = Arc::new(init_db(db_path, self.db.database_args())?); - let factory = ProviderFactory::new(&db, self.chain.clone(), data_dir.static_files())?; + let factory = ProviderFactory::new( + &db, + self.chain.clone(), + StaticFileProvider::read_only(data_dir.static_files())?, + ); let provider = factory.provider()?; // Look up merkle checkpoint diff --git a/bin/reth/src/commands/debug_cmd/merkle.rs b/bin/reth/src/commands/debug_cmd/merkle.rs index 83562d967157..081764ac4294 100644 --- a/bin/reth/src/commands/debug_cmd/merkle.rs +++ b/bin/reth/src/commands/debug_cmd/merkle.rs @@ -24,8 +24,9 @@ use reth_network_api::NetworkInfo; use reth_network_p2p::full_block::FullBlockClient; use reth_primitives::{stage::StageCheckpoint, BlockHashOrNumber, ChainSpec, PruneModes}; use reth_provider::{ - BlockNumReader, BlockWriter, BundleStateWithReceipts, HeaderProvider, LatestStateProviderRef, - OriginalValuesKnown, ProviderError, ProviderFactory, StateWriter, + providers::StaticFileProvider, BlockNumReader, BlockWriter, BundleStateWithReceipts, + HeaderProvider, LatestStateProviderRef, OriginalValuesKnown, ProviderError, ProviderFactory, + StateWriter, }; use reth_revm::database::StateProviderDatabase; use reth_stages::{ @@ -102,8 +103,10 @@ impl Command { .build(ProviderFactory::new( db, self.chain.clone(), - self.datadir.unwrap_or_chain_default(self.chain.chain).static_files(), - )?) + StaticFileProvider::read_only( + self.datadir.unwrap_or_chain_default(self.chain.chain).static_files(), + )?, + )) .start_network() .await?; info!(target: "reth::cli", peer_id = %network.peer_id(), local_addr = %network.local_addr(), "Connected to P2P network"); @@ -122,7 +125,11 @@ impl Command { // initialize the database let db = Arc::new(init_db(db_path, self.db.database_args())?); - let factory = ProviderFactory::new(&db, self.chain.clone(), data_dir.static_files())?; + let factory = ProviderFactory::new( + &db, + self.chain.clone(), + StaticFileProvider::read_only(data_dir.static_files())?, + ); let provider_rw = factory.provider_rw()?; // Configure and build network diff --git a/bin/reth/src/commands/debug_cmd/replay_engine.rs b/bin/reth/src/commands/debug_cmd/replay_engine.rs index 72031ce1b86b..98666888722b 100644 --- a/bin/reth/src/commands/debug_cmd/replay_engine.rs +++ b/bin/reth/src/commands/debug_cmd/replay_engine.rs @@ -25,8 +25,8 @@ use reth_node_core::engine::engine_store::{EngineMessageStore, StoredEngineApiMe use reth_payload_builder::{PayloadBuilderHandle, PayloadBuilderService}; use reth_primitives::{ChainSpec, PruneModes}; use reth_provider::{ - providers::BlockchainProvider, CanonStateSubscriptions, ProviderFactory, - StaticFileProviderFactory, + providers::{BlockchainProvider, StaticFileProvider}, + CanonStateSubscriptions, ProviderFactory, StaticFileProviderFactory, }; use reth_stages::Pipeline; use reth_static_file::StaticFileProducer; @@ -100,8 +100,10 @@ impl Command { .build(ProviderFactory::new( db, self.chain.clone(), - self.datadir.unwrap_or_chain_default(self.chain.chain).static_files(), - )?) + StaticFileProvider::read_only( + self.datadir.unwrap_or_chain_default(self.chain.chain).static_files(), + )?, + )) .start_network() .await?; info!(target: "reth::cli", peer_id = %network.peer_id(), local_addr = %network.local_addr(), "Connected to P2P network"); @@ -120,8 +122,11 @@ impl Command { // Initialize the database let db = Arc::new(init_db(db_path, self.db.database_args())?); - let provider_factory = - ProviderFactory::new(db.clone(), self.chain.clone(), data_dir.static_files())?; + let provider_factory = ProviderFactory::new( + db.clone(), + self.chain.clone(), + StaticFileProvider::read_only(data_dir.static_files())?, + ); let consensus: Arc = Arc::new(EthBeaconConsensus::new(Arc::clone(&self.chain))); diff --git a/bin/reth/src/commands/import.rs b/bin/reth/src/commands/import.rs index 869bcefbd643..9fb967dd9e1d 100644 --- a/bin/reth/src/commands/import.rs +++ b/bin/reth/src/commands/import.rs @@ -29,8 +29,9 @@ use reth_network_p2p::{ use reth_node_events::node::NodeEvent; use reth_primitives::{stage::StageId, ChainSpec, PruneModes, B256}; use reth_provider::{ - BlockNumReader, ChainSpecProvider, HeaderProvider, HeaderSyncMode, ProviderError, - ProviderFactory, StageCheckpointReader, StaticFileProviderFactory, + providers::StaticFileProvider, BlockNumReader, ChainSpecProvider, HeaderProvider, + HeaderSyncMode, ProviderError, ProviderFactory, StageCheckpointReader, + StaticFileProviderFactory, }; use reth_stages::{prelude::*, Pipeline, StageSet}; use reth_static_file::StaticFileProducer; @@ -117,8 +118,11 @@ impl ImportCommand { info!(target: "reth::cli", path = ?db_path, "Opening database"); let db = Arc::new(init_db(db_path, self.db.database_args())?); info!(target: "reth::cli", "Database opened"); - let provider_factory = - ProviderFactory::new(db.clone(), self.chain.clone(), data_dir.static_files())?; + let provider_factory = ProviderFactory::new( + db.clone(), + self.chain.clone(), + StaticFileProvider::read_write(data_dir.static_files())?, + ); debug!(target: "reth::cli", chain=%self.chain.chain, genesis=?self.chain.genesis_hash(), "Initializing genesis"); diff --git a/bin/reth/src/commands/import_op.rs b/bin/reth/src/commands/import_op.rs index 1da75951b991..bb2ef4567b69 100644 --- a/bin/reth/src/commands/import_op.rs +++ b/bin/reth/src/commands/import_op.rs @@ -20,7 +20,10 @@ use reth_downloaders::file_client::{ }; use reth_optimism_primitives::bedrock_import::is_dup_tx; use reth_primitives::{stage::StageId, PruneModes}; -use reth_provider::{ProviderFactory, StageCheckpointReader, StaticFileProviderFactory}; +use reth_provider::{ + providers::StaticFileProvider, ProviderFactory, StageCheckpointReader, + StaticFileProviderFactory, +}; use reth_static_file::StaticFileProducer; use std::{path::PathBuf, sync::Arc}; use tracing::{debug, error, info}; @@ -91,8 +94,11 @@ impl ImportOpCommand { let db = Arc::new(init_db(db_path, self.db.database_args())?); info!(target: "reth::cli", "Database opened"); - let provider_factory = - ProviderFactory::new(db.clone(), chain_spec.clone(), data_dir.static_files())?; + let provider_factory = ProviderFactory::new( + db.clone(), + chain_spec.clone(), + StaticFileProvider::read_write(data_dir.static_files())?, + ); debug!(target: "reth::cli", chain=%chain_spec.chain, genesis=?chain_spec.genesis_hash(), "Initializing genesis"); diff --git a/bin/reth/src/commands/import_receipts_op.rs b/bin/reth/src/commands/import_receipts_op.rs index 44c79cd2fe6a..6fefc4ea2a35 100644 --- a/bin/reth/src/commands/import_receipts_op.rs +++ b/bin/reth/src/commands/import_receipts_op.rs @@ -16,8 +16,8 @@ use reth_node_core::version::SHORT_VERSION; use reth_optimism_primitives::bedrock_import::is_dup_tx; use reth_primitives::{stage::StageId, Receipts, StaticFileSegment}; use reth_provider::{ - BundleStateWithReceipts, OriginalValuesKnown, ProviderFactory, StageCheckpointReader, - StateWriter, StaticFileProviderFactory, StaticFileWriter, StatsReader, + providers::StaticFileProvider, BundleStateWithReceipts, OriginalValuesKnown, ProviderFactory, + StageCheckpointReader, StateWriter, StaticFileProviderFactory, StaticFileWriter, StatsReader, }; use tracing::{debug, error, info, trace}; @@ -77,8 +77,11 @@ impl ImportReceiptsOpCommand { let db = Arc::new(init_db(db_path, self.db.database_args())?); info!(target: "reth::cli", "Database opened"); - let provider_factory = - ProviderFactory::new(db.clone(), chain_spec.clone(), data_dir.static_files())?; + let provider_factory = ProviderFactory::new( + db.clone(), + chain_spec.clone(), + StaticFileProvider::read_write(data_dir.static_files())?, + ); import_receipts_from_file( provider_factory, diff --git a/bin/reth/src/commands/init_cmd.rs b/bin/reth/src/commands/init_cmd.rs index 3b900b3f01a2..0da65aba5033 100644 --- a/bin/reth/src/commands/init_cmd.rs +++ b/bin/reth/src/commands/init_cmd.rs @@ -11,7 +11,7 @@ use clap::Parser; use reth_db::init_db; use reth_db_common::init::init_genesis; use reth_primitives::ChainSpec; -use reth_provider::ProviderFactory; +use reth_provider::{providers::StaticFileProvider, ProviderFactory}; use std::sync::Arc; use tracing::info; @@ -56,7 +56,11 @@ impl InitCommand { let db = Arc::new(init_db(&db_path, self.db.database_args())?); info!(target: "reth::cli", "Database opened"); - let provider_factory = ProviderFactory::new(db, self.chain, data_dir.static_files())?; + let provider_factory = ProviderFactory::new( + db, + self.chain, + StaticFileProvider::read_write(data_dir.static_files())?, + ); info!(target: "reth::cli", "Writing genesis block"); diff --git a/bin/reth/src/commands/init_state.rs b/bin/reth/src/commands/init_state.rs index f5ee0c4b1c15..b672f11f2063 100644 --- a/bin/reth/src/commands/init_state.rs +++ b/bin/reth/src/commands/init_state.rs @@ -12,7 +12,7 @@ use reth_config::config::EtlConfig; use reth_db::{database::Database, init_db}; use reth_db_common::init::init_from_state_dump; use reth_primitives::{ChainSpec, B256}; -use reth_provider::ProviderFactory; +use reth_provider::{providers::StaticFileProvider, ProviderFactory}; use std::{fs::File, io::BufReader, path::PathBuf, sync::Arc}; use tracing::info; @@ -78,7 +78,11 @@ impl InitStateCommand { let db = Arc::new(init_db(&db_path, self.db.database_args())?); info!(target: "reth::cli", "Database opened"); - let provider_factory = ProviderFactory::new(db, self.chain, data_dir.static_files())?; + let provider_factory = ProviderFactory::new( + db, + self.chain, + StaticFileProvider::read_write(data_dir.static_files())?, + ); let etl_config = EtlConfig::new( Some(EtlConfig::from_datadir(data_dir.data_dir())), EtlConfig::default_file_size(), diff --git a/bin/reth/src/commands/p2p/mod.rs b/bin/reth/src/commands/p2p/mod.rs index 64233d6e5b36..627c2d36b44f 100644 --- a/bin/reth/src/commands/p2p/mod.rs +++ b/bin/reth/src/commands/p2p/mod.rs @@ -17,7 +17,7 @@ use reth_db::create_db; use reth_network::NetworkConfigBuilder; use reth_network_p2p::bodies::client::BodiesClient; use reth_primitives::{BlockHashOrNumber, ChainSpec}; -use reth_provider::ProviderFactory; +use reth_provider::{providers::StaticFileProvider, ProviderFactory}; use std::{ net::{IpAddr, SocketAddrV4, SocketAddrV6}, path::PathBuf, @@ -165,8 +165,8 @@ impl Command { .build(Arc::new(ProviderFactory::new( noop_db, self.chain.clone(), - data_dir.static_files(), - )?)) + StaticFileProvider::read_write(data_dir.static_files())?, + ))) .start_network() .await?; diff --git a/bin/reth/src/commands/recover/storage_tries.rs b/bin/reth/src/commands/recover/storage_tries.rs index 583829bc39bc..8aa568974925 100644 --- a/bin/reth/src/commands/recover/storage_tries.rs +++ b/bin/reth/src/commands/recover/storage_tries.rs @@ -12,7 +12,9 @@ use reth_db::{ use reth_db_common::init::init_genesis; use reth_node_core::args::DatabaseArgs; use reth_primitives::ChainSpec; -use reth_provider::{BlockNumReader, HeaderProvider, ProviderError, ProviderFactory}; +use reth_provider::{ + providers::StaticFileProvider, BlockNumReader, HeaderProvider, ProviderError, ProviderFactory, +}; use reth_trie::StateRoot; use std::{fs, sync::Arc}; use tracing::*; @@ -55,7 +57,11 @@ impl Command { fs::create_dir_all(&db_path)?; let db = Arc::new(init_db(db_path, self.db.database_args())?); - let factory = ProviderFactory::new(&db, self.chain.clone(), data_dir.static_files())?; + let factory = ProviderFactory::new( + &db, + self.chain.clone(), + StaticFileProvider::read_write(data_dir.static_files())?, + ); debug!(target: "reth::cli", chain=%self.chain.chain, genesis=?self.chain.genesis_hash(), "Initializing genesis"); init_genesis(factory.clone())?; diff --git a/bin/reth/src/commands/stage/drop.rs b/bin/reth/src/commands/stage/drop.rs index 47af150ce526..042bafa3ada7 100644 --- a/bin/reth/src/commands/stage/drop.rs +++ b/bin/reth/src/commands/stage/drop.rs @@ -16,7 +16,10 @@ use reth_fs_util as fs; use reth_primitives::{ stage::StageId, static_file::find_fixed_range, ChainSpec, StaticFileSegment, }; -use reth_provider::{providers::StaticFileWriter, ProviderFactory, StaticFileProviderFactory}; +use reth_provider::{ + providers::{StaticFileProvider, StaticFileWriter}, + ProviderFactory, StaticFileProviderFactory, +}; use std::sync::Arc; /// `reth drop-stage` command @@ -59,8 +62,11 @@ impl Command { fs::create_dir_all(&db_path)?; let db = open_db(db_path.as_ref(), self.db.database_args())?; - let provider_factory = - ProviderFactory::new(db, self.chain.clone(), data_dir.static_files())?; + let provider_factory = ProviderFactory::new( + db, + self.chain.clone(), + StaticFileProvider::read_write(data_dir.static_files())?, + ); let static_file_provider = provider_factory.static_file_provider(); let tool = DbTool::new(provider_factory, self.chain.clone())?; diff --git a/bin/reth/src/commands/stage/dump/execution.rs b/bin/reth/src/commands/stage/dump/execution.rs index d8f12b50af7c..abe1fccd70fc 100644 --- a/bin/reth/src/commands/stage/dump/execution.rs +++ b/bin/reth/src/commands/stage/dump/execution.rs @@ -6,7 +6,7 @@ use reth_db::{ }; use reth_node_core::dirs::{ChainPath, DataDirPath}; use reth_primitives::stage::StageCheckpoint; -use reth_provider::{ChainSpecProvider, ProviderFactory}; +use reth_provider::{providers::StaticFileProvider, ChainSpecProvider, ProviderFactory}; use reth_stages::{stages::ExecutionStage, Stage, UnwindInput}; use tracing::info; @@ -25,7 +25,11 @@ pub(crate) async fn dump_execution_stage( if should_run { dry_run( - ProviderFactory::new(output_db, db_tool.chain.clone(), output_datadir.static_files())?, + ProviderFactory::new( + output_db, + db_tool.chain.clone(), + StaticFileProvider::read_only(output_datadir.static_files())?, + ), to, from, ) diff --git a/bin/reth/src/commands/stage/dump/hashing_account.rs b/bin/reth/src/commands/stage/dump/hashing_account.rs index 2f28ba129a10..ebcf1ad8c093 100644 --- a/bin/reth/src/commands/stage/dump/hashing_account.rs +++ b/bin/reth/src/commands/stage/dump/hashing_account.rs @@ -4,7 +4,7 @@ use eyre::Result; use reth_db::{database::Database, table::TableImporter, tables, DatabaseEnv}; use reth_node_core::dirs::{ChainPath, DataDirPath}; use reth_primitives::{stage::StageCheckpoint, BlockNumber}; -use reth_provider::ProviderFactory; +use reth_provider::{providers::StaticFileProvider, ProviderFactory}; use reth_stages::{stages::AccountHashingStage, Stage, UnwindInput}; use tracing::info; @@ -30,7 +30,11 @@ pub(crate) async fn dump_hashing_account_stage( if should_run { dry_run( - ProviderFactory::new(output_db, db_tool.chain.clone(), output_datadir.static_files())?, + ProviderFactory::new( + output_db, + db_tool.chain.clone(), + StaticFileProvider::read_only(output_datadir.static_files())?, + ), to, from, ) diff --git a/bin/reth/src/commands/stage/dump/hashing_storage.rs b/bin/reth/src/commands/stage/dump/hashing_storage.rs index 7d38892dc8bd..f990357d1895 100644 --- a/bin/reth/src/commands/stage/dump/hashing_storage.rs +++ b/bin/reth/src/commands/stage/dump/hashing_storage.rs @@ -4,7 +4,7 @@ use eyre::Result; use reth_db::{database::Database, table::TableImporter, tables, DatabaseEnv}; use reth_node_core::dirs::{ChainPath, DataDirPath}; use reth_primitives::stage::StageCheckpoint; -use reth_provider::ProviderFactory; +use reth_provider::{providers::StaticFileProvider, ProviderFactory}; use reth_stages::{stages::StorageHashingStage, Stage, UnwindInput}; use tracing::info; @@ -21,7 +21,11 @@ pub(crate) async fn dump_hashing_storage_stage( if should_run { dry_run( - ProviderFactory::new(output_db, db_tool.chain.clone(), output_datadir.static_files())?, + ProviderFactory::new( + output_db, + db_tool.chain.clone(), + StaticFileProvider::read_only(output_datadir.static_files())?, + ), to, from, ) diff --git a/bin/reth/src/commands/stage/dump/merkle.rs b/bin/reth/src/commands/stage/dump/merkle.rs index 9b421be7ca3f..bb3ab80bfb16 100644 --- a/bin/reth/src/commands/stage/dump/merkle.rs +++ b/bin/reth/src/commands/stage/dump/merkle.rs @@ -6,7 +6,7 @@ use reth_db::{database::Database, table::TableImporter, tables, DatabaseEnv}; use reth_exex::ExExManagerHandle; use reth_node_core::dirs::{ChainPath, DataDirPath}; use reth_primitives::{stage::StageCheckpoint, BlockNumber, PruneModes}; -use reth_provider::ProviderFactory; +use reth_provider::{providers::StaticFileProvider, ProviderFactory}; use reth_stages::{ stages::{ AccountHashingStage, ExecutionStage, ExecutionStageThresholds, MerkleStage, @@ -45,7 +45,11 @@ pub(crate) async fn dump_merkle_stage( if should_run { dry_run( - ProviderFactory::new(output_db, db_tool.chain.clone(), output_datadir.static_files())?, + ProviderFactory::new( + output_db, + db_tool.chain.clone(), + StaticFileProvider::read_only(output_datadir.static_files())?, + ), to, from, ) diff --git a/bin/reth/src/commands/stage/dump/mod.rs b/bin/reth/src/commands/stage/dump/mod.rs index fa4184356558..03cff6055ce5 100644 --- a/bin/reth/src/commands/stage/dump/mod.rs +++ b/bin/reth/src/commands/stage/dump/mod.rs @@ -17,7 +17,7 @@ use reth_db::{ }; use reth_node_core::dirs::PlatformPath; use reth_primitives::ChainSpec; -use reth_provider::ProviderFactory; +use reth_provider::{providers::StaticFileProvider, ProviderFactory}; use std::{path::PathBuf, sync::Arc}; use tracing::info; @@ -105,8 +105,11 @@ impl Command { let db_path = data_dir.db(); info!(target: "reth::cli", path = ?db_path, "Opening database"); let db = Arc::new(init_db(db_path, self.db.database_args())?); - let provider_factory = - ProviderFactory::new(db, self.chain.clone(), data_dir.static_files())?; + let provider_factory = ProviderFactory::new( + db, + self.chain.clone(), + StaticFileProvider::read_write(data_dir.static_files())?, + ); info!(target: "reth::cli", "Database opened"); diff --git a/bin/reth/src/commands/stage/run.rs b/bin/reth/src/commands/stage/run.rs index d34b67db42a1..e6b96bdc6a4a 100644 --- a/bin/reth/src/commands/stage/run.rs +++ b/bin/reth/src/commands/stage/run.rs @@ -25,7 +25,8 @@ use reth_downloaders::bodies::bodies::BodiesDownloaderBuilder; use reth_exex::ExExManagerHandle; use reth_primitives::ChainSpec; use reth_provider::{ - ProviderFactory, StageCheckpointReader, StageCheckpointWriter, StaticFileProviderFactory, + providers::StaticFileProvider, ProviderFactory, StageCheckpointReader, StageCheckpointWriter, + StaticFileProviderFactory, }; use reth_stages::{ stages::{ @@ -145,8 +146,11 @@ impl Command { let db = Arc::new(init_db(db_path, self.db.database_args())?); info!(target: "reth::cli", "Database opened"); - let factory = - ProviderFactory::new(Arc::clone(&db), self.chain.clone(), data_dir.static_files())?; + let factory = ProviderFactory::new( + Arc::clone(&db), + self.chain.clone(), + StaticFileProvider::read_write(data_dir.static_files())?, + ); let mut provider_rw = factory.provider_rw()?; if let Some(listen_addr) = self.metrics { @@ -195,8 +199,8 @@ impl Command { let provider_factory = Arc::new(ProviderFactory::new( db.clone(), self.chain.clone(), - data_dir.static_files(), - )?); + StaticFileProvider::read_write(data_dir.static_files())?, + )); let network = self .network diff --git a/bin/reth/src/commands/stage/unwind.rs b/bin/reth/src/commands/stage/unwind.rs index 320539280975..e5714f57b708 100644 --- a/bin/reth/src/commands/stage/unwind.rs +++ b/bin/reth/src/commands/stage/unwind.rs @@ -10,8 +10,8 @@ use reth_exex::ExExManagerHandle; use reth_node_core::args::NetworkArgs; use reth_primitives::{BlockHashOrNumber, ChainSpec, PruneModes, B256}; use reth_provider::{ - BlockExecutionWriter, BlockNumReader, ChainSpecProvider, HeaderSyncMode, ProviderFactory, - StaticFileProviderFactory, + providers::StaticFileProvider, BlockExecutionWriter, BlockNumReader, ChainSpecProvider, + HeaderSyncMode, ProviderFactory, StaticFileProviderFactory, }; use reth_stages::{ sets::DefaultStages, @@ -81,8 +81,11 @@ impl Command { let config: Config = confy::load_path(config_path).unwrap_or_default(); let db = Arc::new(open_db(db_path.as_ref(), self.db.database_args())?); - let provider_factory = - ProviderFactory::new(db, self.chain.clone(), data_dir.static_files())?; + let provider_factory = ProviderFactory::new( + db, + self.chain.clone(), + StaticFileProvider::read_write(data_dir.static_files())?, + ); let range = self.command.unwind_range(provider_factory.clone())?; if *range.start() == 0 { diff --git a/crates/blockchain-tree-api/src/error.rs b/crates/blockchain-tree-api/src/error.rs index ad9f7abcd3e8..ae9365aa6b06 100644 --- a/crates/blockchain-tree-api/src/error.rs +++ b/crates/blockchain-tree-api/src/error.rs @@ -296,7 +296,6 @@ impl InsertBlockErrorKind { BlockExecutionError::CanonicalRevert { .. } | BlockExecutionError::CanonicalCommit { .. } | BlockExecutionError::AppendChainDoesntConnect { .. } | - BlockExecutionError::UnavailableForTest => false, BlockExecutionError::Other(_) => false, } } diff --git a/crates/config/src/config.rs b/crates/config/src/config.rs index b804f7f9aa17..65fc1c647ab8 100644 --- a/crates/config/src/config.rs +++ b/crates/config/src/config.rs @@ -327,6 +327,13 @@ impl Default for PruneConfig { } } +impl PruneConfig { + /// Returns whether there is any kind of receipt pruning configuration. + pub fn has_receipts_pruning(&self) -> bool { + self.segments.receipts.is_some() || !self.segments.receipts_log_filter.is_empty() + } +} + /// Helper type to support older versions of Duration deserialization. fn deserialize_duration<'de, D>(deserializer: D) -> Result, D::Error> where diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index 136e59aef552..40ab8447b4ea 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -2179,6 +2179,7 @@ mod tests { use super::*; use reth_db::{tables, test_utils::create_test_static_files_dir, transaction::DbTxMut}; use reth_primitives::U256; + use reth_provider::providers::StaticFileProvider; use reth_rpc_types::engine::ForkchoiceUpdateError; use reth_testing_utils::generators::random_block; @@ -2235,8 +2236,11 @@ mod tests { let (_static_dir, static_dir_path) = create_test_static_files_dir(); insert_blocks( - ProviderFactory::new(env.db.as_ref(), chain_spec.clone(), static_dir_path) - .expect("create provider factory with static_files"), + ProviderFactory::new( + env.db.as_ref(), + chain_spec.clone(), + StaticFileProvider::read_write(static_dir_path).unwrap(), + ), [&genesis, &block1].into_iter(), ); env.db @@ -2292,8 +2296,11 @@ mod tests { let (_static_dir, static_dir_path) = create_test_static_files_dir(); insert_blocks( - ProviderFactory::new(env.db.as_ref(), chain_spec.clone(), static_dir_path) - .expect("create provider factory with static_files"), + ProviderFactory::new( + env.db.as_ref(), + chain_spec.clone(), + StaticFileProvider::read_write(static_dir_path).unwrap(), + ), [&genesis, &block1].into_iter(), ); @@ -2313,8 +2320,11 @@ mod tests { // Insert next head immediately after sending forkchoice update insert_blocks( - ProviderFactory::new(env.db.as_ref(), chain_spec.clone(), static_dir_path) - .expect("create provider factory with static_files"), + ProviderFactory::new( + env.db.as_ref(), + chain_spec.clone(), + StaticFileProvider::read_write(static_dir_path).unwrap(), + ), [&next_head].into_iter(), ); @@ -2354,8 +2364,11 @@ mod tests { let (_static_dir, static_dir_path) = create_test_static_files_dir(); insert_blocks( - ProviderFactory::new(env.db.as_ref(), chain_spec.clone(), static_dir_path) - .expect("create provider factory with static_files"), + ProviderFactory::new( + env.db.as_ref(), + chain_spec.clone(), + StaticFileProvider::read_write(static_dir_path).unwrap(), + ), [&genesis, &block1].into_iter(), ); @@ -2406,8 +2419,11 @@ mod tests { let (_static_dir, static_dir_path) = create_test_static_files_dir(); insert_blocks( - ProviderFactory::new(env.db.as_ref(), chain_spec.clone(), static_dir_path) - .expect("create provider factory with static_files"), + ProviderFactory::new( + env.db.as_ref(), + chain_spec.clone(), + StaticFileProvider::read_write(static_dir_path).unwrap(), + ), [&genesis, &block1, &block2, &block3].into_iter(), ); @@ -2452,8 +2468,11 @@ mod tests { let (_temp_dir, temp_dir_path) = create_test_static_files_dir(); insert_blocks( - ProviderFactory::new(env.db.as_ref(), chain_spec.clone(), temp_dir_path) - .expect("create provider factory with static_files"), + ProviderFactory::new( + env.db.as_ref(), + chain_spec.clone(), + StaticFileProvider::read_write(temp_dir_path).unwrap(), + ), [&genesis, &block1].into_iter(), ); @@ -2479,9 +2498,10 @@ mod tests { use super::*; use reth_db::test_utils::create_test_static_files_dir; use reth_primitives::{genesis::Genesis, Hardfork, U256}; - use reth_provider::test_utils::blocks::BlockchainTestData; + use reth_provider::{ + providers::StaticFileProvider, test_utils::blocks::BlockchainTestData, + }; use reth_testing_utils::{generators::random_block, GenesisAllocator}; - #[tokio::test] async fn new_payload_before_forkchoice() { let mut rng = generators::rng(); @@ -2551,8 +2571,11 @@ mod tests { let (_static_dir, static_dir_path) = create_test_static_files_dir(); insert_blocks( - ProviderFactory::new(env.db.as_ref(), chain_spec.clone(), static_dir_path) - .expect("create provider factory with static_files"), + ProviderFactory::new( + env.db.as_ref(), + chain_spec.clone(), + StaticFileProvider::read_write(static_dir_path).unwrap(), + ), [&genesis, &block1, &block2].into_iter(), ); @@ -2620,8 +2643,11 @@ mod tests { let (_static_dir, static_dir_path) = create_test_static_files_dir(); insert_blocks( - ProviderFactory::new(env.db.as_ref(), chain_spec.clone(), static_dir_path) - .expect("create provider factory with static_files"), + ProviderFactory::new( + env.db.as_ref(), + chain_spec.clone(), + StaticFileProvider::read_write(static_dir_path).unwrap(), + ), [&genesis, &block1].into_iter(), ); @@ -2664,8 +2690,11 @@ mod tests { let (_static_dir, static_dir_path) = create_test_static_files_dir(); insert_blocks( - ProviderFactory::new(env.db.as_ref(), chain_spec.clone(), static_dir_path) - .expect("create provider factory with static_files"), + ProviderFactory::new( + env.db.as_ref(), + chain_spec.clone(), + StaticFileProvider::read_write(static_dir_path).unwrap(), + ), [&genesis].into_iter(), ); @@ -2728,8 +2757,11 @@ mod tests { let (_static_dir, static_dir_path) = create_test_static_files_dir(); insert_blocks( - ProviderFactory::new(env.db.as_ref(), chain_spec.clone(), static_dir_path) - .expect("create provider factory with static_files"), + ProviderFactory::new( + env.db.as_ref(), + chain_spec.clone(), + StaticFileProvider::read_write(static_dir_path).unwrap(), + ), [&data.genesis, &block1].into_iter(), ); diff --git a/crates/evm/execution-errors/src/lib.rs b/crates/evm/execution-errors/src/lib.rs index 4d954ba1e178..08db509d8e0c 100644 --- a/crates/evm/execution-errors/src/lib.rs +++ b/crates/evm/execution-errors/src/lib.rs @@ -131,11 +131,6 @@ pub enum BlockExecutionError { /// The fork on the other chain other_chain_fork: Box, }, - /// Only used for TestExecutor - /// - /// Note: this is not feature gated for convenience. - #[error("execution unavailable for tests")] - UnavailableForTest, /// Error when fetching latest block state. #[error(transparent)] LatestBlock(#[from] ProviderError), diff --git a/crates/evm/src/execute.rs b/crates/evm/src/execute.rs index 8409f5c3742d..beefdac7fc42 100644 --- a/crates/evm/src/execute.rs +++ b/crates/evm/src/execute.rs @@ -245,7 +245,7 @@ mod tests { type Error = BlockExecutionError; fn execute(self, _input: Self::Input<'_>) -> Result { - Err(BlockExecutionError::UnavailableForTest) + Err(BlockExecutionError::msg("execution unavailable for tests")) } } diff --git a/crates/evm/src/lib.rs b/crates/evm/src/lib.rs index 93e8035258cf..c7f90057f4b8 100644 --- a/crates/evm/src/lib.rs +++ b/crates/evm/src/lib.rs @@ -16,6 +16,7 @@ use revm_primitives::{BlockEnv, CfgEnvWithHandlerCfg, EnvWithHandlerCfg, SpecId, pub mod either; pub mod execute; +pub mod noop; pub mod provider; #[cfg(any(test, feature = "test-utils"))] diff --git a/crates/evm/src/noop.rs b/crates/evm/src/noop.rs new file mode 100644 index 000000000000..220bbb39a84b --- /dev/null +++ b/crates/evm/src/noop.rs @@ -0,0 +1,68 @@ +//! A no operation block executor implementation. + +use reth_execution_errors::BlockExecutionError; +use reth_primitives::{BlockNumber, BlockWithSenders, PruneModes, Receipt}; +use reth_storage_errors::provider::ProviderError; +use revm_primitives::db::Database; + +use crate::execute::{ + BatchBlockExecutionOutput, BatchExecutor, BlockExecutionInput, BlockExecutionOutput, + BlockExecutorProvider, Executor, +}; + +const UNAVAILABLE_FOR_NOOP: &str = "execution unavailable for noop"; + +/// A [BlockExecutorProvider] implementation that does nothing. +#[derive(Debug, Default, Clone)] +#[non_exhaustive] +pub struct NoopBlockExecutorProvider; + +impl BlockExecutorProvider for NoopBlockExecutorProvider { + type Executor> = Self; + + type BatchExecutor> = Self; + + fn executor(&self, _: DB) -> Self::Executor + where + DB: Database, + { + Self + } + + fn batch_executor(&self, _: DB, _: PruneModes) -> Self::BatchExecutor + where + DB: Database, + { + Self + } +} + +impl Executor for NoopBlockExecutorProvider { + type Input<'a> = BlockExecutionInput<'a, BlockWithSenders>; + type Output = BlockExecutionOutput; + type Error = BlockExecutionError; + + fn execute(self, _: Self::Input<'_>) -> Result { + Err(BlockExecutionError::msg(UNAVAILABLE_FOR_NOOP)) + } +} + +impl BatchExecutor for NoopBlockExecutorProvider { + type Input<'a> = BlockExecutionInput<'a, BlockWithSenders>; + type Output = BatchBlockExecutionOutput; + type Error = BlockExecutionError; + + fn execute_and_verify_one(&mut self, _: Self::Input<'_>) -> Result<(), Self::Error> { + Err(BlockExecutionError::msg(UNAVAILABLE_FOR_NOOP)) + } + + fn finalize(self) -> Self::Output { + unreachable!() + } + + fn set_tip(&mut self, _: BlockNumber) {} + + fn size_hint(&self) -> Option { + None + } +} diff --git a/crates/net/downloaders/src/bodies/bodies.rs b/crates/net/downloaders/src/bodies/bodies.rs index e5042691ed6f..1f7a78d31092 100644 --- a/crates/net/downloaders/src/bodies/bodies.rs +++ b/crates/net/downloaders/src/bodies/bodies.rs @@ -607,7 +607,7 @@ mod tests { use reth_consensus::test_utils::TestConsensus; use reth_db::test_utils::{create_test_rw_db, create_test_static_files_dir}; use reth_primitives::{BlockBody, B256, MAINNET}; - use reth_provider::ProviderFactory; + use reth_provider::{providers::StaticFileProvider, ProviderFactory}; use reth_testing_utils::{generators, generators::random_block_range}; use std::collections::HashMap; @@ -629,7 +629,11 @@ mod tests { let mut downloader = BodiesDownloaderBuilder::default().build( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::new(db, MAINNET.clone(), static_dir_path).unwrap(), + ProviderFactory::new( + db, + MAINNET.clone(), + StaticFileProvider::read_write(static_dir_path).unwrap(), + ), ); downloader.set_download_range(0..=19).expect("failed to set download range"); @@ -675,7 +679,11 @@ mod tests { BodiesDownloaderBuilder::default().with_request_limit(request_limit).build( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::new(db, MAINNET.clone(), static_dir_path).unwrap(), + ProviderFactory::new( + db, + MAINNET.clone(), + StaticFileProvider::read_write(static_dir_path).unwrap(), + ), ); downloader.set_download_range(0..=199).expect("failed to set download range"); @@ -705,7 +713,11 @@ mod tests { .build( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::new(db, MAINNET.clone(), static_dir_path).unwrap(), + ProviderFactory::new( + db, + MAINNET.clone(), + StaticFileProvider::read_write(static_dir_path).unwrap(), + ), ); let mut range_start = 0; @@ -737,7 +749,11 @@ mod tests { let mut downloader = BodiesDownloaderBuilder::default().with_stream_batch_size(100).build( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::new(db, MAINNET.clone(), static_dir_path).unwrap(), + ProviderFactory::new( + db, + MAINNET.clone(), + StaticFileProvider::read_write(static_dir_path).unwrap(), + ), ); // Set and download the first range @@ -779,7 +795,11 @@ mod tests { .build( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::new(db, MAINNET.clone(), static_dir_path).unwrap(), + ProviderFactory::new( + db, + MAINNET.clone(), + StaticFileProvider::read_write(static_dir_path).unwrap(), + ), ); // Set and download the entire range @@ -812,7 +832,11 @@ mod tests { .build( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::new(db, MAINNET.clone(), static_dir_path).unwrap(), + ProviderFactory::new( + db, + MAINNET.clone(), + StaticFileProvider::read_write(static_dir_path).unwrap(), + ), ); // Download the requested range diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index a37f2fd8c730..1ec101925180 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -3,24 +3,32 @@ use eyre::Context; use rayon::ThreadPoolBuilder; use reth_auto_seal_consensus::MiningMode; +use reth_beacon_consensus::EthBeaconConsensus; use reth_config::{config::EtlConfig, PruneConfig}; use reth_db::{database::Database, database_metrics::DatabaseMetrics}; use reth_db_common::init::{init_genesis, InitDatabaseError}; +use reth_downloaders::{bodies::noop::NoopBodiesDownloader, headers::noop::NoopHeaderDownloader}; +use reth_evm::noop::NoopBlockExecutorProvider; use reth_network_p2p::headers::client::HeadersClient; use reth_node_core::{ cli::config::RethRpcConfig, dirs::{ChainPath, DataDirPath}, node_config::NodeConfig, }; -use reth_primitives::{BlockNumber, Chain, ChainSpec, Head, PruneModes, B256}; -use reth_provider::{providers::StaticFileProvider, ProviderFactory, StaticFileProviderFactory}; +use reth_primitives::{ + stage::PipelineTarget, BlockNumber, Chain, ChainSpec, Head, PruneModes, B256, +}; +use reth_provider::{ + providers::StaticFileProvider, HeaderSyncMode, ProviderFactory, StaticFileProviderFactory, +}; use reth_prune::PrunerBuilder; use reth_rpc_layer::JwtSecret; +use reth_stages::{sets::DefaultStages, Pipeline}; use reth_static_file::StaticFileProducer; use reth_tasks::TaskExecutor; use reth_tracing::tracing::{debug, error, info, warn}; use std::{sync::Arc, thread::available_parallelism}; -use tokio::sync::mpsc::Receiver; +use tokio::sync::{mpsc::Receiver, oneshot}; /// Reusable setup for launching a node. /// @@ -319,25 +327,81 @@ impl LaunchContextWith> { impl LaunchContextWith> where - DB: Clone, + DB: Database + Clone + 'static, { - /// Returns the [ProviderFactory] for the attached database. - pub fn create_provider_factory(&self) -> eyre::Result> { + /// Returns the [ProviderFactory] for the attached storage after executing a consistent check + /// between the database and static files. **It may execute a pipeline unwind if it fails this + /// check.** + pub async fn create_provider_factory(&self) -> eyre::Result> { let factory = ProviderFactory::new( self.right().clone(), self.chain_spec(), - self.data_dir().static_files(), - )? + StaticFileProvider::read_write(self.data_dir().static_files())?, + ) .with_static_files_metrics(); + let has_receipt_pruning = + self.toml_config().prune.as_ref().map_or(false, |a| a.has_receipts_pruning()); + + info!(target: "reth::cli", "Verifying storage consistency."); + + // Check for consistency between database and static files. If it fails, it unwinds to + // the first block that's consistent between database and static files. + if let Some(unwind_target) = factory + .static_file_provider() + .check_consistency(&factory.provider()?, has_receipt_pruning)? + { + // Highly unlikely to happen, and given its destructive nature, it's better to panic + // instead. + if PipelineTarget::Unwind(0) == unwind_target { + panic!("A static file <> database inconsistency was found that would trigger an unwind to block 0.") + } + + info!(target: "reth::cli", unwind_target = %unwind_target, "Executing an unwind after a failed storage consistency check."); + + // Builds an unwind-only pipeline + let pipeline = Pipeline::builder() + .add_stages(DefaultStages::new( + factory.clone(), + HeaderSyncMode::Continuous, + Arc::new(EthBeaconConsensus::new(self.chain_spec())), + NoopHeaderDownloader::default(), + NoopBodiesDownloader::default(), + NoopBlockExecutorProvider::default(), + self.toml_config().stages.clone(), + self.prune_modes().unwrap_or_default(), + )) + .build( + factory.clone(), + StaticFileProducer::new( + factory.clone(), + factory.static_file_provider(), + self.prune_modes().unwrap_or_default(), + ), + ); + + // Unwinds to block + let (tx, rx) = oneshot::channel(); + + // Pipeline should be run as blocking and panic if it fails. + self.task_executor().spawn_critical_blocking( + "pipeline task", + Box::pin(async move { + let (_, result) = pipeline.run_as_fut(Some(unwind_target)).await; + let _ = tx.send(result); + }), + ); + rx.await??; + } + Ok(factory) } /// Creates a new [ProviderFactory] and attaches it to the launch context. - pub fn with_provider_factory( + pub async fn with_provider_factory( self, ) -> eyre::Result>>> { - let factory = self.create_provider_factory()?; + let factory = self.create_provider_factory().await?; let ctx = LaunchContextWith { inner: self.inner, attachment: self.attachment.map_right(|_| factory), diff --git a/crates/node/builder/src/launch/mod.rs b/crates/node/builder/src/launch/mod.rs index 2df6e7824962..8146461d6974 100644 --- a/crates/node/builder/src/launch/mod.rs +++ b/crates/node/builder/src/launch/mod.rs @@ -100,7 +100,7 @@ where // ensure certain settings take effect .with_adjusted_configs() // Create the provider factory - .with_provider_factory()? + .with_provider_factory().await? .inspect(|_| { info!(target: "reth::cli", "Database opened"); }) diff --git a/crates/primitives/src/stage/mod.rs b/crates/primitives/src/stage/mod.rs index 74f84409b10e..6637cb41e8fa 100644 --- a/crates/primitives/src/stage/mod.rs +++ b/crates/primitives/src/stage/mod.rs @@ -12,7 +12,7 @@ pub use checkpoints::{ }; /// Direction and target block for pipeline operations. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PipelineTarget { /// Target for forward synchronization, indicating a block hash to sync to. Sync(BlockHash), @@ -53,3 +53,14 @@ impl From for PipelineTarget { Self::Sync(hash) } } + +impl std::fmt::Display for PipelineTarget { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Sync(block) => { + write!(f, "Sync({block})") + } + Self::Unwind(block) => write!(f, "Unwind({block})"), + } + } +} diff --git a/crates/prune/src/pruner.rs b/crates/prune/src/pruner.rs index f4111f131a50..c6e0fffae8eb 100644 --- a/crates/prune/src/pruner.rs +++ b/crates/prune/src/pruner.rs @@ -335,14 +335,17 @@ mod tests { use crate::Pruner; use reth_db::test_utils::{create_test_rw_db, create_test_static_files_dir}; use reth_primitives::{FinishedExExHeight, MAINNET}; - use reth_provider::ProviderFactory; + use reth_provider::{providers::StaticFileProvider, ProviderFactory}; #[test] fn is_pruning_needed() { let db = create_test_rw_db(); let (_static_dir, static_dir_path) = create_test_static_files_dir(); - let provider_factory = ProviderFactory::new(db, MAINNET.clone(), static_dir_path) - .expect("create provide factory with static_files"); + let provider_factory = ProviderFactory::new( + db, + MAINNET.clone(), + StaticFileProvider::read_write(static_dir_path).unwrap(), + ); let (finished_exex_height_tx, finished_exex_height_rx) = tokio::sync::watch::channel(FinishedExExHeight::NoExExs); diff --git a/crates/stages/api/src/pipeline/mod.rs b/crates/stages/api/src/pipeline/mod.rs index e8c0cefd736a..6b7b65b9b89f 100644 --- a/crates/stages/api/src/pipeline/mod.rs +++ b/crates/stages/api/src/pipeline/mod.rs @@ -316,7 +316,7 @@ where continue } - debug!( + info!( target: "sync::pipeline", from = %checkpoint.block_number, %to, @@ -353,8 +353,12 @@ where self.event_sender .notify(PipelineEvent::Unwound { stage_id, result: unwind_output }); - self.provider_factory.static_file_provider().commit()?; + // For unwinding it makes more sense to commit the database first, since if + // this function is interrupted before the static files commit, we can just + // truncate the static files according to the + // checkpoints on the next start-up. provider_rw.commit()?; + self.provider_factory.static_file_provider().commit()?; provider_rw = self.provider_factory.provider_rw()?; } @@ -459,6 +463,10 @@ where result: out.clone(), }); + // For execution it makes more sense to commit the static files first, since if + // this function is interrupted before the database commit, we can just truncate + // the static files according to the checkpoints on the next + // start-up. self.provider_factory.static_file_provider().commit()?; provider_rw.commit()?; diff --git a/crates/stages/stages/src/stages/execution.rs b/crates/stages/stages/src/stages/execution.rs index 2388d0d6ff9d..e76b210925de 100644 --- a/crates/stages/stages/src/stages/execution.rs +++ b/crates/stages/stages/src/stages/execution.rs @@ -606,7 +606,9 @@ where // Check if we had any unexpected shutdown after committing to static files, but // NOT committing to database. match next_static_file_receipt_num.cmp(&next_receipt_num) { - Ordering::Greater => static_file_producer.prune_receipts( + // It can be equal when it's a chain of empty blocks, but we still need to update the last + // block in the range. + Ordering::Greater | Ordering::Equal => static_file_producer.prune_receipts( next_static_file_receipt_num - next_receipt_num, start_block.saturating_sub(1), )?, @@ -641,7 +643,6 @@ where segment: StaticFileSegment::Receipts, }) } - Ordering::Equal => {} } Ok(static_file_producer) diff --git a/crates/stages/stages/src/stages/mod.rs b/crates/stages/stages/src/stages/mod.rs index 9af26247348e..1ecc8ac1e3fc 100644 --- a/crates/stages/stages/src/stages/mod.rs +++ b/crates/stages/stages/src/stages/mod.rs @@ -40,11 +40,12 @@ use utils::*; #[cfg(test)] mod tests { use super::*; - use crate::test_utils::TestStageDB; + use crate::test_utils::{StorageKind, TestStageDB}; use alloy_rlp::Decodable; use reth_db::{ - cursor::DbCursorRO, + cursor::{DbCursorRO, DbCursorRW}, mdbx::{cursor::Cursor, RW}, + table::Table, tables, test_utils::TempDatabase, transaction::{DbTx, DbTxMut}, @@ -53,16 +54,21 @@ mod tests { use reth_evm_ethereum::execute::EthExecutorProvider; use reth_exex::ExExManagerHandle; use reth_primitives::{ - address, hex_literal::hex, keccak256, Account, Bytecode, ChainSpecBuilder, PruneMode, - PruneModes, SealedBlock, StaticFileSegment, U256, + address, + hex_literal::hex, + keccak256, + stage::{PipelineTarget, StageCheckpoint, StageId}, + Account, BlockNumber, Bytecode, ChainSpecBuilder, PruneMode, PruneModes, SealedBlock, + StaticFileSegment, B256, U256, }; use reth_provider::{ - providers::StaticFileWriter, AccountExtReader, ProviderFactory, ReceiptProvider, - StorageReader, + providers::StaticFileWriter, AccountExtReader, BlockReader, DatabaseProviderFactory, + ProviderFactory, ProviderResult, ReceiptProvider, StageCheckpointWriter, + StaticFileProviderFactory, StorageReader, }; use reth_stages_api::{ExecInput, Stage}; - use reth_testing_utils::generators::{self, random_block}; - use std::sync::Arc; + use reth_testing_utils::generators::{self, random_block, random_block_range, random_receipt}; + use std::{io::Write, sync::Arc}; #[tokio::test] #[ignore] @@ -239,4 +245,252 @@ mod tests { // The one account is the miner check_pruning(test_db.factory.clone(), prune.clone(), 0, 1, 0).await; } + + /// It will generate `num_blocks`, push them to static files and set all stage checkpoints to + /// `num_blocks - 1`. + fn seed_data(num_blocks: usize) -> ProviderResult { + let db = TestStageDB::default(); + let mut rng = generators::rng(); + let genesis_hash = B256::ZERO; + let tip = (num_blocks - 1) as u64; + + let blocks = random_block_range(&mut rng, 0..=tip, genesis_hash, 2..3); + db.insert_blocks(blocks.iter(), StorageKind::Static)?; + + let mut receipts = Vec::new(); + let mut tx_num = 0u64; + for block in &blocks { + let mut block_receipts = Vec::with_capacity(block.body.len()); + for transaction in &block.body { + block_receipts.push((tx_num, random_receipt(&mut rng, transaction, Some(0)))); + tx_num += 1; + } + receipts.push((block.number, block_receipts)); + } + db.insert_receipts_by_block(receipts, StorageKind::Static)?; + + // simulate pipeline by setting all checkpoints to inserted height. + let provider_rw = db.factory.provider_rw()?; + for stage in StageId::ALL { + provider_rw.save_stage_checkpoint(stage, StageCheckpoint::new(tip))?; + } + provider_rw.commit()?; + + Ok(db) + } + + /// Simulates losing data to corruption and compare the check consistency result + /// against the expected one. + fn simulate_behind_checkpoint_corruption( + db: &TestStageDB, + prune_count: usize, + segment: StaticFileSegment, + is_full_node: bool, + expected: Option, + ) { + let static_file_provider = db.factory.static_file_provider(); + + // Simulate corruption by removing `prune_count` rows from the data file without updating + // its offset list and configuration. + { + let mut headers_writer = static_file_provider.latest_writer(segment).unwrap(); + let reader = headers_writer.inner().jar().open_data_reader().unwrap(); + let columns = headers_writer.inner().jar().columns(); + let data_file = headers_writer.inner().data_file(); + let last_offset = reader.reverse_offset(prune_count * columns).unwrap(); + data_file.get_mut().set_len(last_offset).unwrap(); + data_file.flush().unwrap(); + data_file.get_ref().sync_all().unwrap(); + } + + assert_eq!( + static_file_provider + .check_consistency(&db.factory.database_provider_ro().unwrap(), is_full_node,), + Ok(expected) + ); + } + + /// Saves a checkpoint with `checkpoint_block_number` and compare the check consistency result + /// against the expected one. + fn save_checkpoint_and_check( + db: &TestStageDB, + stage_id: StageId, + checkpoint_block_number: BlockNumber, + expected: Option, + ) { + let provider_rw = db.factory.provider_rw().unwrap(); + provider_rw + .save_stage_checkpoint(stage_id, StageCheckpoint::new(checkpoint_block_number)) + .unwrap(); + provider_rw.commit().unwrap(); + + assert_eq!( + db.factory + .static_file_provider() + .check_consistency(&db.factory.database_provider_ro().unwrap(), false,), + Ok(expected) + ); + } + + /// Inserts a dummy value at key and compare the check consistency result against the expected + /// one. + fn update_db_and_check>( + db: &TestStageDB, + key: u64, + expected: Option, + ) where + ::Value: Default, + { + let provider_rw = db.factory.provider_rw().unwrap(); + let mut cursor = provider_rw.tx_ref().cursor_write::().unwrap(); + cursor.insert(key, Default::default()).unwrap(); + provider_rw.commit().unwrap(); + + assert_eq!( + db.factory + .static_file_provider() + .check_consistency(&db.factory.database_provider_ro().unwrap(), false), + Ok(expected) + ); + } + + #[test] + fn test_consistency() { + let db = seed_data(90).unwrap(); + let db_provider = db.factory.database_provider_ro().unwrap(); + + assert_eq!( + db.factory.static_file_provider().check_consistency(&db_provider, false), + Ok(None) + ); + } + + #[test] + fn test_consistency_no_commit_prune() { + let db = seed_data(90).unwrap(); + let full_node = true; + let archive_node = !full_node; + + // Full node does not use receipts, therefore doesn't check for consistency on receipts + // segment + simulate_behind_checkpoint_corruption(&db, 1, StaticFileSegment::Receipts, full_node, None); + + // there are 2 to 3 transactions per block. however, if we lose one tx, we need to unwind to + // the previous block. + simulate_behind_checkpoint_corruption( + &db, + 1, + StaticFileSegment::Receipts, + archive_node, + Some(PipelineTarget::Unwind(88)), + ); + + simulate_behind_checkpoint_corruption( + &db, + 3, + StaticFileSegment::Headers, + archive_node, + Some(PipelineTarget::Unwind(86)), + ); + } + + #[test] + fn test_consistency_checkpoints() { + let db = seed_data(90).unwrap(); + + // When a checkpoint is behind, we delete data from static files. + let block = 87; + save_checkpoint_and_check(&db, StageId::Bodies, block, None); + assert_eq!( + db.factory + .static_file_provider() + .get_highest_static_file_block(StaticFileSegment::Transactions), + Some(block) + ); + assert_eq!( + db.factory + .static_file_provider() + .get_highest_static_file_tx(StaticFileSegment::Transactions), + db.factory.block_body_indices(block).unwrap().map(|b| b.last_tx_num()) + ); + + let block = 86; + save_checkpoint_and_check(&db, StageId::Execution, block, None); + assert_eq!( + db.factory + .static_file_provider() + .get_highest_static_file_block(StaticFileSegment::Receipts), + Some(block) + ); + assert_eq!( + db.factory + .static_file_provider() + .get_highest_static_file_tx(StaticFileSegment::Receipts), + db.factory.block_body_indices(block).unwrap().map(|b| b.last_tx_num()) + ); + + let block = 80; + save_checkpoint_and_check(&db, StageId::Headers, block, None); + assert_eq!( + db.factory + .static_file_provider() + .get_highest_static_file_block(StaticFileSegment::Headers), + Some(block) + ); + + // When a checkpoint is ahead, we request a pipeline unwind. + save_checkpoint_and_check(&db, StageId::Headers, 91, Some(PipelineTarget::Unwind(block))); + } + + #[test] + fn test_consistency_headers_gap() { + let db = seed_data(90).unwrap(); + let current = db + .factory + .static_file_provider() + .get_highest_static_file_block(StaticFileSegment::Headers) + .unwrap(); + + // Creates a gap of one header: static_file db + update_db_and_check::(&db, current + 2, Some(PipelineTarget::Unwind(89))); + + // Fill the gap, and ensure no unwind is necessary. + update_db_and_check::(&db, current + 1, None); + } + + #[test] + fn test_consistency_tx_gap() { + let db = seed_data(90).unwrap(); + let current = db + .factory + .static_file_provider() + .get_highest_static_file_tx(StaticFileSegment::Transactions) + .unwrap(); + + // Creates a gap of one transaction: static_file db + update_db_and_check::( + &db, + current + 2, + Some(PipelineTarget::Unwind(89)), + ); + + // Fill the gap, and ensure no unwind is necessary. + update_db_and_check::(&db, current + 1, None); + } + + #[test] + fn test_consistency_receipt_gap() { + let db = seed_data(90).unwrap(); + let current = db + .factory + .static_file_provider() + .get_highest_static_file_tx(StaticFileSegment::Receipts) + .unwrap(); + + // Creates a gap of one receipt: static_file db + update_db_and_check::(&db, current + 2, Some(PipelineTarget::Unwind(89))); + + // Fill the gap, and ensure no unwind is necessary. + update_db_and_check::(&db, current + 1, None); + } } diff --git a/crates/stages/stages/src/test_utils/test_db.rs b/crates/stages/stages/src/test_utils/test_db.rs index 234aace74413..c92ccc15bf32 100644 --- a/crates/stages/stages/src/test_utils/test_db.rs +++ b/crates/stages/stages/src/test_utils/test_db.rs @@ -16,7 +16,7 @@ use reth_primitives::{ StaticFileSegment, StorageEntry, TxHash, TxNumber, B256, MAINNET, U256, }; use reth_provider::{ - providers::{StaticFileProviderRWRefMut, StaticFileWriter}, + providers::{StaticFileProvider, StaticFileProviderRWRefMut, StaticFileWriter}, HistoryWriter, ProviderError, ProviderFactory, StaticFileProviderFactory, }; use reth_storage_errors::provider::ProviderResult; @@ -37,8 +37,11 @@ impl Default for TestStageDB { let (static_dir, static_dir_path) = create_test_static_files_dir(); Self { temp_static_files_dir: static_dir, - factory: ProviderFactory::new(create_test_rw_db(), MAINNET.clone(), static_dir_path) - .unwrap(), + factory: ProviderFactory::new( + create_test_rw_db(), + MAINNET.clone(), + StaticFileProvider::read_write(static_dir_path).unwrap(), + ), } } } @@ -52,9 +55,8 @@ impl TestStageDB { factory: ProviderFactory::new( create_test_rw_db_with_path(path), MAINNET.clone(), - static_dir_path, - ) - .unwrap(), + StaticFileProvider::read_write(static_dir_path).unwrap(), + ), } } @@ -225,13 +227,17 @@ impl TestStageDB { let blocks = blocks.into_iter().collect::>(); { - let mut headers_writer = provider.latest_writer(StaticFileSegment::Headers)?; + let mut headers_writer = storage_kind + .is_static() + .then(|| provider.latest_writer(StaticFileSegment::Headers).unwrap()); blocks.iter().try_for_each(|block| { - Self::insert_header(Some(&mut headers_writer), &tx, &block.header, U256::ZERO) + Self::insert_header(headers_writer.as_mut(), &tx, &block.header, U256::ZERO) })?; - headers_writer.commit()?; + if let Some(mut writer) = headers_writer { + writer.commit()?; + } } { @@ -315,6 +321,42 @@ impl TestStageDB { }) } + /// Insert collection of ([TxNumber], [Receipt]) organized by respective block numbers into the + /// corresponding table or static file segment. + pub fn insert_receipts_by_block( + &self, + receipts: I, + storage_kind: StorageKind, + ) -> ProviderResult<()> + where + I: IntoIterator, + J: IntoIterator, + { + match storage_kind { + StorageKind::Database(_) => self.commit(|tx| { + receipts.into_iter().try_for_each(|(_, receipts)| { + for (tx_num, receipt) in receipts { + tx.put::(tx_num, receipt)?; + } + Ok(()) + }) + }), + StorageKind::Static => { + let provider = self.factory.static_file_provider(); + let mut writer = provider.latest_writer(StaticFileSegment::Receipts)?; + let res = receipts.into_iter().try_for_each(|(block_num, receipts)| { + writer.increment_block(StaticFileSegment::Receipts, block_num)?; + for (tx_num, receipt) in receipts { + writer.append_receipt(tx_num, receipt)?; + } + Ok(()) + }); + writer.commit_without_sync_all()?; + res + } + } + } + pub fn insert_transaction_senders(&self, transaction_senders: I) -> ProviderResult<()> where I: IntoIterator, diff --git a/crates/static-file-types/src/segment.rs b/crates/static-file-types/src/segment.rs index a69fab525c18..666d47258ad1 100644 --- a/crates/static-file-types/src/segment.rs +++ b/crates/static-file-types/src/segment.rs @@ -136,6 +136,11 @@ impl StaticFileSegment { pub const fn is_headers(&self) -> bool { matches!(self, Self::Headers) } + + /// Returns `true` if the segment is `StaticFileSegment::Receipts`. + pub const fn is_receipts(&self) -> bool { + matches!(self, Self::Receipts) + } } /// A segment header that contains information common to all segments. Used for storage. diff --git a/crates/storage/db-common/src/init.rs b/crates/storage/db-common/src/init.rs index 27728313b105..b193f04583d1 100644 --- a/crates/storage/db-common/src/init.rs +++ b/crates/storage/db-common/src/init.rs @@ -579,14 +579,11 @@ mod tests { init_genesis(factory.clone()).unwrap(); // Try to init db with a different genesis block - let genesis_hash = init_genesis( - ProviderFactory::new( - factory.into_db(), - MAINNET.clone(), - static_file_provider.path().into(), - ) - .unwrap(), - ); + let genesis_hash = init_genesis(ProviderFactory::new( + factory.into_db(), + MAINNET.clone(), + StaticFileProvider::read_write(static_file_provider.path()).unwrap(), + )); assert_eq!( genesis_hash.unwrap_err(), diff --git a/crates/storage/errors/src/provider.rs b/crates/storage/errors/src/provider.rs index aeecf64a7b41..5420b8f3c6cb 100644 --- a/crates/storage/errors/src/provider.rs +++ b/crates/storage/errors/src/provider.rs @@ -122,6 +122,9 @@ pub enum ProviderError { /// Trying to insert data from an unexpected block number. #[error("trying to append data to {0} as block #{1} but expected block #{2}")] UnexpectedStaticFileBlockNumber(StaticFileSegment, BlockNumber, BlockNumber), + /// Static File Provider was initialized as read-only. + #[error("cannot get a writer on a read-only environment.")] + ReadOnlyStaticFileAccess, /// Error encountered when the block number conversion from U256 to u64 causes an overflow. #[error("failed to convert block number U256 to u64: {0}")] BlockNumberOverflow(U256), diff --git a/crates/storage/nippy-jar/src/error.rs b/crates/storage/nippy-jar/src/error.rs index d59500842c7a..58e27a76b4c5 100644 --- a/crates/storage/nippy-jar/src/error.rs +++ b/crates/storage/nippy-jar/src/error.rs @@ -57,4 +57,6 @@ pub enum NippyJarError { InvalidPruning(u64, u64), #[error("jar has been frozen and cannot be modified.")] FrozenJar, + #[error("File is in an inconsistent state.")] + InconsistentState, } diff --git a/crates/storage/nippy-jar/src/lib.rs b/crates/storage/nippy-jar/src/lib.rs index 3258ebd10595..b1da4429355a 100644 --- a/crates/storage/nippy-jar/src/lib.rs +++ b/crates/storage/nippy-jar/src/lib.rs @@ -41,7 +41,7 @@ mod cursor; pub use cursor::NippyJarCursor; mod writer; -pub use writer::NippyJarWriter; +pub use writer::{ConsistencyFailStrategy, NippyJarWriter}; const NIPPY_JAR_VERSION: usize = 1; @@ -382,7 +382,7 @@ impl NippyJar { self.freeze_filters()?; // Creates the writer, data and offsets file - let mut writer = NippyJarWriter::new(self)?; + let mut writer = NippyJarWriter::new(self, ConsistencyFailStrategy::Heal)?; // Append rows to file while holding offsets in memory writer.append_rows(columns, total_rows)?; @@ -1114,7 +1114,7 @@ mod tests { assert!(initial_offset_size > 0); // Appends a third row - let mut writer = NippyJarWriter::new(nippy).unwrap(); + let mut writer = NippyJarWriter::new(nippy, ConsistencyFailStrategy::Heal).unwrap(); writer.append_column(Some(Ok(&col1[2]))).unwrap(); writer.append_column(Some(Ok(&col2[2]))).unwrap(); @@ -1145,7 +1145,7 @@ mod tests { // Writer will execute a consistency check and verify first that the offset list on disk // doesn't match the nippy.rows, and prune it. Then, it will prune the data file // accordingly as well. - let writer = NippyJarWriter::new(nippy).unwrap(); + let writer = NippyJarWriter::new(nippy, ConsistencyFailStrategy::Heal).unwrap(); assert_eq!(initial_rows, writer.rows()); assert_eq!( initial_offset_size, @@ -1171,7 +1171,7 @@ mod tests { // Appends a third row, so we have an offset list in memory, which is not flushed to disk, // while the data has been. - let mut writer = NippyJarWriter::new(nippy).unwrap(); + let mut writer = NippyJarWriter::new(nippy, ConsistencyFailStrategy::Heal).unwrap(); writer.append_column(Some(Ok(&col1[2]))).unwrap(); writer.append_column(Some(Ok(&col2[2]))).unwrap(); @@ -1194,7 +1194,7 @@ mod tests { // Writer will execute a consistency check and verify that the data file has more data than // it should, and resets it to the last offset of the list (on disk here) - let writer = NippyJarWriter::new(nippy).unwrap(); + let writer = NippyJarWriter::new(nippy, ConsistencyFailStrategy::Heal).unwrap(); assert_eq!(initial_rows, writer.rows()); assert_eq!( initial_data_size, @@ -1210,7 +1210,7 @@ mod tests { assert_eq!(nippy.max_row_size, 0); assert_eq!(nippy.rows, 0); - let mut writer = NippyJarWriter::new(nippy).unwrap(); + let mut writer = NippyJarWriter::new(nippy, ConsistencyFailStrategy::Heal).unwrap(); assert_eq!(writer.column(), 0); writer.append_column(Some(Ok(&col1[0]))).unwrap(); @@ -1245,7 +1245,7 @@ mod tests { assert_eq!(nippy.max_row_size, col1[0].len() + col2[0].len()); assert_eq!(nippy.rows, 1); - let mut writer = NippyJarWriter::new(nippy).unwrap(); + let mut writer = NippyJarWriter::new(nippy, ConsistencyFailStrategy::Heal).unwrap(); assert_eq!(writer.column(), 0); writer.append_column(Some(Ok(&col1[1]))).unwrap(); @@ -1276,7 +1276,7 @@ mod tests { fn prune_rows(num_columns: usize, file_path: &Path, col1: &[Vec], col2: &[Vec]) { let nippy = NippyJar::load_without_header(file_path).unwrap(); - let mut writer = NippyJarWriter::new(nippy).unwrap(); + let mut writer = NippyJarWriter::new(nippy, ConsistencyFailStrategy::Heal).unwrap(); // Appends a third row, so we have an offset list in memory, which is not flushed to disk writer.append_column(Some(Ok(&col1[2]))).unwrap(); @@ -1306,7 +1306,7 @@ mod tests { } // This should prune from the ondisk offset list and clear the jar. - let mut writer = NippyJarWriter::new(nippy).unwrap(); + let mut writer = NippyJarWriter::new(nippy, ConsistencyFailStrategy::Heal).unwrap(); writer.prune_rows(1).unwrap(); assert_eq!(writer.rows(), 0); assert_eq!(writer.max_row_size(), 0); @@ -1343,6 +1343,6 @@ mod tests { data_file.set_len(data_len - 32 * missing_offsets).unwrap(); // runs the consistency check. - let _ = NippyJarWriter::new(nippy).unwrap(); + let _ = NippyJarWriter::new(nippy, ConsistencyFailStrategy::Heal).unwrap(); } } diff --git a/crates/storage/nippy-jar/src/writer.rs b/crates/storage/nippy-jar/src/writer.rs index 8bd47b1b4113..5c405856d132 100644 --- a/crates/storage/nippy-jar/src/writer.rs +++ b/crates/storage/nippy-jar/src/writer.rs @@ -43,7 +43,13 @@ pub struct NippyJarWriter { impl NippyJarWriter { /// Creates a [`NippyJarWriter`] from [`NippyJar`]. - pub fn new(jar: NippyJar) -> Result { + /// + /// If `read_only` is set to `true`, any inconsistency issue won't be healed, and will return + /// [NippyJarError::InconsistentState] instead. + pub fn new( + jar: NippyJar, + check_mode: ConsistencyFailStrategy, + ) -> Result { let (data_file, offsets_file, is_created) = Self::create_or_open_files(jar.data_path(), &jar.offsets_path())?; @@ -63,8 +69,10 @@ impl NippyJarWriter { // If we are opening a previously created jar, we need to check its consistency, and make // changes if necessary. if !is_created { - writer.check_consistency_and_heal()?; - writer.commit()?; + writer.ensure_file_consistency(check_mode)?; + if check_mode.should_heal() { + writer.commit()?; + } } Ok(writer) @@ -118,13 +126,17 @@ impl NippyJarWriter { Ok((data_file, offsets_file, is_created)) } - /// Performs consistency checks on the [`NippyJar`] file and acts upon any issues: + /// Performs consistency checks on the [`NippyJar`] file and might self-heal or throw an error + /// according to [ConsistencyFailStrategy]. /// * Is the offsets file size expected? /// * Is the data file size expected? /// /// This is based on the assumption that [`NippyJar`] configuration is **always** the last one /// to be updated when something is written, as by the `commit()` function shows. - fn check_consistency_and_heal(&mut self) -> Result<(), NippyJarError> { + pub fn ensure_file_consistency( + &mut self, + check_mode: ConsistencyFailStrategy, + ) -> Result<(), NippyJarError> { let reader = self.jar.open_data_reader()?; // When an offset size is smaller than the initial (8), we are dealing with immutable @@ -138,6 +150,12 @@ impl NippyJarWriter { OFFSET_SIZE_BYTES as usize) as u64; // expected size of the data file let actual_offsets_file_size = self.offsets_file.get_ref().metadata()?.len(); + if check_mode.should_err() && + expected_offsets_file_size.cmp(&actual_offsets_file_size) != Ordering::Equal + { + return Err(NippyJarError::InconsistentState) + } + // Offsets configuration wasn't properly committed match expected_offsets_file_size.cmp(&actual_offsets_file_size) { Ordering::Less => { @@ -165,6 +183,10 @@ impl NippyJarWriter { let last_offset = reader.reverse_offset(0)?; let data_file_len = self.data_file.get_ref().metadata()?.len(); + if check_mode.should_err() && last_offset.cmp(&data_file_len) != Ordering::Equal { + return Err(NippyJarError::InconsistentState) + } + // Offset list wasn't properly committed match last_offset.cmp(&data_file_len) { Ordering::Less => { @@ -177,7 +199,8 @@ impl NippyJarWriter { // find the matching one. for index in 0..reader.offsets_count()? { let offset = reader.reverse_offset(index + 1)?; - if offset == data_file_len { + // It would only be equal if the previous row was fully pruned. + if offset <= data_file_len { let new_len = self .offsets_file .get_ref() @@ -190,7 +213,7 @@ impl NippyJarWriter { // Since we decrease the offset list, we need to check the consistency of // `self.jar.rows` again - self.check_consistency_and_heal()?; + self.ensure_file_consistency(ConsistencyFailStrategy::Heal)?; break } } @@ -479,4 +502,35 @@ impl NippyJarWriter { pub fn data_path(&self) -> &Path { self.jar.data_path() } + + #[cfg(any(test, feature = "test-utils"))] + pub fn data_file(&mut self) -> &mut BufWriter { + &mut self.data_file + } + + #[cfg(any(test, feature = "test-utils"))] + pub const fn jar(&self) -> &NippyJar { + &self.jar + } +} + +/// Strategy on encountering an inconsistent state when creating a [NippyJarWriter]. +#[derive(Debug, Copy, Clone)] +pub enum ConsistencyFailStrategy { + /// Writer should heal. + Heal, + /// Writer should throw an error. + ThrowError, +} + +impl ConsistencyFailStrategy { + /// Whether writer should heal. + const fn should_heal(&self) -> bool { + matches!(self, Self::Heal) + } + + /// Whether writer should throw an error. + const fn should_err(&self) -> bool { + matches!(self, Self::ThrowError) + } } diff --git a/crates/storage/provider/src/lib.rs b/crates/storage/provider/src/lib.rs index 021372bb647e..17c6af875818 100644 --- a/crates/storage/provider/src/lib.rs +++ b/crates/storage/provider/src/lib.rs @@ -21,7 +21,7 @@ pub mod providers; pub use providers::{ DatabaseProvider, DatabaseProviderRO, DatabaseProviderRW, HistoricalStateProvider, HistoricalStateProviderRef, LatestStateProvider, LatestStateProviderRef, ProviderFactory, - StaticFileWriter, + StaticFileAccess, StaticFileWriter, }; #[cfg(any(test, feature = "test-utils"))] diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 8b83cc26a44a..08148e95e1f8 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -25,7 +25,7 @@ use reth_storage_errors::provider::ProviderResult; use revm::primitives::{BlockEnv, CfgEnvWithHandlerCfg}; use std::{ ops::{RangeBounds, RangeInclusive}, - path::{Path, PathBuf}, + path::Path, sync::Arc, }; use tracing::trace; @@ -53,13 +53,9 @@ impl ProviderFactory { pub fn new( db: DB, chain_spec: Arc, - static_files_path: PathBuf, - ) -> ProviderResult { - Ok(Self { - db: Arc::new(db), - chain_spec, - static_file_provider: StaticFileProvider::new(static_files_path)?, - }) + static_file_provider: StaticFileProvider, + ) -> Self { + Self { db: Arc::new(db), chain_spec, static_file_provider } } /// Enables metrics on the static file provider. @@ -87,12 +83,12 @@ impl ProviderFactory { path: P, chain_spec: Arc, args: DatabaseArguments, - static_files_path: PathBuf, + static_file_provider: StaticFileProvider, ) -> RethResult { Ok(Self { db: Arc::new(init_db(path, args).map_err(RethError::msg)?), chain_spec, - static_file_provider: StaticFileProvider::new(static_files_path)?, + static_file_provider, }) } } @@ -588,8 +584,10 @@ impl Clone for ProviderFactory { mod tests { use super::*; use crate::{ - providers::StaticFileWriter, test_utils::create_test_provider_factory, BlockHashReader, - BlockNumReader, BlockWriter, HeaderSyncGapProvider, HeaderSyncMode, TransactionsProvider, + providers::{StaticFileProvider, StaticFileWriter}, + test_utils::create_test_provider_factory, + BlockHashReader, BlockNumReader, BlockWriter, HeaderSyncGapProvider, HeaderSyncMode, + TransactionsProvider, }; use alloy_rlp::Decodable; use assert_matches::assert_matches; @@ -645,7 +643,7 @@ mod tests { tempfile::TempDir::new().expect(ERROR_TEMPDIR).into_path(), Arc::new(chain_spec), DatabaseArguments::new(Default::default()), - static_dir_path, + StaticFileProvider::read_write(static_dir_path).unwrap(), ) .unwrap(); diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index 9bbdcb26f1d1..9b72b3bd9740 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -40,8 +40,8 @@ pub use database::*; mod static_file; pub use static_file::{ - StaticFileJarProvider, StaticFileProvider, StaticFileProviderRW, StaticFileProviderRWRefMut, - StaticFileWriter, + StaticFileAccess, StaticFileJarProvider, StaticFileProvider, StaticFileProviderRW, + StaticFileProviderRWRefMut, StaticFileWriter, }; mod state; diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index fd877a4470eb..0052868ad9c0 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -3,22 +3,25 @@ use super::{ StaticFileProviderRWRefMut, BLOCKS_PER_STATIC_FILE, }; use crate::{ - to_range, BlockHashReader, BlockNumReader, BlockReader, BlockSource, HeaderProvider, - ReceiptProvider, RequestsProvider, StatsReader, TransactionVariant, TransactionsProvider, - TransactionsProviderExt, WithdrawalsProvider, + to_range, BlockHashReader, BlockNumReader, BlockReader, BlockSource, DatabaseProvider, + HeaderProvider, ReceiptProvider, RequestsProvider, StageCheckpointReader, StatsReader, + TransactionVariant, TransactionsProvider, TransactionsProviderExt, WithdrawalsProvider, }; use dashmap::{mapref::entry::Entry as DashMapEntry, DashMap}; use parking_lot::RwLock; use reth_db::{ codecs::CompactU256, + cursor::DbCursorRO, models::StoredBlockBodyIndices, static_file::{iter_static_files, HeaderMask, ReceiptMask, StaticFileCursor, TransactionMask}, table::Table, tables, + transaction::DbTx, }; use reth_nippy_jar::NippyJar; use reth_primitives::{ keccak256, + stage::{PipelineTarget, StageId}, static_file::{find_fixed_range, HighestStaticFiles, SegmentHeader, SegmentRangeInclusive}, Address, Block, BlockHash, BlockHashOrNumber, BlockNumber, BlockWithSenders, ChainInfo, Header, Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader, StaticFileSegment, TransactionMeta, @@ -32,24 +35,52 @@ use std::{ path::{Path, PathBuf}, sync::{mpsc, Arc}, }; -use tracing::warn; +use strum::IntoEnumIterator; +use tracing::{info, warn}; /// Alias type for a map that can be queried for block ranges from a transaction /// segment respectively. It uses `TxNumber` to represent the transaction end of a static file /// range. type SegmentRanges = HashMap>; +/// Access mode on a static file provider. RO/RW. +#[derive(Debug, Default, PartialEq, Eq)] +pub enum StaticFileAccess { + /// Read-only access. + #[default] + RO, + /// Read-write access. + RW, +} + +impl StaticFileAccess { + /// Returns `true` if read-only access. + pub const fn is_read_only(&self) -> bool { + matches!(self, Self::RO) + } +} + /// [`StaticFileProvider`] manages all existing [`StaticFileJarProvider`]. #[derive(Debug, Default, Clone)] pub struct StaticFileProvider(pub(crate) Arc); impl StaticFileProvider { /// Creates a new [`StaticFileProvider`]. - pub fn new(path: impl AsRef) -> ProviderResult { - let provider = Self(Arc::new(StaticFileProviderInner::new(path)?)); + fn new(path: impl AsRef, access: StaticFileAccess) -> ProviderResult { + let provider = Self(Arc::new(StaticFileProviderInner::new(path, access)?)); provider.initialize_index()?; Ok(provider) } + + /// Creates a new [`StaticFileProvider`] with read-only access. + pub fn read_only(path: impl AsRef) -> ProviderResult { + Self::new(path, StaticFileAccess::RO) + } + + /// Creates a new [`StaticFileProvider`] with read-write access. + pub fn read_write(path: impl AsRef) -> ProviderResult { + Self::new(path, StaticFileAccess::RW) + } } impl Deref for StaticFileProvider { @@ -78,11 +109,13 @@ pub struct StaticFileProviderInner { /// Maintains a map of StaticFile writers for each [`StaticFileSegment`] writers: DashMap, metrics: Option>, + /// Access rights of the provider. + access: StaticFileAccess, } impl StaticFileProviderInner { /// Creates a new [`StaticFileProviderInner`]. - fn new(path: impl AsRef) -> ProviderResult { + fn new(path: impl AsRef, access: StaticFileAccess) -> ProviderResult { let provider = Self { map: Default::default(), writers: Default::default(), @@ -91,10 +124,15 @@ impl StaticFileProviderInner { path: path.as_ref().to_path_buf(), load_filters: false, metrics: None, + access, }; Ok(provider) } + + pub const fn is_read_only(&self) -> bool { + self.access.is_read_only() + } } impl StaticFileProvider { @@ -448,6 +486,209 @@ impl StaticFileProvider { Ok(()) } + /// Ensures that any broken invariants which cannot be healed on the spot return a pipeline + /// target to unwind to. + /// + /// Two types of consistency checks are done for: + /// + /// 1) When a static file fails to commit but the underlying data was changed. + /// 2) When a static file was committed, but the required database transaction was not. + /// + /// For 1) it can self-heal if `self.access.is_read_only()` is set to `false`. Otherwise, it + /// will return an error. + /// For 2) the invariants below are checked, and if broken, might require a pipeline unwind + /// to heal. + /// + /// For each static file segment: + /// * the corresponding database table should overlap or have continuity in their keys + /// ([TxNumber] or [BlockNumber]). + /// * its highest block should match the stage checkpoint block number if it's equal or higher + /// than the corresponding database table last entry. + /// + /// Returns a [`Option`] of [`PipelineTarget::Unwind`] if any healing is further required. + /// + /// WARNING: No static file writer should be held before calling this function, otherwise it + /// will deadlock. + #[allow(clippy::while_let_loop)] + pub fn check_consistency( + &self, + provider: &DatabaseProvider, + has_receipt_pruning: bool, + ) -> ProviderResult> { + let mut unwind_target: Option = None; + let mut update_unwind_target = |new_target: BlockNumber| { + if let Some(target) = unwind_target.as_mut() { + *target = (*target).min(new_target); + } else { + unwind_target = Some(new_target); + } + }; + + for segment in StaticFileSegment::iter() { + if has_receipt_pruning && segment.is_receipts() { + // Pruned nodes (including full node) do not store receipts as static files. + continue + } + + let initial_highest_block = self.get_highest_static_file_block(segment); + + // File consistency is broken if: + // + // * appending data was interrupted before a config commit, then data file will be + // truncated according to the config. + // + // * pruning data was interrupted before a config commit, then we have deleted data that + // we are expected to still have. We need to check the Database and unwind everything + // accordingly. + self.ensure_file_consistency(segment)?; + + // Only applies to block-based static files. (Headers) + // + // The updated `highest_block` may have decreased if we healed from a pruning + // interruption. + let mut highest_block = self.get_highest_static_file_block(segment); + if initial_highest_block != highest_block { + update_unwind_target(highest_block.unwrap_or_default()); + } + + // Only applies to transaction-based static files. (Receipts & Transactions) + // + // Make sure the last transaction matches the last block from its indices, since a heal + // from a pruning interruption might have decreased the number of transactions without + // being able to update the last block of the static file segment. + let highest_tx = self.get_highest_static_file_tx(segment); + if let Some(highest_tx) = highest_tx { + let mut last_block = highest_block.unwrap_or_default(); + loop { + if let Some(indices) = provider.block_body_indices(last_block)? { + if indices.last_tx_num() <= highest_tx { + break + } + } else { + // If the block body indices can not be found, then it means that static + // files is ahead of database, and the `ensure_invariants` check will fix + // it by comparing with stage checkpoints. + break + } + if last_block == 0 { + break + } + last_block -= 1; + + highest_block = Some(last_block); + update_unwind_target(last_block); + } + } + + if let Some(unwind) = match segment { + StaticFileSegment::Headers => self.ensure_invariants::<_, tables::Headers>( + provider, + segment, + highest_block, + highest_block, + )?, + StaticFileSegment::Transactions => self + .ensure_invariants::<_, tables::Transactions>( + provider, + segment, + highest_tx, + highest_block, + )?, + StaticFileSegment::Receipts => self.ensure_invariants::<_, tables::Receipts>( + provider, + segment, + highest_tx, + highest_block, + )?, + } { + update_unwind_target(unwind); + } + } + + Ok(unwind_target.map(PipelineTarget::Unwind)) + } + + /// Check invariants for each corresponding table and static file segment: + /// + /// * the corresponding database table should overlap or have continuity in their keys + /// ([TxNumber] or [BlockNumber]). + /// * its highest block should match the stage checkpoint block number if it's equal or higher + /// than the corresponding database table last entry. + /// * If the checkpoint block is higher, then request a pipeline unwind to the static file + /// block. + /// * If the checkpoint block is lower, then heal by removing rows from the static file. + fn ensure_invariants>( + &self, + provider: &DatabaseProvider, + segment: StaticFileSegment, + highest_static_file_entry: Option, + highest_static_file_block: Option, + ) -> ProviderResult> { + let highest_static_file_entry = highest_static_file_entry.unwrap_or_default(); + let highest_static_file_block = highest_static_file_block.unwrap_or_default(); + let mut db_cursor = provider.tx_ref().cursor_read::()?; + + if let Some((db_first_entry, _)) = db_cursor.first()? { + // If there is a gap between the entry found in static file and + // database, then we have most likely lost static file data and need to unwind so we can + // load it again + if !(db_first_entry <= highest_static_file_entry || + highest_static_file_entry + 1 == db_first_entry) + { + return Ok(Some(highest_static_file_block)) + } + + if let Some((db_last_entry, _)) = db_cursor.last()? { + if db_last_entry > highest_static_file_entry { + return Ok(None) + } + } + } + + // If static file entry is ahead of the database entries, then ensure the checkpoint block + // number matches. + let checkpoint_block_number = provider + .get_stage_checkpoint(match segment { + StaticFileSegment::Headers => StageId::Headers, + StaticFileSegment::Transactions => StageId::Bodies, + StaticFileSegment::Receipts => StageId::Execution, + })? + .unwrap_or_default() + .block_number; + + // If the checkpoint is ahead, then we lost static file data. May be data corruption. + if checkpoint_block_number > highest_static_file_block { + return Ok(Some(highest_static_file_block)); + } + + // If the checkpoint is behind, then we failed to do a database commit **but committed** to + // static files on executing a stage, or the reverse on unwinding a stage. + // All we need to do is to prune the extra static file rows. + if checkpoint_block_number < highest_static_file_block { + info!( + target: "reth::providers", + ?segment, + from = highest_static_file_block, + to = checkpoint_block_number, + "Unwinding static file segment." + ); + let mut writer = self.latest_writer(segment)?; + if segment.is_headers() { + writer.prune_headers(highest_static_file_block - checkpoint_block_number)?; + } else if let Some(block) = provider.block_body_indices(checkpoint_block_number)? { + let number = highest_static_file_entry - block.last_tx_num(); + if segment.is_receipts() { + writer.prune_receipts(number, checkpoint_block_number)?; + } else { + writer.prune_transactions(number, checkpoint_block_number)?; + } + } + writer.commit()?; + } + + Ok(None) + } + /// Gets the highest static file block if it exists for a static file segment. pub fn get_highest_static_file_block(&self, segment: StaticFileSegment) -> Option { self.static_files_max_block.read().get(&segment).copied() @@ -717,6 +958,9 @@ pub trait StaticFileWriter { /// Commits all changes of all [`StaticFileProviderRW`] of all [`StaticFileSegment`]. fn commit(&self) -> ProviderResult<()>; + + /// Checks consistency of the segment latest file and heals if possible. + fn ensure_file_consistency(&self, segment: StaticFileSegment) -> ProviderResult<()>; } impl StaticFileWriter for StaticFileProvider { @@ -725,6 +969,10 @@ impl StaticFileWriter for StaticFileProvider { block: BlockNumber, segment: StaticFileSegment, ) -> ProviderResult> { + if self.access.is_read_only() { + return Err(ProviderError::ReadOnlyStaticFileAccess) + } + tracing::trace!(target: "providers::static_file", ?block, ?segment, "Getting static file writer."); Ok(match self.writers.entry(segment) { DashMapEntry::Occupied(entry) => entry.into_ref(), @@ -753,6 +1001,28 @@ impl StaticFileWriter for StaticFileProvider { } Ok(()) } + + fn ensure_file_consistency(&self, segment: StaticFileSegment) -> ProviderResult<()> { + match self.access { + StaticFileAccess::RO => { + let latest_block = self.get_highest_static_file_block(segment).unwrap_or_default(); + + let mut writer = StaticFileProviderRW::new( + segment, + latest_block, + Arc::downgrade(&self.0), + self.metrics.clone(), + )?; + + writer.ensure_file_consistency(self.access.is_read_only())?; + } + StaticFileAccess::RW => { + self.latest_writer(segment)?.ensure_file_consistency(self.access.is_read_only())?; + } + } + + Ok(()) + } } impl HeaderProvider for StaticFileProvider { @@ -771,8 +1041,15 @@ impl HeaderProvider for StaticFileProvider { } fn header_by_number(&self, num: BlockNumber) -> ProviderResult> { - self.get_segment_provider_from_block(StaticFileSegment::Headers, num, None)? - .header_by_number(num) + self.get_segment_provider_from_block(StaticFileSegment::Headers, num, None) + .and_then(|provider| provider.header_by_number(num)) + .or_else(|err| { + if let ProviderError::MissingStaticFileBlock(_, _) = err { + Ok(None) + } else { + Err(err) + } + }) } fn header_td(&self, block_hash: &BlockHash) -> ProviderResult> { @@ -785,8 +1062,15 @@ impl HeaderProvider for StaticFileProvider { } fn header_td_by_number(&self, num: BlockNumber) -> ProviderResult> { - self.get_segment_provider_from_block(StaticFileSegment::Headers, num, None)? - .header_td_by_number(num) + self.get_segment_provider_from_block(StaticFileSegment::Headers, num, None) + .and_then(|provider| provider.header_td_by_number(num)) + .or_else(|err| { + if let ProviderError::MissingStaticFileBlock(_, _) = err { + Ok(None) + } else { + Err(err) + } + }) } fn headers_range(&self, range: impl RangeBounds) -> ProviderResult> { @@ -799,8 +1083,15 @@ impl HeaderProvider for StaticFileProvider { } fn sealed_header(&self, num: BlockNumber) -> ProviderResult> { - self.get_segment_provider_from_block(StaticFileSegment::Headers, num, None)? - .sealed_header(num) + self.get_segment_provider_from_block(StaticFileSegment::Headers, num, None) + .and_then(|provider| provider.sealed_header(num)) + .or_else(|err| { + if let ProviderError::MissingStaticFileBlock(_, _) = err { + Ok(None) + } else { + Err(err) + } + }) } fn sealed_headers_while( @@ -842,8 +1133,15 @@ impl BlockHashReader for StaticFileProvider { impl ReceiptProvider for StaticFileProvider { fn receipt(&self, num: TxNumber) -> ProviderResult> { - self.get_segment_provider_from_transaction(StaticFileSegment::Receipts, num, None)? - .receipt(num) + self.get_segment_provider_from_transaction(StaticFileSegment::Receipts, num, None) + .and_then(|provider| provider.receipt(num)) + .or_else(|err| { + if let ProviderError::MissingStaticFileTx(_, _) = err { + Ok(None) + } else { + Err(err) + } + }) } fn receipt_by_hash(&self, hash: TxHash) -> ProviderResult> { @@ -947,16 +1245,30 @@ impl TransactionsProvider for StaticFileProvider { } fn transaction_by_id(&self, num: TxNumber) -> ProviderResult> { - self.get_segment_provider_from_transaction(StaticFileSegment::Transactions, num, None)? - .transaction_by_id(num) + self.get_segment_provider_from_transaction(StaticFileSegment::Transactions, num, None) + .and_then(|provider| provider.transaction_by_id(num)) + .or_else(|err| { + if let ProviderError::MissingStaticFileTx(_, _) = err { + Ok(None) + } else { + Err(err) + } + }) } fn transaction_by_id_no_hash( &self, num: TxNumber, ) -> ProviderResult> { - self.get_segment_provider_from_transaction(StaticFileSegment::Transactions, num, None)? - .transaction_by_id_no_hash(num) + self.get_segment_provider_from_transaction(StaticFileSegment::Transactions, num, None) + .and_then(|provider| provider.transaction_by_id_no_hash(num)) + .or_else(|err| { + if let ProviderError::MissingStaticFileTx(_, _) = err { + Ok(None) + } else { + Err(err) + } + }) } fn transaction_by_hash(&self, hash: TxHash) -> ProviderResult> { diff --git a/crates/storage/provider/src/providers/static_file/mod.rs b/crates/storage/provider/src/providers/static_file/mod.rs index 4ea8c525d2e8..97fc649a46cc 100644 --- a/crates/storage/provider/src/providers/static_file/mod.rs +++ b/crates/storage/provider/src/providers/static_file/mod.rs @@ -1,5 +1,5 @@ mod manager; -pub use manager::{StaticFileProvider, StaticFileWriter}; +pub use manager::{StaticFileAccess, StaticFileProvider, StaticFileWriter}; mod jar; pub use jar::StaticFileJarProvider; @@ -150,7 +150,8 @@ mod tests { // Use providers to query Header data and compare if it matches { let db_provider = factory.provider().unwrap(); - let manager = StaticFileProvider::new(static_files_path.path()).unwrap().with_filters(); + let manager = + StaticFileProvider::read_write(static_files_path.path()).unwrap().with_filters(); let jar_provider = manager .get_segment_provider_from_block(StaticFileSegment::Headers, 0, Some(&static_file)) .unwrap(); diff --git a/crates/storage/provider/src/providers/static_file/writer.rs b/crates/storage/provider/src/providers/static_file/writer.rs index 7289da860595..3b88812da7f3 100644 --- a/crates/storage/provider/src/providers/static_file/writer.rs +++ b/crates/storage/provider/src/providers/static_file/writer.rs @@ -6,7 +6,7 @@ use super::{ use dashmap::mapref::one::RefMut; use reth_codecs::Compact; use reth_db::codecs::CompactU256; -use reth_nippy_jar::{NippyJar, NippyJarError, NippyJarWriter}; +use reth_nippy_jar::{ConsistencyFailStrategy, NippyJar, NippyJarError, NippyJarWriter}; use reth_primitives::{ static_file::{find_fixed_range, SegmentHeader, SegmentRangeInclusive}, BlockHash, BlockNumber, Header, Receipt, StaticFileSegment, TransactionSignedNoHash, TxNumber, @@ -90,7 +90,14 @@ impl StaticFileProviderRW { Err(err) => return Err(err), }; - let result = match NippyJarWriter::new(jar) { + let reader = Self::upgrade_provider_to_strong_reference(&reader); + let access = if reader.is_read_only() { + ConsistencyFailStrategy::ThrowError + } else { + ConsistencyFailStrategy::Heal + }; + + let result = match NippyJarWriter::new(jar, access) { Ok(writer) => Ok((writer, path)), Err(NippyJarError::FrozenJar) => { // This static file has been frozen, so we should @@ -110,6 +117,54 @@ impl StaticFileProviderRW { Ok(result) } + /// Checks the consistency of the file and heals it if necessary and `read_only` is set to + /// false. If the check fails, it will return an error. + /// + /// If healing does happen, it will update the end range on the [SegmentHeader]. However, for + /// transaction based segments, the block end range has to be found and healed externally. + /// + /// Check [NippyJarWriter::ensure_file_consistency] for more on healing. + pub fn ensure_file_consistency(&mut self, read_only: bool) -> ProviderResult<()> { + let inconsistent_error = || { + ProviderError::NippyJar( + "Inconsistent state found. Restart the node to heal.".to_string(), + ) + }; + + let check_mode = if read_only { + ConsistencyFailStrategy::ThrowError + } else { + ConsistencyFailStrategy::Heal + }; + + self.writer.ensure_file_consistency(check_mode).map_err(|error| { + if matches!(error, NippyJarError::InconsistentState) { + return inconsistent_error() + } + ProviderError::NippyJar(error.to_string()) + })?; + + // If we have lost rows (in this run or previous), we need to update the [SegmentHeader]. + let expected_rows = if self.user_header().segment().is_headers() { + self.user_header().block_len().unwrap_or_default() + } else { + self.user_header().tx_len().unwrap_or_default() + }; + let pruned_rows = expected_rows - self.writer.rows() as u64; + if pruned_rows > 0 { + if read_only { + return Err(inconsistent_error()) + } + self.user_header_mut().prune(pruned_rows); + } + + self.writer.commit().map_err(|error| ProviderError::NippyJar(error.to_string()))?; + + // Updates the [SnapshotProvider] manager + self.update_index()?; + Ok(()) + } + /// Commits configuration changes to disk and updates the reader index with the new changes. pub fn commit(&mut self) -> ProviderResult<()> { let start = Instant::now(); @@ -285,10 +340,11 @@ impl StaticFileProviderRW { fn truncate( &mut self, segment: StaticFileSegment, - mut num_rows: u64, + num_rows: u64, last_block: Option, ) -> ProviderResult<()> { - while num_rows > 0 { + let mut remaining_rows = num_rows; + while remaining_rows > 0 { let len = match segment { StaticFileSegment::Headers => { self.writer.user_header().block_len().unwrap_or_default() @@ -298,26 +354,13 @@ impl StaticFileProviderRW { } }; - if num_rows >= len { + if remaining_rows >= len { // If there's more rows to delete than this static file contains, then just // delete the whole file and go to the next static file - let previous_snap = self.data_path.clone(); let block_start = self.writer.user_header().expected_block_start(); if block_start != 0 { - let (writer, data_path) = Self::open( - segment, - self.writer.user_header().expected_block_start() - 1, - self.reader.clone(), - self.metrics.clone(), - )?; - self.writer = writer; - self.data_path = data_path; - - NippyJar::::load(&previous_snap) - .map_err(|e| ProviderError::NippyJar(e.to_string()))? - .delete() - .map_err(|e| ProviderError::NippyJar(e.to_string()))?; + self.delete_current_and_open_previous()?; } else { // Update `SegmentHeader` self.writer.user_header_mut().prune(len); @@ -327,23 +370,33 @@ impl StaticFileProviderRW { break } - num_rows -= len; + remaining_rows -= len; } else { // Update `SegmentHeader` - self.writer.user_header_mut().prune(num_rows); + self.writer.user_header_mut().prune(remaining_rows); // Truncate data self.writer - .prune_rows(num_rows as usize) + .prune_rows(remaining_rows as usize) .map_err(|e| ProviderError::NippyJar(e.to_string()))?; - num_rows = 0; + remaining_rows = 0; } } // Only Transactions and Receipts if let Some(last_block) = last_block { - let header = self.writer.user_header_mut(); - header.set_block_range(header.expected_block_start(), last_block); + let mut expected_block_start = self.writer.user_header().expected_block_start(); + + if num_rows == 0 { + // Edge case for when we are unwinding a chain of empty blocks that goes across + // files, and therefore, the only reference point to know which file + // we are supposed to be at is `last_block`. + while last_block < expected_block_start { + self.delete_current_and_open_previous()?; + expected_block_start = self.writer.user_header().expected_block_start(); + } + } + self.writer.user_header_mut().set_block_range(expected_block_start, last_block); } // Commits new changes to disk. @@ -352,6 +405,25 @@ impl StaticFileProviderRW { Ok(()) } + /// Delete the current static file, and replace this provider writer with the previous static + /// file. + fn delete_current_and_open_previous(&mut self) -> Result<(), ProviderError> { + let current_path = self.data_path.clone(); + let (previous_writer, data_path) = Self::open( + self.user_header().segment(), + self.writer.user_header().expected_block_start() - 1, + self.reader.clone(), + self.metrics.clone(), + )?; + self.writer = previous_writer; + self.data_path = data_path; + NippyJar::::load(¤t_path) + .map_err(|e| ProviderError::NippyJar(e.to_string()))? + .delete() + .map_err(|e| ProviderError::NippyJar(e.to_string()))?; + Ok(()) + } + /// Appends column to static file. fn append_column(&mut self, column: T) -> ProviderResult<()> { self.buf.clear(); @@ -613,16 +685,26 @@ impl StaticFileProviderRW { provider.upgrade().map(StaticFileProvider).expect("StaticFileProvider is dropped") } - #[cfg(any(test, feature = "test-utils"))] + /// Helper function to access [`SegmentHeader`]. + pub const fn user_header(&self) -> &SegmentHeader { + self.writer.user_header() + } + + /// Helper function to access a mutable reference to [`SegmentHeader`]. + pub fn user_header_mut(&mut self) -> &mut SegmentHeader { + self.writer.user_header_mut() + } + /// Helper function to override block range for testing. + #[cfg(any(test, feature = "test-utils"))] pub fn set_block_range(&mut self, block_range: std::ops::RangeInclusive) { self.writer.user_header_mut().set_block_range(*block_range.start(), *block_range.end()) } + /// Helper function to override block range for testing. #[cfg(any(test, feature = "test-utils"))] - /// Helper function to access [`SegmentHeader`]. - pub const fn user_header(&self) -> &SegmentHeader { - self.writer.user_header() + pub fn inner(&mut self) -> &mut NippyJarWriter { + &mut self.writer } } diff --git a/crates/storage/provider/src/test_utils/mod.rs b/crates/storage/provider/src/test_utils/mod.rs index 2f5462309442..6f5ecd526783 100644 --- a/crates/storage/provider/src/test_utils/mod.rs +++ b/crates/storage/provider/src/test_utils/mod.rs @@ -1,4 +1,4 @@ -use crate::ProviderFactory; +use crate::{providers::StaticFileProvider, ProviderFactory}; use reth_db::{ test_utils::{create_test_rw_db, create_test_static_files_dir, TempDatabase}, DatabaseEnv, @@ -26,6 +26,9 @@ pub fn create_test_provider_factory_with_chain_spec( ) -> ProviderFactory>> { let (static_dir, _) = create_test_static_files_dir(); let db = create_test_rw_db(); - ProviderFactory::new(db, chain_spec, static_dir.into_path()) - .expect("create provider factory with static_files") + ProviderFactory::new( + db, + chain_spec, + StaticFileProvider::read_write(static_dir.into_path()).expect("static file provider"), + ) } diff --git a/examples/db-access/src/main.rs b/examples/db-access/src/main.rs index c076b76dcf24..c43aec47ce0c 100644 --- a/examples/db-access/src/main.rs +++ b/examples/db-access/src/main.rs @@ -1,8 +1,8 @@ use reth_db::open_db_read_only; use reth_primitives::{Address, ChainSpecBuilder, B256}; use reth_provider::{ - AccountReader, BlockReader, BlockSource, HeaderProvider, ProviderFactory, ReceiptProvider, - StateProvider, TransactionsProvider, + providers::StaticFileProvider, AccountReader, BlockReader, BlockSource, HeaderProvider, + ProviderFactory, ReceiptProvider, StateProvider, TransactionsProvider, }; use reth_rpc_types::{Filter, FilteredParams}; use std::path::Path; @@ -24,7 +24,11 @@ fn main() -> eyre::Result<()> { // Instantiate a provider factory for Ethereum mainnet using the provided DB. // TODO: Should the DB version include the spec so that you do not need to specify it here? let spec = ChainSpecBuilder::mainnet().build(); - let factory = ProviderFactory::new(db, spec.into(), db_path.join("static_files"))?; + let factory = ProviderFactory::new( + db, + spec.into(), + StaticFileProvider::read_only(db_path.join("static_files"))?, + ); // This call opens a RO transaction on the database. To write to the DB you'd need to call // the `provider_rw` function and look for the `Writer` variants of the traits. diff --git a/examples/rpc-db/src/main.rs b/examples/rpc-db/src/main.rs index 627da093c591..e51f8fe1091a 100644 --- a/examples/rpc-db/src/main.rs +++ b/examples/rpc-db/src/main.rs @@ -14,7 +14,10 @@ use reth::{ primitives::ChainSpecBuilder, - providers::{providers::BlockchainProvider, ProviderFactory}, + providers::{ + providers::{BlockchainProvider, StaticFileProvider}, + ProviderFactory, + }, utils::db::open_db_read_only, }; use reth_db::{mdbx::DatabaseArguments, models::client_version::ClientVersion}; @@ -44,7 +47,11 @@ async fn main() -> eyre::Result<()> { DatabaseArguments::new(ClientVersion::default()), )?); let spec = Arc::new(ChainSpecBuilder::mainnet().build()); - let factory = ProviderFactory::new(db.clone(), spec.clone(), db_path.join("static_files"))?; + let factory = ProviderFactory::new( + db.clone(), + spec.clone(), + StaticFileProvider::read_only(db_path.join("static_files"))?, + ); // 2. Setup the blockchain provider using only the database provider and a noop for the tree to // satisfy trait bounds. Tree is not used in this example since we are only operating on the diff --git a/testing/ef-tests/src/cases/blockchain_test.rs b/testing/ef-tests/src/cases/blockchain_test.rs index fb71d1a868a7..a0aee453fe63 100644 --- a/testing/ef-tests/src/cases/blockchain_test.rs +++ b/testing/ef-tests/src/cases/blockchain_test.rs @@ -8,7 +8,10 @@ use alloy_rlp::Decodable; use rayon::iter::{ParallelBridge, ParallelIterator}; use reth_db::test_utils::{create_test_rw_db, create_test_static_files_dir}; use reth_primitives::{BlockBody, SealedBlock, StaticFileSegment}; -use reth_provider::{providers::StaticFileWriter, HashingWriter, ProviderFactory}; +use reth_provider::{ + providers::{StaticFileProvider, StaticFileWriter}, + HashingWriter, ProviderFactory, +}; use reth_stages::{stages::ExecutionStage, ExecInput, Stage}; use std::{collections::BTreeMap, fs, path::Path, sync::Arc}; @@ -86,8 +89,8 @@ impl Case for BlockchainTestCase { let provider = ProviderFactory::new( db.as_ref(), Arc::new(case.network.clone().into()), - static_files_dir_path, - )? + StaticFileProvider::read_write(static_files_dir_path).unwrap(), + ) .provider_rw() .unwrap(); From 72d594745486fc01ba13b393251718400f39548d Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Sat, 1 Jun 2024 13:16:26 +0200 Subject: [PATCH 06/11] improve `estimate_gas_with` (#8535) --- crates/rpc/rpc/src/eth/api/call.rs | 177 +++++++++++---------- crates/rpc/rpc/src/eth/api/mod.rs | 2 +- crates/storage/storage-api/src/block_id.rs | 23 ++- crates/storage/storage-api/src/state.rs | 12 +- 4 files changed, 108 insertions(+), 106 deletions(-) diff --git a/crates/rpc/rpc/src/eth/api/call.rs b/crates/rpc/rpc/src/eth/api/call.rs index d638251322c8..4143f11ba65c 100644 --- a/crates/rpc/rpc/src/eth/api/call.rs +++ b/crates/rpc/rpc/src/eth/api/call.rs @@ -198,31 +198,34 @@ where // cfg.disable_base_fee = true; - // keep a copy of gas related request values - let request_gas = request.gas; - let request_gas_price = request.gas_price; - let env_gas_limit = block.gas_limit; + // Keep a copy of gas related request values + let tx_request_gas_limit = request.gas; + let tx_request_gas_price = request.gas_price; + let block_env_gas_limit = block.gas_limit; - // get the highest possible gas limit, either the request's set value or the currently - // configured gas limit - let mut highest_gas_limit = request.gas.map(U256::from).unwrap_or(block.gas_limit); + // Determine the highest possible gas limit, considering both the request's specified limit + // and the block's limit. + let mut highest_gas_limit = tx_request_gas_limit + .map(|tx_gas_limit| U256::from(tx_gas_limit).max(block_env_gas_limit)) + .unwrap_or(block_env_gas_limit); // Configure the evm env let mut env = build_call_evm_env(cfg, block, request)?; let mut db = CacheDB::new(StateProviderDatabase::new(state)); + // Apply any state overrides if specified. if let Some(state_override) = state_override { - // apply state overrides apply_state_overrides(state_override, &mut db)?; } - // if the request is a simple transfer we can optimize + + // Optimize for simple transfer transactions, potentially reducing the gas estimate. if env.tx.data.is_empty() { if let TransactTo::Call(to) = env.tx.transact_to { if let Ok(code) = db.db.account_code(to) { let no_code_callee = code.map(|code| code.is_empty()).unwrap_or(true); if no_code_callee { // If the tx is a simple transfer (call to an account with no code) we can - // shortcircuit But simply returning + // shortcircuit. But simply returning // `MIN_TRANSACTION_GAS` is dangerous because there might be additional // field combos that bump the price up, so we try executing the function // with the minimum gas limit to make sure. @@ -238,43 +241,39 @@ where } } - // check funds of the sender + // Check funds of the sender (only useful to check if transaction gas price is more than 0). + // + // The caller allowance is check by doing `(account.balance - tx.value) / tx.gas_price` if env.tx.gas_price > U256::ZERO { - let allowance = caller_gas_allowance(&mut db, &env.tx)?; - - if highest_gas_limit > allowance { - // cap the highest gas limit by max gas caller can afford with given gas price - highest_gas_limit = allowance; - } + // cap the highest gas limit by max gas caller can afford with given gas price + highest_gas_limit = highest_gas_limit.min(caller_gas_allowance(&mut db, &env.tx)?); } - // if the provided gas limit is less than computed cap, use that - let gas_limit = std::cmp::min(U256::from(env.tx.gas_limit), highest_gas_limit); - env.tx.gas_limit = gas_limit.saturating_to(); + // We can now normalize the highest gas limit to a u64 + let mut highest_gas_limit: u64 = highest_gas_limit.try_into().unwrap_or(u64::MAX); - trace!(target: "rpc::eth::estimate", ?env, "Starting gas estimation"); + // If the provided gas limit is less than computed cap, use that + env.tx.gas_limit = env.tx.gas_limit.min(highest_gas_limit); - // transact with the highest __possible__ gas limit - let ethres = self.transact(&mut db, env.clone()); + trace!(target: "rpc::eth::estimate", ?env, "Starting gas estimation"); - // Exceptional case: init used too much gas, we need to increase the gas limit and try - // again - if matches!( - ethres, + // Execute the transaction with the highest possible gas limit. + let (mut res, mut env) = match self.transact(&mut db, env.clone()) { + // Handle the exceptional case where the transaction initialization uses too much gas. + // If the gas price or gas limit was specified in the request, retry the transaction + // with the block's gas limit to determine if the failure was due to + // insufficient gas. Err(EthApiError::InvalidTransaction(RpcInvalidTransactionError::GasTooHigh)) - ) { - // if price or limit was included in the request then we can execute the request - // again with the block's gas limit to check if revert is gas related or not - if request_gas.is_some() || request_gas_price.is_some() { - return Err(self.map_out_of_gas_err(env_gas_limit, env, &mut db)) + if tx_request_gas_limit.is_some() || tx_request_gas_price.is_some() => + { + return Err(self.map_out_of_gas_err(block_env_gas_limit, env, &mut db)); } - } + // Propagate other results (successful or other errors). + ethres => ethres?, + }; - let (mut res, mut env) = ethres?; - match res.result { - ExecutionResult::Success { .. } => { - // succeeded - } + let gas_refund = match res.result { + ExecutionResult::Success { gas_refunded, .. } => gas_refunded, ExecutionResult::Halt { reason, gas_used } => { // here we don't check for invalid opcode because already executed with highest gas // limit @@ -283,35 +282,38 @@ where ExecutionResult::Revert { output, .. } => { // if price or limit was included in the request then we can execute the request // again with the block's gas limit to check if revert is gas related or not - return if request_gas.is_some() || request_gas_price.is_some() { - Err(self.map_out_of_gas_err(env_gas_limit, env, &mut db)) + return if tx_request_gas_limit.is_some() || tx_request_gas_price.is_some() { + Err(self.map_out_of_gas_err(block_env_gas_limit, env, &mut db)) } else { // the transaction did revert Err(RpcInvalidTransactionError::Revert(RevertError::new(output)).into()) } } - } + }; - // at this point we know the call succeeded but want to find the _best_ (lowest) gas the - // transaction succeeds with. We find this by doing a binary search over the - // possible range NOTE: this is the gas the transaction used, which is less than the - // transaction requires to succeed + // At this point we know the call succeeded but want to find the _best_ (lowest) gas the + // transaction succeeds with. We find this by doing a binary search over the possible range. + // + // NOTE: this is the gas the transaction used, which is less than the + // transaction requires to succeed. let gas_used = res.result.gas_used(); - let mut highest_gas_limit: u64 = highest_gas_limit.try_into().unwrap_or(u64::MAX); // the lowest value is capped by the gas used by the unconstrained transaction let mut lowest_gas_limit = gas_used.saturating_sub(1); - let gas_refund = match res.result { - ExecutionResult::Success { gas_refunded, .. } => gas_refunded, - _ => 0, - }; - // As stated in Geth, there is a good change that the transaction will pass if we set the + // As stated in Geth, there is a good chance that the transaction will pass if we set the // gas limit to the execution gas used plus the gas refund, so we check this first // 1 { // An estimation error is allowed once the current gas limit range used in the binary // search is small enough (less than 1.5% of the highest gas limit) @@ -340,27 +343,29 @@ where }; env.tx.gas_limit = mid_gas_limit; - let ethres = self.transact(&mut db, env.clone()); - - // Exceptional case: init used too much gas, we need to increase the gas limit and try - // again - if matches!( - ethres, - Err(EthApiError::InvalidTransaction(RpcInvalidTransactionError::GasTooHigh)) - ) { - // increase the lowest gas limit - lowest_gas_limit = mid_gas_limit; - } else { - (res, env) = ethres?; - update_estimated_gas_range( - res.result, - mid_gas_limit, - &mut highest_gas_limit, - &mut lowest_gas_limit, - )?; + + // Execute transaction and handle potential gas errors, adjusting limits accordingly. + match self.transact(&mut db, env.clone()) { + // Check if the error is due to gas being too high. + Err(EthApiError::InvalidTransaction(RpcInvalidTransactionError::GasTooHigh)) => { + // Increase the lowest gas limit if gas is too high + lowest_gas_limit = mid_gas_limit; + } + // Handle other cases, including successful transactions. + ethres => { + // Unpack the result and environment if the transaction was successful. + (res, env) = ethres?; + // Update the estimated gas range based on the transaction result. + update_estimated_gas_range( + res.result, + mid_gas_limit, + &mut highest_gas_limit, + &mut lowest_gas_limit, + )?; + } } - // new midpoint + // New midpoint mid_gas_limit = ((highest_gas_limit as u128 + lowest_gas_limit as u128) / 2) as u64; } @@ -480,8 +485,11 @@ where } } -/// Updates the highest and lowest gas limits for binary search -/// based on the result of the execution +/// Updates the highest and lowest gas limits for binary search based on the execution result. +/// +/// This function refines the gas limit estimates used in a binary search to find the optimal gas +/// limit for a transaction. It adjusts the highest or lowest gas limits depending on whether the +/// execution succeeded, reverted, or halted due to specific reasons. #[inline] fn update_estimated_gas_range( result: ExecutionResult, @@ -491,26 +499,29 @@ fn update_estimated_gas_range( ) -> EthResult<()> { match result { ExecutionResult::Success { .. } => { - // cap the highest gas limit with succeeding gas limit + // Cap the highest gas limit with the succeeding gas limit. *highest_gas_limit = tx_gas_limit; } ExecutionResult::Revert { .. } => { - // increase the lowest gas limit + // Increase the lowest gas limit. *lowest_gas_limit = tx_gas_limit; } ExecutionResult::Halt { reason, .. } => { match reason { HaltReason::OutOfGas(_) | HaltReason::InvalidFEOpcode => { - // either out of gas or invalid opcode can be thrown dynamically if - // gasLeft is too low, so we treat this as `out of gas`, we know this - // call succeeds with a higher gaslimit. common usage of invalid opcode in openzeppelin - - // increase the lowest gas limit + // Both `OutOfGas` and `InvalidFEOpcode` can occur dynamically if the gas left + // is too low. Treat this as an out of gas condition, + // knowing that the call succeeds with a higher gas limit. + // + // Common usage of invalid opcode in OpenZeppelin: + // + + // Increase the lowest gas limit. *lowest_gas_limit = tx_gas_limit; } err => { - // these should be unreachable because we know the transaction succeeds, - // but we consider these cases an error + // These cases should be unreachable because we know the transaction succeeds, + // but if they occur, treat them as an error. return Err(RpcInvalidTransactionError::EvmHalt(err).into()) } } diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index 484cf73c91a8..90d0b218ec76 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -10,6 +10,7 @@ use crate::eth::{ error::{EthApiError, EthResult}, gas_oracle::GasPriceOracle, signer::EthSigner, + traits::RawTransactionForwarder, }; use async_trait::async_trait; use reth_errors::{RethError, RethResult}; @@ -48,7 +49,6 @@ mod sign; mod state; mod transactions; -use crate::eth::traits::RawTransactionForwarder; pub use transactions::{EthTransactions, TransactionSource}; /// `Eth` API trait. diff --git a/crates/storage/storage-api/src/block_id.rs b/crates/storage/storage-api/src/block_id.rs index 7aac3d5c14ff..e648aa609eda 100644 --- a/crates/storage/storage-api/src/block_id.rs +++ b/crates/storage/storage-api/src/block_id.rs @@ -77,22 +77,17 @@ pub trait BlockIdReader: BlockNumReader + Send + Sync { fn block_hash_for_id(&self, block_id: BlockId) -> ProviderResult> { match block_id { BlockId::Hash(hash) => Ok(Some(hash.into())), - BlockId::Number(num) => { - if matches!(num, BlockNumberOrTag::Latest) { - return Ok(Some(self.chain_info()?.best_hash)) - } - - if matches!(num, BlockNumberOrTag::Pending) { - return self - .pending_block_num_hash() - .map(|res_opt| res_opt.map(|num_hash| num_hash.hash)) - } - - self.convert_block_number(num)? + BlockId::Number(num) => match num { + BlockNumberOrTag::Latest => Ok(Some(self.chain_info()?.best_hash)), + BlockNumberOrTag::Pending => self + .pending_block_num_hash() + .map(|res_opt| res_opt.map(|num_hash| num_hash.hash)), + _ => self + .convert_block_number(num)? .map(|num| self.block_hash(num)) .transpose() - .map(|maybe_hash| maybe_hash.flatten()) - } + .map(|maybe_hash| maybe_hash.flatten()), + }, } } diff --git a/crates/storage/storage-api/src/state.rs b/crates/storage/storage-api/src/state.rs index 24209b7ab8d2..28d721eb6b6e 100644 --- a/crates/storage/storage-api/src/state.rs +++ b/crates/storage/storage-api/src/state.rs @@ -124,19 +124,15 @@ pub trait StateProviderFactory: BlockIdReader + Send + Sync { BlockNumberOrTag::Latest => self.latest(), BlockNumberOrTag::Finalized => { // we can only get the finalized state by hash, not by num - let hash = match self.finalized_block_hash()? { - Some(hash) => hash, - None => return Err(ProviderError::FinalizedBlockNotFound), - }; + let hash = + self.finalized_block_hash()?.ok_or(ProviderError::FinalizedBlockNotFound)?; + // only look at historical state self.history_by_block_hash(hash) } BlockNumberOrTag::Safe => { // we can only get the safe state by hash, not by num - let hash = match self.safe_block_hash()? { - Some(hash) => hash, - None => return Err(ProviderError::SafeBlockNotFound), - }; + let hash = self.safe_block_hash()?.ok_or(ProviderError::SafeBlockNotFound)?; self.history_by_block_hash(hash) } From dfce570ceb33c9b30ec664a053469561215f3a81 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Sat, 1 Jun 2024 16:18:14 +0100 Subject: [PATCH 07/11] test(rpc): payload V4 decode hashing (#8531) --- .../rpc-types-compat/src/engine/payload.rs | 61 +++++++++++++++++-- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/crates/rpc/rpc-types-compat/src/engine/payload.rs b/crates/rpc/rpc-types-compat/src/engine/payload.rs index e68cee2bbc43..e7f4ba04689a 100644 --- a/crates/rpc/rpc-types-compat/src/engine/payload.rs +++ b/crates/rpc/rpc-types-compat/src/engine/payload.rs @@ -375,11 +375,12 @@ pub fn execution_payload_from_sealed_block(value: SealedBlock) -> ExecutionPaylo #[cfg(test)] mod tests { use super::{ - block_to_payload_v3, try_into_block, try_payload_v3_to_block, validate_block_hash, + block_to_payload_v3, try_into_block, try_payload_v3_to_block, try_payload_v4_to_block, + validate_block_hash, }; use reth_primitives::{b256, hex, Bytes, U256}; use reth_rpc_types::{ - engine::{CancunPayloadFields, ExecutionPayloadV3}, + engine::{CancunPayloadFields, ExecutionPayloadV3, ExecutionPayloadV4}, ExecutionPayload, ExecutionPayloadV1, ExecutionPayloadV2, }; @@ -417,13 +418,13 @@ mod tests { // this newPayload came with a parent beacon block root, we need to manually insert it // before hashing let parent_beacon_block_root = - hex!("531cd53b8e68deef0ea65edfa3cda927a846c307b0907657af34bc3f313b5871"); - block.header.parent_beacon_block_root = Some(parent_beacon_block_root.into()); + b256!("531cd53b8e68deef0ea65edfa3cda927a846c307b0907657af34bc3f313b5871"); + block.header.parent_beacon_block_root = Some(parent_beacon_block_root); let converted_payload = block_to_payload_v3(block.seal_slow()); // ensure the payloads are the same - assert_eq!((new_payload, Some(parent_beacon_block_root.into())), converted_payload); + assert_eq!((new_payload, Some(parent_beacon_block_root)), converted_payload); } #[test] @@ -603,4 +604,54 @@ mod tests { // Ensure the actual hash is calculated if we set the fields to what they should be validate_block_hash(block_hash_with_blob_fee_fields, block).unwrap(); } + + #[test] + fn parse_payload_v4() { + let s = r#"{ + "baseFeePerGas": "0x2ada43", + "blobGasUsed": "0x0", + "blockHash": "0x86eeb2a4b656499f313b601e1dcaedfeacccab27131b6d4ea99bc69a57607f7d", + "blockNumber": "0x2c", + "depositRequests": [ + { + "amount": "0xe8d4a51000", + "index": "0x0", + "pubkey": "0xaab5f2b3aad5c2075faf0c1d8937c7de51a53b765a21b4173eb2975878cea05d9ed3428b77f16a981716aa32af74c464", + "signature": "0xa889cd238be2dae44f2a3c24c04d686c548f6f82eb44d4604e1bc455b6960efb72b117e878068a8f2cfb91ad84b7ebce05b9254207aa51a1e8a3383d75b5a5bd2439f707636ea5b17b2b594b989c93b000b33e5dff6e4bed9d53a6d2d6889b0c", + "withdrawalCredentials": "0x00ab9364f8bf7561862ea0fc3b69c424c94ace406c4dc36ddfbf8a9d72051c80" + }, + { + "amount": "0xe8d4a51000", + "index": "0x1", + "pubkey": "0xb0b1b3b51cf688ead965a954c5cc206ba4e76f3f8efac60656ae708a9aad63487a2ca1fb30ccaf2ebe1028a2b2886b1b", + "signature": "0xb9759766e9bb191b1c457ae1da6cdf71a23fb9d8bc9f845eaa49ee4af280b3b9720ac4d81e64b1b50a65db7b8b4e76f1176a12e19d293d75574600e99fbdfecc1ab48edaeeffb3226cd47691d24473821dad0c6ff3973f03e4aa89f418933a56", + "withdrawalCredentials": "0x002d2b75f4a27f78e585a4735a40ab2437eceb12ec39938a94dc785a54d62513" + } + ], + "excessBlobGas": "0x0", + "extraData": "0x726574682f76302e322e302d626574612e372f6c696e7578", + "feeRecipient": "0x8943545177806ed17b9f23f0a21ee5948ecaa776", + "gasLimit": "0x1855e85", + "gasUsed": "0x25f98", + "logsBloom": "0x10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000020000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008000000000000000000000000", + "parentHash": "0xd753194ef19b5c566b7eca6e9ebcca03895b548e1e93a20a23d922ba0bc210d4", + "prevRandao": "0x8c52256fd491776dc32f531ad4c0dc1444684741bca15f54c9cd40c60142df90", + "receiptsRoot": "0x510e7fb94279897e5dcd6c1795f6137d8fe02e49e871bfea7999fd21a89f66aa", + "stateRoot": "0x59ae0706a2b47162666fc7af3e30ff7aa34154954b68cc6aed58c3af3d58c9c2", + "timestamp": "0x6643c5a9", + "transactions": [ + "0x02f9021e8330182480843b9aca0085174876e80083030d40944242424242424242424242424242424242424242893635c9adc5dea00000b901a422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000e0000000000000000000000000000000000000000000000000000000000000012049f42823819771c6bbbd9cb6649850083fd3b6e5d0beb1069342c32d65a3b0990000000000000000000000000000000000000000000000000000000000000030aab5f2b3aad5c2075faf0c1d8937c7de51a53b765a21b4173eb2975878cea05d9ed3428b77f16a981716aa32af74c46400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000ab9364f8bf7561862ea0fc3b69c424c94ace406c4dc36ddfbf8a9d72051c800000000000000000000000000000000000000000000000000000000000000060a889cd238be2dae44f2a3c24c04d686c548f6f82eb44d4604e1bc455b6960efb72b117e878068a8f2cfb91ad84b7ebce05b9254207aa51a1e8a3383d75b5a5bd2439f707636ea5b17b2b594b989c93b000b33e5dff6e4bed9d53a6d2d6889b0cc080a0db786f0d89923949e533680524f003cebd66f32fbd30429a6b6bfbd3258dcf60a05241c54e05574765f7ddc1a742ae06b044edfe02bffb202bf172be97397eeca9", + "0x02f9021e8330182401843b9aca0085174876e80083030d40944242424242424242424242424242424242424242893635c9adc5dea00000b901a422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000120d694d6a0b0103651aafd87db6c88297175d7317c6e6da53ccf706c3c991c91fd0000000000000000000000000000000000000000000000000000000000000030b0b1b3b51cf688ead965a954c5cc206ba4e76f3f8efac60656ae708a9aad63487a2ca1fb30ccaf2ebe1028a2b2886b1b000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020002d2b75f4a27f78e585a4735a40ab2437eceb12ec39938a94dc785a54d625130000000000000000000000000000000000000000000000000000000000000060b9759766e9bb191b1c457ae1da6cdf71a23fb9d8bc9f845eaa49ee4af280b3b9720ac4d81e64b1b50a65db7b8b4e76f1176a12e19d293d75574600e99fbdfecc1ab48edaeeffb3226cd47691d24473821dad0c6ff3973f03e4aa89f418933a56c080a099dc5b94a51e9b91a6425b1fed9792863006496ab71a4178524819d7db0c5e88a0119748e62700234079d91ae80f4676f9e0f71b260e9b46ef9b4aff331d3c2318" + ], + "withdrawalRequests": [], + "withdrawals": [] + }"#; + + let payload = serde_json::from_str::(s).unwrap(); + let mut block = try_payload_v4_to_block(payload).unwrap(); + block.header.parent_beacon_block_root = + Some(b256!("d9851db05fa63593f75e2b12c4bba9f47740613ca57da3b523a381b8c27f3297")); + let hash = block.seal_slow().hash(); + assert_eq!(hash, b256!("86eeb2a4b656499f313b601e1dcaedfeacccab27131b6d4ea99bc69a57607f7d")) + } } From da233bf32284ecbb395a6ccbece94e1786d63180 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Sat, 1 Jun 2024 17:21:50 +0200 Subject: [PATCH 08/11] add `empty_line_after_doc_comments` and `remove option_if_let_else` (#8539) --- Cargo.toml | 3 +-- crates/node-core/src/args/types.rs | 1 - crates/rpc/ipc/src/server/mod.rs | 4 ++-- crates/rpc/rpc-builder/src/lib.rs | 2 +- crates/storage/provider/src/providers/database/provider.rs | 3 +-- 5 files changed, 5 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5e93ee684dd1..f16049004f9c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -154,6 +154,7 @@ unused_rounding = "warn" useless_let_if_seq = "warn" use_self = "warn" missing_const_for_fn = "warn" +empty_line_after_doc_comments = "warn" # These are nursery lints which have findings. Allow them for now. Some are not # quite mature enough for use in our codebase and some we don't really want. @@ -162,13 +163,11 @@ as_ptr_cast_mut = "allow" cognitive_complexity = "allow" collection_is_never_read = "allow" debug_assert_with_mut_call = "allow" -empty_line_after_doc_comments = "allow" fallible_impl_from = "allow" future_not_send = "allow" iter_on_single_items = "allow" needless_collect = "allow" non_send_fields_in_send_ty = "allow" -option_if_let_else = "allow" redundant_pub_crate = "allow" significant_drop_in_scrutinee = "allow" significant_drop_tightening = "allow" diff --git a/crates/node-core/src/args/types.rs b/crates/node-core/src/args/types.rs index b7fcbe5298ba..4e1d3e08cf59 100644 --- a/crates/node-core/src/args/types.rs +++ b/crates/node-core/src/args/types.rs @@ -3,7 +3,6 @@ use std::{fmt, num::ParseIntError, str::FromStr}; /// A macro that generates types that maps "0" to "None" when parsing CLI arguments. - macro_rules! zero_as_none { ($type_name:ident, $inner_type:ty) => { #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/crates/rpc/ipc/src/server/mod.rs b/crates/rpc/ipc/src/server/mod.rs index 5a154e6f7c15..44b24bf01efe 100644 --- a/crates/rpc/ipc/src/server/mod.rs +++ b/crates/rpc/ipc/src/server/mod.rs @@ -48,8 +48,8 @@ mod ipc; mod rpc_service; /// Ipc Server implementation - -// This is an adapted `jsonrpsee` Server, but for `Ipc` connections. +/// +/// This is an adapted `jsonrpsee` Server, but for `Ipc` connections. pub struct IpcServer { /// The endpoint we listen for incoming transactions endpoint: String, diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index 75b18aaec7e8..0bb398b2aeda 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -1518,7 +1518,7 @@ pub struct RpcServerConfig { jwt_secret: Option, } -/// === impl RpcServerConfig === +// === impl RpcServerConfig === impl RpcServerConfig { /// Creates a new config with only http set diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 79f2960e70de..8cde416d73ff 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -1641,8 +1641,7 @@ impl TransactionsProviderExt for DatabaseProvider { } } -/// Calculates the hash of the given transaction - +// Calculates the hash of the given transaction impl TransactionsProvider for DatabaseProvider { fn transaction_id(&self, tx_hash: TxHash) -> ProviderResult> { Ok(self.tx.get::(tx_hash)?) From b311de91cf532b6bb6586ccb8f8fcbbaeed217ce Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Sat, 1 Jun 2024 18:24:34 +0200 Subject: [PATCH 09/11] small fix in `estimate_gas_with` gas used (#8541) --- crates/rpc/rpc/src/eth/api/call.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc/src/eth/api/call.rs b/crates/rpc/rpc/src/eth/api/call.rs index 4143f11ba65c..7024fa35cbc8 100644 --- a/crates/rpc/rpc/src/eth/api/call.rs +++ b/crates/rpc/rpc/src/eth/api/call.rs @@ -296,7 +296,7 @@ where // // NOTE: this is the gas the transaction used, which is less than the // transaction requires to succeed. - let gas_used = res.result.gas_used(); + let mut gas_used = res.result.gas_used(); // the lowest value is capped by the gas used by the unconstrained transaction let mut lowest_gas_limit = gas_used.saturating_sub(1); @@ -313,6 +313,8 @@ where // Re-execute the transaction with the new gas limit and update the result and // environment. (res, env) = self.transact(&mut db, env)?; + // Update the gas used based on the new result. + gas_used = res.result.gas_used(); // Update the gas limit estimates (highest and lowest) based on the execution result. update_estimated_gas_range( res.result, From e75edf3be7d2fc48e510737a06b8717d26bb09b3 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Sat, 1 Jun 2024 18:30:26 +0200 Subject: [PATCH 10/11] add `iter_on_single_items`clippy lint (#8542) --- Cargo.toml | 2 +- crates/consensus/beacon/src/engine/mod.rs | 4 ++-- crates/trie/trie/src/hashed_cursor/post_state.rs | 11 +++++------ 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f16049004f9c..5772a280dc85 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -155,6 +155,7 @@ useless_let_if_seq = "warn" use_self = "warn" missing_const_for_fn = "warn" empty_line_after_doc_comments = "warn" +iter_on_single_items = "warn" # These are nursery lints which have findings. Allow them for now. Some are not # quite mature enough for use in our codebase and some we don't really want. @@ -165,7 +166,6 @@ collection_is_never_read = "allow" debug_assert_with_mut_call = "allow" fallible_impl_from = "allow" future_not_send = "allow" -iter_on_single_items = "allow" needless_collect = "allow" non_send_fields_in_send_ty = "allow" redundant_pub_crate = "allow" diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index 40ab8447b4ea..35e75320bd60 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -2325,7 +2325,7 @@ mod tests { chain_spec.clone(), StaticFileProvider::read_write(static_dir_path).unwrap(), ), - [&next_head].into_iter(), + std::iter::once(&next_head), ); let expected_result = ForkchoiceUpdated::from_status(PayloadStatusEnum::Syncing); @@ -2695,7 +2695,7 @@ mod tests { chain_spec.clone(), StaticFileProvider::read_write(static_dir_path).unwrap(), ), - [&genesis].into_iter(), + std::iter::once(&genesis), ); let mut engine_rx = spawn_consensus_engine(consensus_engine); diff --git a/crates/trie/trie/src/hashed_cursor/post_state.rs b/crates/trie/trie/src/hashed_cursor/post_state.rs index 100ea131f2df..039ad429d819 100644 --- a/crates/trie/trie/src/hashed_cursor/post_state.rs +++ b/crates/trie/trie/src/hashed_cursor/post_state.rs @@ -689,7 +689,7 @@ mod tests { let tx = db.tx().unwrap(); let factory = HashedPostStateCursorFactory::new(&tx, &sorted); let expected = - [(address, db_storage.into_iter().chain(post_state_storage).collect())].into_iter(); + std::iter::once((address, db_storage.into_iter().chain(post_state_storage).collect())); assert_storage_cursor_order(&factory, expected); } @@ -724,11 +724,10 @@ mod tests { let sorted = hashed_post_state.into_sorted(); let tx = db.tx().unwrap(); let factory = HashedPostStateCursorFactory::new(&tx, &sorted); - let expected = [( + let expected = std::iter::once(( address, post_state_storage.into_iter().filter(|(_, value)| *value > U256::ZERO).collect(), - )] - .into_iter(); + )); assert_storage_cursor_order(&factory, expected); } @@ -762,7 +761,7 @@ mod tests { let sorted = hashed_post_state.into_sorted(); let tx = db.tx().unwrap(); let factory = HashedPostStateCursorFactory::new(&tx, &sorted); - let expected = [(address, post_state_storage)].into_iter(); + let expected = std::iter::once((address, post_state_storage)); assert_storage_cursor_order(&factory, expected); } @@ -797,7 +796,7 @@ mod tests { let sorted = hashed_post_state.into_sorted(); let tx = db.tx().unwrap(); let factory = HashedPostStateCursorFactory::new(&tx, &sorted); - let expected = [(address, storage)].into_iter(); + let expected = std::iter::once((address, storage)); assert_storage_cursor_order(&factory, expected); } From 99c951a8bdde3c1f733e569832a08c75b7183f61 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 2 Jun 2024 09:16:17 +0000 Subject: [PATCH 11/11] chore(deps): weekly `cargo update` (#8543) Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com> --- Cargo.lock | 190 ++++++++++++++++++++++------------------------------- 1 file changed, 78 insertions(+), 112 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 59eb3b42a933..e6b9fdcb6356 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" dependencies = [ "gimli", ] @@ -140,7 +140,7 @@ dependencies = [ [[package]] name = "alloy-consensus" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy#dd7a999d9efe259c47a34dde046952de795a8f6a" +source = "git+https://github.com/alloy-rs/alloy#4ecb7d86882ece8a9a7a5a892b71a3c198030731" dependencies = [ "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy)", "alloy-primitives", @@ -165,7 +165,7 @@ dependencies = [ "itoa", "serde", "serde_json", - "winnow 0.6.8", + "winnow 0.6.9", ] [[package]] @@ -189,7 +189,7 @@ dependencies = [ [[package]] name = "alloy-eips" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy#dd7a999d9efe259c47a34dde046952de795a8f6a" +source = "git+https://github.com/alloy-rs/alloy#4ecb7d86882ece8a9a7a5a892b71a3c198030731" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -214,7 +214,7 @@ dependencies = [ [[package]] name = "alloy-genesis" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy#dd7a999d9efe259c47a34dde046952de795a8f6a" +source = "git+https://github.com/alloy-rs/alloy#4ecb7d86882ece8a9a7a5a892b71a3c198030731" dependencies = [ "alloy-primitives", "alloy-serde 0.1.0 (git+https://github.com/alloy-rs/alloy)", @@ -340,9 +340,9 @@ dependencies = [ [[package]] name = "alloy-rlp" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d58d9f5da7b40e9bfff0b7e7816700be4019db97d4b6359fe7f94a9e22e42ac" +checksum = "b155716bab55763c95ba212806cf43d05bcc70e5f35b02bad20cf5ec7fe11fed" dependencies = [ "alloy-rlp-derive", "arrayvec", @@ -351,9 +351,9 @@ dependencies = [ [[package]] name = "alloy-rlp-derive" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a047897373be4bbb0224c1afdabca92648dc57a9c9ef6e7b0be3aff7a859c83" +checksum = "8037e03c7f462a063f28daec9fda285a9a89da003c552f8637a80b9c8fd96241" dependencies = [ "proc-macro2", "quote", @@ -405,7 +405,7 @@ dependencies = [ [[package]] name = "alloy-rpc-types" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy#dd7a999d9efe259c47a34dde046952de795a8f6a" +source = "git+https://github.com/alloy-rs/alloy#4ecb7d86882ece8a9a7a5a892b71a3c198030731" dependencies = [ "alloy-consensus 0.1.0 (git+https://github.com/alloy-rs/alloy)", "alloy-eips 0.1.0 (git+https://github.com/alloy-rs/alloy)", @@ -486,7 +486,7 @@ dependencies = [ [[package]] name = "alloy-serde" version = "0.1.0" -source = "git+https://github.com/alloy-rs/alloy#dd7a999d9efe259c47a34dde046952de795a8f6a" +source = "git+https://github.com/alloy-rs/alloy#4ecb7d86882ece8a9a7a5a892b71a3c198030731" dependencies = [ "alloy-primitives", "serde", @@ -579,7 +579,7 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "368cae4dc052cad1d8f72eb2ae0c38027116933eeb49213c200a9e9875f208d7" dependencies = [ - "winnow 0.6.8", + "winnow 0.6.9", ] [[package]] @@ -907,22 +907,21 @@ dependencies = [ [[package]] name = "async-channel" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f2776ead772134d55b62dd45e59a79e21612d85d0af729b8b7d3967d601a62a" +checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" dependencies = [ "concurrent-queue", - "event-listener 5.3.0", - "event-listener-strategy 0.5.2", + "event-listener-strategy", "futures-core", "pin-project-lite", ] [[package]] name = "async-compression" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c90a406b4495d129f00461241616194cb8a032c8d1c53c657f0961d5f8e0498" +checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" dependencies = [ "brotli", "flate2", @@ -934,17 +933,6 @@ dependencies = [ "zstd-safe", ] -[[package]] -name = "async-lock" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" -dependencies = [ - "event-listener 4.0.3", - "event-listener-strategy 0.4.0", - "pin-project-lite", -] - [[package]] name = "async-sse" version = "5.1.0" @@ -1045,9 +1033,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.71" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +checksum = "17c6a35df3749d2e8bb1b7b21a976d82b15548788d2735b9d82f329268f71a11" dependencies = [ "addr2line", "cc", @@ -1259,12 +1247,11 @@ dependencies = [ [[package]] name = "blocking" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "495f7104e962b7356f0aeb34247aca1fe7d2e783b346582db7f2904cb5717e88" +checksum = "703f41c54fc768e63e091340b424302bb1c29ef4aa0c7f10fe849dfb114d29ea" dependencies = [ - "async-channel 2.3.0", - "async-lock", + "async-channel 2.3.1", "async-task", "futures-io", "futures-lite 2.3.0", @@ -1273,9 +1260,9 @@ dependencies = [ [[package]] name = "blst" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c94087b935a822949d3291a9989ad2b2051ea141eda0fd4e478a75f6aa3e604b" +checksum = "62dc83a094a71d43eeadd254b1ec2d24cb6a0bb6cadce00df51f0db594711a32" dependencies = [ "cc", "glob", @@ -1429,9 +1416,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6221fe77a248b9117d431ad93761222e1cf8ff282d9d1d5d9f53d6299a1cf76" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1496,9 +1483,9 @@ dependencies = [ [[package]] name = "bytemuck_derive" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "369cfaf2a5bed5d8f8202073b2e093c9f508251de1551a0deb4253e4c7d80909" +checksum = "1ee891b04274a59bd38b412188e24b849617b2e45a0fd8d057deb63e7403761b" dependencies = [ "proc-macro2", "quote", @@ -1676,9 +1663,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", @@ -1852,9 +1839,9 @@ dependencies = [ [[package]] name = "const-hex" -version = "1.11.4" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ff96486ccc291d36a958107caf2c0af8c78c0af7d31ae2f35ce055130de1a6" +checksum = "94fb8a24a26d37e1ffd45343323dc9fe6654ceea44c12f2fcb3d7ac29e610bc6" dependencies = [ "cfg-if", "cpufeatures", @@ -2764,9 +2751,9 @@ dependencies = [ [[package]] name = "ethereum_ssz" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e61ffea29f26e8249d35128a82ec8d3bd4fbc80179ea5f5e5e3daafef6a80fcb" +checksum = "7d3627f83d8b87b432a5fad9934b4565260722a141a2c40f371f8080adec9425" dependencies = [ "ethereum-types", "itertools 0.10.5", @@ -2781,43 +2768,22 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-listener" -version = "4.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener" -version = "5.3.0" +version = "5.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9944b8ca13534cdfb2800775f8dd4902ff3fc75a50101466decadfdf322a24" +checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" dependencies = [ "concurrent-queue", "parking", "pin-project-lite", ] -[[package]] -name = "event-listener-strategy" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" -dependencies = [ - "event-listener 4.0.3", - "pin-project-lite", -] - [[package]] name = "event-listener-strategy" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" dependencies = [ - "event-listener 5.3.0", + "event-listener 5.3.1", "pin-project-lite", ] @@ -3224,9 +3190,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" [[package]] name = "glob" @@ -3663,9 +3629,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d8d52be92d09acc2e01dddb7fde3ad983fc6489c7db4837e605bc3fca4cb63e" +checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" dependencies = [ "bytes", "futures-channel", @@ -3786,9 +3752,9 @@ checksum = "545c6c3e8bf9580e2dafee8de6f9ec14826aaf359787789c7724f1f85f47d3dc" [[package]] name = "icu_normalizer" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "183072b0ba2f336279c830a3d594a04168494a726c3c94b50c53d788178cf2c2" +checksum = "accb85c5b2e76f8dade22978b3795ae1e550198c6cfc7e915144e17cd6e2ab56" dependencies = [ "displaydoc", "icu_collections", @@ -3810,9 +3776,9 @@ checksum = "e3744fecc0df9ce19999cdaf1f9f3a48c253431ce1d67ef499128fe9d0b607ab" [[package]] name = "icu_properties" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a89401989d8fdf571b829ce1022801367ec89affc7b1e162b79eff4ae029e69" +checksum = "d8173ba888885d250016e957b8ebfd5a65cdb690123d8833a19f6833f9c2b579" dependencies = [ "displaydoc", "icu_collections", @@ -4634,9 +4600,9 @@ checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "litemap" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d642685b028806386b2b6e75685faadd3eb65a85fff7df711ce18446a422da" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" [[package]] name = "lock_api" @@ -5210,9 +5176,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.2" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "b8ec7ab813848ba4522158d5517a6093db1ded27575b070f4177b8d12b41db5e" dependencies = [ "memchr", ] @@ -5506,9 +5472,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "piper" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464db0c665917b13ebb5d453ccdec4add5658ee1adc7affc7677615356a8afaf" +checksum = "ae1d5c74c9876f070d3e8fd503d748c7d974c3e48da8f41350fa5222ef9b4391" dependencies = [ "atomic-waker", "fastrand 2.1.0", @@ -9033,11 +8999,11 @@ dependencies = [ [[package]] name = "strum_macros" -version = "0.26.2" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6cf59daf282c0a494ba14fd21610a0325f9f90ec9d1231dea26bcb1d696c946" +checksum = "f7993a8e3a9e88a00351486baae9522c91b123a088f76469e5bd5cc17198ea87" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "rustversion", @@ -9373,9 +9339,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83c02bf3c538ab32ba913408224323915f4ef9a6d61c0e85d493f355921c0ece" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" dependencies = [ "displaydoc", "zerovec", @@ -9414,9 +9380,9 @@ checksum = "c7c4ceeeca15c8384bbc3e011dbd8fccb7f068a440b752b7d9b32ceb0ca0e2e8" [[package]] name = "tokio" -version = "1.37.0" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "bytes", @@ -9433,9 +9399,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", @@ -9532,7 +9498,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.8", + "winnow 0.6.9", ] [[package]] @@ -10403,9 +10369,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.8" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c52e9c97a68071b23e836c9380edae937f17b9c4667bd021973efc689f618d" +checksum = "86c949fede1d13936a99f14fafd3e76fd642b556dd2ce96287fbe2e0151bfac6" dependencies = [ "memchr", ] @@ -10438,9 +10404,9 @@ checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" [[package]] name = "writeable" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad7bb64b8ef9c0aa27b6da38b452b0ee9fd82beaf276a87dd796fb55cbae14e" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" [[package]] name = "wyhash" @@ -10462,9 +10428,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e71b2e4f287f467794c671e2b8f8a5f3716b3c829079a1c44740148eff07e4" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" dependencies = [ "serde", "stable_deref_trait", @@ -10474,9 +10440,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e6936f0cce458098a201c245a11bef556c6a0181129c7034d10d76d1ec3a2b8" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" dependencies = [ "proc-macro2", "quote", @@ -10506,18 +10472,18 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "655b0814c5c0b19ade497851070c640773304939a6c0fd5f5fb43da0696d05b7" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6a647510471d372f2e6c2e6b7219e44d8c574d24fdc11c610a61455782f18c3" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" dependencies = [ "proc-macro2", "quote", @@ -10547,9 +10513,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff4439ae91fb5c72b8abc12f3f2dbf51bd27e6eadb9f8a5bc8898dddb0e27ea" +checksum = "bb2cc8827d6c0994478a15c53f374f46fbd41bea663d809b14744bc42e6b109c" dependencies = [ "yoke", "zerofrom", @@ -10558,9 +10524,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4e5997cbf58990550ef1f0e5124a05e47e1ebd33a84af25739be6031a62c20" +checksum = "97cf56601ee5052b4417d90c8755c6683473c926039908196cf35d99f893ebe7" dependencies = [ "proc-macro2", "quote",