From ed18a73939f89a1796e1bd0b326249fb2adf6e67 Mon Sep 17 00:00:00 2001 From: Preston Evans Date: Mon, 11 Dec 2023 16:19:43 -0600 Subject: [PATCH 1/5] impl HasPreimage for StateDB --- full-node/db/sov-db/src/state_db.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/full-node/db/sov-db/src/state_db.rs b/full-node/db/sov-db/src/state_db.rs index 8b3b1dc5b..848bb6f9f 100644 --- a/full-node/db/sov-db/src/state_db.rs +++ b/full-node/db/sov-db/src/state_db.rs @@ -1,7 +1,7 @@ use std::path::Path; use std::sync::{Arc, Mutex}; -use jmt::storage::{TreeReader, TreeWriter}; +use jmt::storage::{HasPreimage, TreeReader, TreeWriter}; use jmt::{KeyHash, Version}; use sov_schema_db::{SchemaBatch, DB}; @@ -153,6 +153,12 @@ impl TreeWriter for StateDB { } } +impl HasPreimage for StateDB { + fn preimage(&self, key_hash: KeyHash) -> anyhow::Result>> { + self.db.get::(&key_hash.0) + } +} + #[cfg(feature = "arbitrary")] pub mod arbitrary { //! Arbitrary definitions for the [`StateDB`]. From 6d4e07efb33b8be5e384d7610570882c883225a3 Mon Sep 17 00:00:00 2001 From: Preston Evans Date: Wed, 13 Dec 2023 14:18:05 -0600 Subject: [PATCH 2/5] Minor db api changes --- full-node/db/sov-db/src/ledger_db/mod.rs | 6 ++--- full-node/db/sov-db/src/native_db.rs | 2 +- full-node/db/sov-db/src/state_db.rs | 4 ++-- full-node/db/sov-schema-db/src/iterator.rs | 24 ++++++++++++++++--- full-node/db/sov-schema-db/src/lib.rs | 2 ++ full-node/db/sov-schema-db/tests/db_test.rs | 4 +++- .../db/sov-schema-db/tests/iterator_test.rs | 2 +- 7 files changed, 33 insertions(+), 11 deletions(-) diff --git a/full-node/db/sov-db/src/ledger_db/mod.rs b/full-node/db/sov-db/src/ledger_db/mod.rs index 801f1d334..21653b460 100644 --- a/full-node/db/sov-db/src/ledger_db/mod.rs +++ b/full-node/db/sov-db/src/ledger_db/mod.rs @@ -166,7 +166,7 @@ impl LedgerDB { let iter = raw_iter.take(max_items); let mut out = Vec::with_capacity(max_items); for res in iter { - let (_, batch) = res?; + let batch = res?.value; out.push(batch) } Ok(out) @@ -307,7 +307,7 @@ impl LedgerDB { iter.seek_to_last(); match iter.next() { - Some(Ok((version, _))) => Ok(Some(version.into())), + Some(Ok(item)) => Ok(Some(item.key.into())), Some(Err(e)) => Err(e), _ => Ok(None), } @@ -319,7 +319,7 @@ impl LedgerDB { iter.seek_to_last(); match iter.next() { - Some(Ok((slot_number, slot))) => Ok(Some((slot_number, slot))), + Some(Ok(item)) => Ok(Some(item.to_tuple())), Some(Err(e)) => Err(e), _ => Ok(None), } diff --git a/full-node/db/sov-db/src/native_db.rs b/full-node/db/sov-db/src/native_db.rs index 14b41a6cb..3ec41e85c 100644 --- a/full-node/db/sov-db/src/native_db.rs +++ b/full-node/db/sov-db/src/native_db.rs @@ -50,7 +50,7 @@ impl NativeDB { let found = iter.next(); match found { Some(result) => { - let ((found_key, found_version), value) = result?; + let ((found_key, found_version), value) = result?.to_tuple(); if &found_key == key { anyhow::ensure!(found_version <= version, "Bug! iterator isn't returning expected values. expected a version <= {version:} but found {found_version:}"); Ok(value) diff --git a/full-node/db/sov-db/src/state_db.rs b/full-node/db/sov-db/src/state_db.rs index 848bb6f9f..6c005fd7b 100644 --- a/full-node/db/sov-db/src/state_db.rs +++ b/full-node/db/sov-db/src/state_db.rs @@ -69,7 +69,7 @@ impl StateDB { let found = iter.next(); match found { Some(result) => { - let ((found_key, found_version), value) = result?; + let ((found_key, found_version), value) = result?.to_tuple(); if &found_key == key { anyhow::ensure!(found_version <= version, "Bug! iterator isn't returning expected values. expected a version <= {version:} but found {found_version:}"); Ok(value) @@ -98,7 +98,7 @@ impl StateDB { iter.seek_to_last(); let version = match iter.next() { - Some(Ok((key, _))) => Some(key.version()), + Some(Ok(item)) => Some(item.key.version()), _ => None, }; Ok(version) diff --git a/full-node/db/sov-schema-db/src/iterator.rs b/full-node/db/sov-schema-db/src/iterator.rs index 8005736b8..7aeb6a7c5 100644 --- a/full-node/db/sov-schema-db/src/iterator.rs +++ b/full-node/db/sov-schema-db/src/iterator.rs @@ -95,7 +95,7 @@ where } } - fn next_impl(&mut self) -> Result> { + fn next_impl(&mut self) -> Result>> { let _timer = SCHEMADB_ITER_LATENCY_SECONDS .with_label_values(&[S::COLUMN_FAMILY_NAME]) .start_timer(); @@ -107,6 +107,7 @@ where let raw_key = self.db_iter.key().expect("db_iter.key() failed."); let raw_value = self.db_iter.value().expect("db_iter.value() failed."); + let value_size_bytes = raw_value.len(); SCHEMADB_ITER_BYTES .with_label_values(&[S::COLUMN_FAMILY_NAME]) .observe((raw_key.len() + raw_value.len()) as f64); @@ -119,7 +120,24 @@ where ScanDirection::Backward => self.db_iter.prev(), } - Ok(Some((key, value))) + Ok(Some(IteratorOutput { + key, + value, + value_size_bytes, + })) + } +} + +/// The output of [`SchemaIterator`]'s next_impl +pub struct IteratorOutput { + pub key: K, + pub value: V, + pub value_size_bytes: usize, +} + +impl IteratorOutput { + pub fn to_tuple(self) -> (K, V) { + (self.key, self.value) } } @@ -127,7 +145,7 @@ impl<'a, S> Iterator for SchemaIterator<'a, S> where S: Schema, { - type Item = Result<(S::Key, S::Value)>; + type Item = Result>; fn next(&mut self) -> Option { self.next_impl().transpose() diff --git a/full-node/db/sov-schema-db/src/lib.rs b/full-node/db/sov-schema-db/src/lib.rs index 021ad7951..cfd08895a 100644 --- a/full-node/db/sov-schema-db/src/lib.rs +++ b/full-node/db/sov-schema-db/src/lib.rs @@ -40,6 +40,8 @@ pub use crate::schema::Schema; use crate::schema::{ColumnFamilyName, KeyCodec, ValueCodec}; pub use crate::schema_batch::{SchemaBatch, SchemaBatchIterator}; +pub use rocksdb; + /// This DB is a schematized RocksDB wrapper where all data passed in and out are typed according to /// [`Schema`]s. #[derive(Debug)] diff --git a/full-node/db/sov-schema-db/tests/db_test.rs b/full-node/db/sov-schema-db/tests/db_test.rs index 8cf83851c..069f78427 100644 --- a/full-node/db/sov-schema-db/tests/db_test.rs +++ b/full-node/db/sov-schema-db/tests/db_test.rs @@ -132,7 +132,9 @@ fn test_schema_put_get() { fn collect_values(db: &TestDB) -> Vec<(S::Key, S::Value)> { let mut iter = db.iter::().expect("Failed to create iterator."); iter.seek_to_first(); - iter.collect::, anyhow::Error>>().unwrap() + iter.map(|res| res.map(|item| item.to_tuple())) + .collect::, anyhow::Error>>() + .unwrap() } fn gen_expected_values(values: &[(u32, u32)]) -> Vec<(TestField, TestField)> { diff --git a/full-node/db/sov-schema-db/tests/iterator_test.rs b/full-node/db/sov-schema-db/tests/iterator_test.rs index 7e8f3c42b..9b6103051 100644 --- a/full-node/db/sov-schema-db/tests/iterator_test.rs +++ b/full-node/db/sov-schema-db/tests/iterator_test.rs @@ -19,7 +19,7 @@ define_schema!(TestSchema, TestCompositeField, TestField, "TestCF"); type S = TestSchema; fn collect_values(iter: SchemaIterator) -> Vec { - iter.map(|row| row.unwrap().1 .0).collect() + iter.map(|row| row.unwrap().value.0).collect() } fn decode_key(key: &[u8]) -> TestCompositeField { From eb7588f4f68f3ff92ef330197f922e14f9657c1a Mon Sep 17 00:00:00 2001 From: Preston Evans Date: Wed, 13 Dec 2023 17:38:56 -0600 Subject: [PATCH 3/5] Avoid unnecessary allocation on db::get --- full-node/db/sov-schema-db/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/full-node/db/sov-schema-db/src/lib.rs b/full-node/db/sov-schema-db/src/lib.rs index cfd08895a..7dda85453 100644 --- a/full-node/db/sov-schema-db/src/lib.rs +++ b/full-node/db/sov-schema-db/src/lib.rs @@ -129,7 +129,7 @@ impl DB { let k = schema_key.encode_key()?; let cf_handle = self.get_cf_handle(S::COLUMN_FAMILY_NAME)?; - let result = self.inner.get_cf(cf_handle, k)?; + let result = self.inner.get_pinned_cf(cf_handle, k)?; SCHEMADB_GET_BYTES .with_label_values(&[S::COLUMN_FAMILY_NAME]) .observe(result.as_ref().map_or(0.0, |v| v.len() as f64)); From ab8bbec42056cffcadcc3d947985dde4e547975d Mon Sep 17 00:00:00 2001 From: Preston Evans Date: Wed, 13 Dec 2023 19:31:38 -0600 Subject: [PATCH 4/5] lint --- full-node/db/sov-schema-db/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/full-node/db/sov-schema-db/src/lib.rs b/full-node/db/sov-schema-db/src/lib.rs index 7dda85453..1a80bfd8a 100644 --- a/full-node/db/sov-schema-db/src/lib.rs +++ b/full-node/db/sov-schema-db/src/lib.rs @@ -31,6 +31,7 @@ use metrics::{ SCHEMADB_BATCH_COMMIT_BYTES, SCHEMADB_BATCH_COMMIT_LATENCY_SECONDS, SCHEMADB_DELETES, SCHEMADB_GET_BYTES, SCHEMADB_GET_LATENCY_SECONDS, SCHEMADB_PUT_BYTES, }; +pub use rocksdb; use rocksdb::ReadOptions; pub use rocksdb::DEFAULT_COLUMN_FAMILY_NAME; use thiserror::Error; @@ -40,8 +41,6 @@ pub use crate::schema::Schema; use crate::schema::{ColumnFamilyName, KeyCodec, ValueCodec}; pub use crate::schema_batch::{SchemaBatch, SchemaBatchIterator}; -pub use rocksdb; - /// This DB is a schematized RocksDB wrapper where all data passed in and out are typed according to /// [`Schema`]s. #[derive(Debug)] From 946746dbdd8e0610e5232bea85e7784ced74407f Mon Sep 17 00:00:00 2001 From: Preston Evans Date: Wed, 13 Dec 2023 19:39:00 -0600 Subject: [PATCH 5/5] Rename to_tuple to into_tuple --- full-node/db/sov-db/src/ledger_db/mod.rs | 2 +- full-node/db/sov-db/src/native_db.rs | 2 +- full-node/db/sov-db/src/state_db.rs | 2 +- full-node/db/sov-schema-db/src/iterator.rs | 2 +- full-node/db/sov-schema-db/tests/db_test.rs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/full-node/db/sov-db/src/ledger_db/mod.rs b/full-node/db/sov-db/src/ledger_db/mod.rs index 21653b460..a9c731f55 100644 --- a/full-node/db/sov-db/src/ledger_db/mod.rs +++ b/full-node/db/sov-db/src/ledger_db/mod.rs @@ -319,7 +319,7 @@ impl LedgerDB { iter.seek_to_last(); match iter.next() { - Some(Ok(item)) => Ok(Some(item.to_tuple())), + Some(Ok(item)) => Ok(Some(item.into_tuple())), Some(Err(e)) => Err(e), _ => Ok(None), } diff --git a/full-node/db/sov-db/src/native_db.rs b/full-node/db/sov-db/src/native_db.rs index 3ec41e85c..2003bb7e4 100644 --- a/full-node/db/sov-db/src/native_db.rs +++ b/full-node/db/sov-db/src/native_db.rs @@ -50,7 +50,7 @@ impl NativeDB { let found = iter.next(); match found { Some(result) => { - let ((found_key, found_version), value) = result?.to_tuple(); + let ((found_key, found_version), value) = result?.into_tuple(); if &found_key == key { anyhow::ensure!(found_version <= version, "Bug! iterator isn't returning expected values. expected a version <= {version:} but found {found_version:}"); Ok(value) diff --git a/full-node/db/sov-db/src/state_db.rs b/full-node/db/sov-db/src/state_db.rs index 6c005fd7b..0f5d2f380 100644 --- a/full-node/db/sov-db/src/state_db.rs +++ b/full-node/db/sov-db/src/state_db.rs @@ -69,7 +69,7 @@ impl StateDB { let found = iter.next(); match found { Some(result) => { - let ((found_key, found_version), value) = result?.to_tuple(); + let ((found_key, found_version), value) = result?.into_tuple(); if &found_key == key { anyhow::ensure!(found_version <= version, "Bug! iterator isn't returning expected values. expected a version <= {version:} but found {found_version:}"); Ok(value) diff --git a/full-node/db/sov-schema-db/src/iterator.rs b/full-node/db/sov-schema-db/src/iterator.rs index 7aeb6a7c5..ed792e3d4 100644 --- a/full-node/db/sov-schema-db/src/iterator.rs +++ b/full-node/db/sov-schema-db/src/iterator.rs @@ -136,7 +136,7 @@ pub struct IteratorOutput { } impl IteratorOutput { - pub fn to_tuple(self) -> (K, V) { + pub fn into_tuple(self) -> (K, V) { (self.key, self.value) } } diff --git a/full-node/db/sov-schema-db/tests/db_test.rs b/full-node/db/sov-schema-db/tests/db_test.rs index 069f78427..5a9b0e0c7 100644 --- a/full-node/db/sov-schema-db/tests/db_test.rs +++ b/full-node/db/sov-schema-db/tests/db_test.rs @@ -132,7 +132,7 @@ fn test_schema_put_get() { fn collect_values(db: &TestDB) -> Vec<(S::Key, S::Value)> { let mut iter = db.iter::().expect("Failed to create iterator."); iter.seek_to_first(); - iter.map(|res| res.map(|item| item.to_tuple())) + iter.map(|res| res.map(|item| item.into_tuple())) .collect::, anyhow::Error>>() .unwrap() }