From 6c8401967bba87bcf0a97d83b03f05f33169d710 Mon Sep 17 00:00:00 2001 From: cchudant Date: Sat, 28 Sep 2024 14:05:37 +0000 Subject: [PATCH] fix: sneaky little bug --- Cargo.toml | 4 - src/lib.rs | 18 +-- src/tests/mod.rs | 1 - src/tests/proof.rs | 371 -------------------------------------------- src/tests/simple.rs | 3 + src/trie/proof.rs | 6 +- src/trie/tree.rs | 2 + src/trie/trees.rs | 15 +- 8 files changed, 30 insertions(+), 390 deletions(-) delete mode 100644 src/tests/proof.rs diff --git a/Cargo.toml b/Cargo.toml index 220450c..d9384cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,10 +49,6 @@ rocksdb = { optional = true, version = "0.21.0", features = [ env_logger = "0.11.3" once_cell = "1.19.0" pprof = { version = "0.3", features = ["flamegraph"] } -pathfinder-common = { git = "https://github.com/massalabs/pathfinder.git", package = "pathfinder-common", rev = "b7b6d76a76ab0e10f92e5f84ce099b5f727cb4db" } -pathfinder-crypto = { git = "https://github.com/massalabs/pathfinder.git", package = "pathfinder-crypto", rev = "b7b6d76a76ab0e10f92e5f84ce099b5f727cb4db" } -pathfinder-merkle-tree = { git = "https://github.com/massalabs/pathfinder.git", package = "pathfinder-merkle-tree", rev = "b7b6d76a76ab0e10f92e5f84ce099b5f727cb4db" } -pathfinder-storage = { git = "https://github.com/massalabs/pathfinder.git", package = "pathfinder-storage", rev = "b7b6d76a76ab0e10f92e5f84ce099b5f727cb4db" } rand = { version = "0.8.5", features = ["small_rng"] } tempfile = "3.8.0" rstest = "0.18.2" diff --git a/src/lib.rs b/src/lib.rs index 9a179a0..144764e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -153,7 +153,7 @@ impl EncodeExt for T {} use changes::ChangeBatch; use key_value_db::KeyValueDB; use starknet_types_core::{felt::Felt, hash::StarkHash}; -use trie::{tree::bytes_to_bitvec, trees::MerkleTrees}; +use trie::{proof::MultiProof, tree::bytes_to_bitvec, trees::MerkleTrees}; /// Structure that contains the configuration for the BonsaiStorage. /// A default implementation is provided with coherent values. @@ -473,15 +473,13 @@ where self.tries.db_ref().get_latest_id() } - // /// Verifies a merkle-proof for a given `key` and `value`. - // pub fn verify_proof( - // root: Felt, - // key: &BitSlice, - // value: Felt, - // proofs: &[ProofNode], - // ) -> Option { - // MerkleTree::::verify_proof(root, key, value, proofs) - // } + pub fn get_multi_proof( + &mut self, + identifier: &[u8], + keys: impl IntoIterator>, + ) -> Result> { + self.tries.get_multi_proof(identifier, keys) + } } impl BonsaiStorage diff --git a/src/tests/mod.rs b/src/tests/mod.rs index 78efcf3..da125d9 100644 --- a/src/tests/mod.rs +++ b/src/tests/mod.rs @@ -1,6 +1,5 @@ mod madara_comparison; mod merge; -// mod proof; mod merkle_tree; mod proptest; mod simple; diff --git a/src/tests/proof.rs b/src/tests/proof.rs deleted file mode 100644 index 1a8b350..0000000 --- a/src/tests/proof.rs +++ /dev/null @@ -1,371 +0,0 @@ -#![cfg(all(feature = "std", feature = "rocksdb"))] -use pathfinder_common::{hash::PedersenHash, trie::TrieNode}; -use pathfinder_crypto::Felt as PathfinderFelt; -use pathfinder_merkle_tree::tree::{MerkleTree, TestStorage}; -use pathfinder_storage::{Node, StoredNode}; -use rand::Rng; -use starknet_types_core::{felt::Felt, hash::Pedersen}; -use std::collections::HashMap; - -use crate::{ - databases::{create_rocks_db, RocksDB, RocksDBConfig}, - id::{BasicId, BasicIdBuilder}, - trie::merkle_tree::{Membership, ProofNode}, - BonsaiStorage, BonsaiStorageConfig, -}; - -/// Commits the tree changes and persists them to storage. -fn commit_and_persist( - tree: MerkleTree, - storage: &mut TestStorage, -) -> (PathfinderFelt, u64) { - use pathfinder_storage::Child; - - for (key, value) in &tree.leaves { - let key = PathfinderFelt::from_bits(key).unwrap(); - storage.leaves.insert(key, *value); - } - - let update = tree.commit(storage).unwrap(); - - let mut indices = HashMap::new(); - let mut idx = storage.nodes.len(); - for hash in update.nodes.keys() { - indices.insert(*hash, idx as u64); - idx += 1; - } - - for (hash, node) in update.nodes { - let node = match node { - Node::Binary { left, right } => { - let left = match left { - Child::Id(idx) => idx, - Child::Hash(hash) => { - *indices.get(&hash).expect("Left child should have an index") - } - }; - - let right = match right { - Child::Id(idx) => idx, - Child::Hash(hash) => *indices - .get(&hash) - .expect("Right child should have an index"), - }; - - StoredNode::Binary { left, right } - } - Node::Edge { child, path } => { - let child = match child { - Child::Id(idx) => idx, - Child::Hash(hash) => *indices.get(&hash).expect("Child should have an index"), - }; - - StoredNode::Edge { child, path } - } - Node::LeafBinary => StoredNode::LeafBinary, - Node::LeafEdge { path } => StoredNode::LeafEdge { path }, - }; - - storage - .nodes - .insert(*indices.get(&hash).unwrap(), (hash, node)); - } - - let index = *indices.get(&update.root).unwrap(); - - (update.root, index) -} - -fn assert_eq_proof(bonsai_proof: &[ProofNode], pathfinder_proof: &[TrieNode]) { - for (bonsai_node, pathfinder_node) in bonsai_proof.iter().zip(pathfinder_proof.iter()) { - match (bonsai_node, pathfinder_node) { - ( - ProofNode::Binary { left, right }, - pathfinder_common::trie::TrieNode::Binary { - left: pathfinder_left, - right: pathfinder_right, - }, - ) => { - let pathfinder_left_bits = pathfinder_left.to_hex_str(); - let pathfinder_felt = Felt::from_hex(&pathfinder_left_bits).unwrap(); - assert_eq!(left, &pathfinder_felt); - let pathfinder_right_bits = pathfinder_right.to_hex_str(); - let pathfinder_felt = Felt::from_hex(&pathfinder_right_bits).unwrap(); - assert_eq!(right, &pathfinder_felt); - } - ( - ProofNode::Edge { child, path }, - pathfinder_common::trie::TrieNode::Edge { - child: pathfinder_child, - path: pathfinder_path, - }, - ) => { - let pathfinder_child_bits = pathfinder_child.to_hex_str(); - let pathfinder_felt = Felt::from_hex(&pathfinder_child_bits).unwrap(); - assert_eq!(child, &pathfinder_felt); - assert_eq!(&path.0, pathfinder_path); - } - _ => panic!("Proofs are not the same"), - } - } -} - -#[test] -fn debug_deoxys() { - // Load storage_data.json file - let storage_data = include_str!("storage_data.json"); - let storage_data: Vec> = serde_json::from_str(storage_data).unwrap(); - let tempdir = tempfile::tempdir().unwrap(); - let db = create_rocks_db(tempdir.path()).unwrap(); - let config = BonsaiStorageConfig::default(); - let mut storage = pathfinder_merkle_tree::tree::TestStorage::default(); - let mut id_builder = BasicIdBuilder::new(); - let mut bonsai_storage = - BonsaiStorage::<_, _, Pedersen>::new(RocksDB::new(&db, RocksDBConfig::default()), config) - .unwrap(); - let mut pathfinder_merkle_tree: MerkleTree = - pathfinder_merkle_tree::tree::MerkleTree::empty(); - let identifier = - Felt::from_hex("0x04acd4b2a59eae7196f6a5c26ead8cb5f9d7ad3d911096418a23357bb2eac075") - .unwrap() - .to_bytes_be() - .to_vec(); - for block_changes in storage_data.iter() { - for pair in block_changes.iter() { - let key = keyer(Felt::from_hex(&pair.0).unwrap()); - let value = Felt::from_hex(&pair.1).unwrap(); - bonsai_storage.insert(&identifier, &key, &value).unwrap(); - pathfinder_merkle_tree - .set( - &storage, - key, - PathfinderFelt::from_hex_str(&pair.1).unwrap(), - ) - .unwrap(); - } - bonsai_storage.commit(id_builder.new_id()).unwrap(); - let (_, root_id) = commit_and_persist(pathfinder_merkle_tree.clone(), &mut storage); - let pathfinder_root = storage.nodes.get(&root_id).unwrap().0; - let bonsai_root = bonsai_storage.root_hash(&identifier).unwrap(); - println!("{:#02x}", bonsai_root); - println!("{:#02x}", pathfinder_root); - assert_eq!(pathfinder_root.to_be_bytes(), bonsai_root.to_bytes_be()); - } -} - -fn keyer(felt: Felt) -> BitVec { - felt.to_bytes_be().view_bits()[5..].to_bitvec() -} - -#[test] -fn basic_proof() { - let identifier = vec![]; - let tempdir = tempfile::tempdir().unwrap(); - let db = create_rocks_db(tempdir.path()).unwrap(); - let config = BonsaiStorageConfig::default(); - let mut storage = pathfinder_merkle_tree::tree::TestStorage::default(); - let mut bonsai_storage = - BonsaiStorage::<_, _, Pedersen>::new(RocksDB::new(&db, RocksDBConfig::default()), config) - .unwrap(); - let mut pathfinder_merkle_tree: MerkleTree = - pathfinder_merkle_tree::tree::MerkleTree::empty(); - let mut id_builder = BasicIdBuilder::new(); - let pair1 = ( - vec![1, 2, 1], - Felt::from_hex("0x66342762FDD54D033c195fec3ce2568b62052e").unwrap(), - ); - let bitvec = BitVec::from_vec(pair1.0.clone()); - bonsai_storage - .insert(&identifier, &bitvec, &pair1.1) - .unwrap(); - pathfinder_merkle_tree - .set( - &storage, - bitvec, - PathfinderFelt::from_hex_str("0x66342762FDD54D033c195fec3ce2568b62052e").unwrap(), - ) - .unwrap(); - let pair2 = ( - vec![1, 2, 2], - Felt::from_hex("0x66342762FD54D033c195fec3ce2568b62052e").unwrap(), - ); - let bitvec = BitVec::from_vec(pair2.0.clone()); - bonsai_storage - .insert(&identifier, &bitvec, &pair2.1) - .unwrap(); - pathfinder_merkle_tree - .set( - &storage, - bitvec, - PathfinderFelt::from_hex_str("0x66342762FD54D033c195fec3ce2568b62052e").unwrap(), - ) - .unwrap(); - let pair3 = ( - vec![1, 2, 3], - Felt::from_hex("0x66342762FD54D033c195fec3ce2568b62052e").unwrap(), - ); - let bitvec = BitVec::from_vec(pair3.0.clone()); - bonsai_storage - .insert(&identifier, &bitvec, &pair3.1) - .unwrap(); - pathfinder_merkle_tree - .set( - &storage, - bitvec, - PathfinderFelt::from_hex_str("0x66342762FD54D033c195fec3ce2568b62052e").unwrap(), - ) - .unwrap(); - bonsai_storage.commit(id_builder.new_id()).unwrap(); - let (_, root_id) = commit_and_persist(pathfinder_merkle_tree.clone(), &mut storage); - let bonsai_proof = bonsai_storage - .get_proof(&identifier, &BitVec::from_vec(vec![1, 2, 1])) - .unwrap(); - let pathfinder_proof = - pathfinder_merkle_tree::tree::MerkleTree::::get_proof( - root_id, - &storage, - &BitVec::from_vec(vec![1, 2, 1]), - ) - .unwrap(); - assert_eq_proof(&bonsai_proof, &pathfinder_proof); - assert_eq!( - BonsaiStorage::, Pedersen>::verify_proof( - bonsai_storage.root_hash(&identifier).unwrap(), - &BitVec::from_vec(vec![1, 2, 1]), - pair1.1, - &bonsai_proof - ), - Some(Membership::Member) - ); -} - -#[test] -fn multiple_proofs() { - let identifier = vec![]; - let tempdir = tempfile::tempdir().unwrap(); - let db = create_rocks_db(tempdir.path()).unwrap(); - let config = BonsaiStorageConfig::default(); - let mut storage = pathfinder_merkle_tree::tree::TestStorage::default(); - let mut bonsai_storage = - BonsaiStorage::<_, _, Pedersen>::new(RocksDB::new(&db, RocksDBConfig::default()), config) - .unwrap(); - let mut pathfinder_merkle_tree: MerkleTree = - pathfinder_merkle_tree::tree::MerkleTree::empty(); - let mut id_builder = BasicIdBuilder::new(); - let mut rng = rand::thread_rng(); - let tree_size = rng.gen_range(10..1000); - let mut elements = vec![]; - for _ in 0..tree_size { - let mut element = String::from("0x"); - let element_size = rng.gen_range(10..32); - for _ in 0..element_size { - let random_byte: u8 = rng.gen(); - element.push_str(&format!("{:02x}", random_byte)); - } - let value = Felt::from_hex(&element).unwrap(); - let key = &value.to_bytes_be()[..31]; - bonsai_storage - .insert(&identifier, &BitVec::from_vec(key.to_vec()), &value) - .unwrap(); - pathfinder_merkle_tree - .set( - &storage, - BitVec::from_vec(key.to_vec()), - PathfinderFelt::from_hex_str(&element).unwrap(), - ) - .unwrap(); - elements.push((key.to_vec(), value)); - } - bonsai_storage.commit(id_builder.new_id()).unwrap(); - let (_, root_id) = commit_and_persist(pathfinder_merkle_tree.clone(), &mut storage); - for element in elements.iter() { - let proof = bonsai_storage - .get_proof(&identifier, &BitVec::from_vec(element.0.clone())) - .unwrap(); - let pathfinder_proof = - pathfinder_merkle_tree::tree::MerkleTree::::get_proof( - root_id, - &storage, - &BitVec::from_vec(element.0.clone()), - ) - .unwrap(); - assert_eq_proof(&proof, &pathfinder_proof); - assert_eq!( - BonsaiStorage::, Pedersen>::verify_proof( - bonsai_storage.root_hash(&identifier).unwrap(), - &BitVec::from_vec(element.0.clone()), - element.1, - &proof - ), - Some(Membership::Member) - ); - } -} - -#[test] -fn one_element_proof() { - let identifier = vec![]; - let tempdir = tempfile::tempdir().unwrap(); - let db = create_rocks_db(tempdir.path()).unwrap(); - let config = BonsaiStorageConfig::default(); - let mut storage = pathfinder_merkle_tree::tree::TestStorage::default(); - let mut bonsai_storage = - BonsaiStorage::<_, _, Pedersen>::new(RocksDB::new(&db, RocksDBConfig::default()), config) - .unwrap(); - let mut pathfinder_merkle_tree: MerkleTree = - pathfinder_merkle_tree::tree::MerkleTree::empty(); - let mut id_builder = BasicIdBuilder::new(); - let pair1 = ( - vec![1, 2, 1], - Felt::from_hex("0x66342762FDD54D033c195fec3ce2568b62052e").unwrap(), - ); - let bitvec = BitVec::from_vec(pair1.0.clone()); - bonsai_storage - .insert(&identifier, &bitvec, &pair1.1) - .unwrap(); - pathfinder_merkle_tree - .set( - &storage, - bitvec, - PathfinderFelt::from_hex_str("0x66342762FDD54D033c195fec3ce2568b62052e").unwrap(), - ) - .unwrap(); - bonsai_storage.commit(id_builder.new_id()).unwrap(); - let (_, root_id) = commit_and_persist(pathfinder_merkle_tree.clone(), &mut storage); - let bonsai_proof = bonsai_storage - .get_proof(&identifier, &BitVec::from_vec(vec![1, 2, 1])) - .unwrap(); - let pathfinder_proof = - pathfinder_merkle_tree::tree::MerkleTree::::get_proof( - root_id, - &storage, - &BitVec::from_vec(vec![1, 2, 1]), - ) - .unwrap(); - assert_eq_proof(&bonsai_proof, &pathfinder_proof); - assert_eq!( - BonsaiStorage::, Pedersen>::verify_proof( - bonsai_storage.root_hash(&identifier).unwrap(), - &BitVec::from_vec(vec![1, 2, 1]), - pair1.1, - &bonsai_proof - ), - Some(Membership::Member) - ); -} - -#[test] -fn zero_not_crashing() { - let identifier = vec![]; - let tempdir = tempfile::tempdir().unwrap(); - let db = create_rocks_db(tempdir.path()).unwrap(); - let config = BonsaiStorageConfig::default(); - let mut bonsai_storage = - BonsaiStorage::<_, _, Pedersen>::new(RocksDB::new(&db, RocksDBConfig::default()), config) - .unwrap(); - let mut id_builder = BasicIdBuilder::new(); - bonsai_storage.commit(id_builder.new_id()).unwrap(); - bonsai_storage - .get_proof(&identifier, &BitVec::from_vec(vec![1, 2, 1])) - .expect_err("Should error"); -} diff --git a/src/tests/simple.rs b/src/tests/simple.rs index 171ce9f..1133127 100644 --- a/src/tests/simple.rs +++ b/src/tests/simple.rs @@ -642,6 +642,7 @@ fn test_insert_zero() { #[test] fn test_block_7_starknet() { + let _ = env_logger::builder().is_test(true).try_init(); let config = BonsaiStorageConfig::default(); let bonsai_db = HashMapDb::::default(); let mut bonsai_storage = BonsaiStorage::<_, _, Pedersen>::new(bonsai_db, config) @@ -678,6 +679,7 @@ fn test_block_7_starknet() { .expect("Failed to insert storage update into trie"); } + bonsai_storage.dump(); let mut id_builder = BasicIdBuilder::new(); let id = id_builder.new_id(); bonsai_storage @@ -747,6 +749,7 @@ fn test_block_7_starknet() { .expect("Failed to insert storage update into trie"); } + bonsai_storage.dump(); let id = id_builder.new_id(); bonsai_storage .commit(id) diff --git a/src/trie/proof.rs b/src/trie/proof.rs index 6f0448a..612defc 100644 --- a/src/trie/proof.rs +++ b/src/trie/proof.rs @@ -59,13 +59,14 @@ pub struct MultiProof(pub HashMap); impl MultiProof { /// If the proof proves more than just the provided `key_values`, this function will not fail. /// Not the most optimized way of doing it, but we don't actually need to verify proofs in madara. - /// As such, it has not been properly proptested. + /// As such, it has also not been properly proptested. pub fn verify_proof<'a, 'b: 'a, H: StarkHash>( &'b self, root: Felt, key_values: impl IntoIterator, Felt)> + 'a, ) -> impl Iterator + 'a { let mut checked_cache: HashSet = Default::default(); + let mut current_path = BitVec::with_capacity(251); key_values.into_iter().map(move |(k, v)| { let k = k.as_ref(); @@ -75,7 +76,7 @@ impl MultiProof { // } // Go down the tree, starting from the root. - let mut current_path = BitVec::with_capacity(251); + current_path.clear(); // hoisted alloc let mut current_felt = root; loop { @@ -261,6 +262,5 @@ mod tests { .all(|v| v.into()), true ); - todo!() } } diff --git a/src/trie/tree.rs b/src/trie/tree.rs index f2c373a..2b1a3bc 100644 --- a/src/trie/tree.rs +++ b/src/trie/tree.rs @@ -653,6 +653,7 @@ impl MerkleTree { log::trace!("change val: {:?} => {:#x}", key_bytes, value); self.cache_leaf_modified .insert(key_bytes, InsertOrRemove::Insert(value)); + self.node_storage.nodes[*node_id] = node; return Ok(()); } // Height of the binary node's children @@ -782,6 +783,7 @@ impl MerkleTree { db: &KeyValueDB, key: &BitSlice, ) -> Result<(), BonsaiStorageError> { + log::trace!("delete leaf"); // Algorithm explanation: // // The leaf's parent node is either an edge, or a binary node. diff --git a/src/trie/trees.rs b/src/trie/trees.rs index e5f0899..c6e4f8a 100644 --- a/src/trie/trees.rs +++ b/src/trie/trees.rs @@ -1,4 +1,4 @@ -use super::tree::MerkleTree; +use super::{proof::MultiProof, tree::MerkleTree}; use crate::{ id::Id, key_value_db::KeyValueDB, trie::tree::InsertOrRemove, BitSlice, BonsaiDatabase, BonsaiStorageError, ByteVec, HashMap, @@ -228,4 +228,17 @@ impl MerkleTrees Vec> { self.trees.keys().cloned().map(ByteVec::into_vec).collect() } + + pub fn get_multi_proof( + &mut self, + identifier: &[u8], + keys: impl IntoIterator>, + ) -> Result> { + let tree = self + .trees + .entry_ref(identifier) + .or_insert_with(|| MerkleTree::new(identifier.into())); + + tree.get_multi_proof(&self.db, keys) + } }