From 46c4315bcfc55a32a3ccaae275bc2eb173454cb9 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Fri, 29 Sep 2023 07:12:13 +0100 Subject: [PATCH 1/3] wip --- grovedb/src/operations/delete/mod.rs | 221 ++++++++++++++++++++++++++- 1 file changed, 220 insertions(+), 1 deletion(-) diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index 512a33ef..9b518ad0 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -38,6 +38,7 @@ mod worst_case; #[cfg(feature = "full")] use std::collections::{BTreeSet, HashMap}; +use bincode::options; #[cfg(feature = "full")] pub use delete_up_tree::DeleteUpTreeOptions; #[cfg(feature = "full")] @@ -46,6 +47,7 @@ use grovedb_costs::{ storage_cost::removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, CostResult, CostsExt, OperationCost, }; +use grovedb_merk::{proofs::Query, KVIterator}; #[cfg(feature = "full")] use grovedb_merk::{Error as MerkError, Merk, MerkOptions}; use grovedb_path::SubtreePath; @@ -55,13 +57,13 @@ use grovedb_storage::{ Storage, StorageBatch, StorageContext, }; -use crate::util::merk_optional_tx_path_not_empty; #[cfg(feature = "full")] use crate::{ batch::{GroveDbOp, Op}, util::{storage_context_optional_tx, storage_context_with_parent_optional_tx}, Element, ElementFlags, Error, GroveDb, Transaction, TransactionArg, }; +use crate::{raw_decode, util::merk_optional_tx_path_not_empty}; #[cfg(feature = "full")] #[derive(Clone)] @@ -138,6 +140,135 @@ impl GroveDb { }) } + /// Delete all elements in a specified subtree + pub fn clear_subtree<'b, B, P>( + &self, + path: P, + transaction: TransactionArg, + ) -> CostResult<(), Error> + where + B: AsRef<[u8]> + 'b, + P: Into>, + { + let subtree_path: SubtreePath = path.into(); + let mut cost = OperationCost::default(); + let batch = StorageBatch::new(); + + if let Some(transaction) = transaction { + let mut merk_to_clear = cost_return_on_error!( + &mut cost, + self.open_transactional_merk_at_path( + subtree_path.clone(), + transaction, + Some(&batch) + ) + ); + + let mut all_query = Query::new(); + all_query.insert_all(); + + let mut element_iterator = + KVIterator::new(merk_to_clear.storage.raw_iter(), &all_query).unwrap(); + + // delete all nested subtrees + while let Some((key, element_value)) = + element_iterator.next_kv().unwrap_add_cost(&mut cost) + { + let element = raw_decode(&element_value).unwrap(); + if element.is_tree() { + cost_return_on_error!( + &mut cost, + self.delete( + subtree_path.clone(), + key.as_slice(), + Some(DeleteOptions { + allow_deleting_non_empty_trees: true, + deleting_non_empty_trees_returns_error: false, + ..Default::default() + }), + Some(transaction), + ) + ); + } + } + + // delete non subtree values + merk_to_clear.clear(); + + // propagate changes + let mut merk_cache: HashMap, Merk> = + HashMap::default(); + merk_cache.insert(subtree_path.clone(), merk_to_clear); + cost_return_on_error!( + &mut cost, + self.propagate_changes_with_transaction( + merk_cache, + subtree_path.clone(), + transaction, + &batch, + ) + ); + } else { + let mut merk_to_clear = cost_return_on_error!( + &mut cost, + self.open_non_transactional_merk_at_path(subtree_path.clone(), Some(&batch)) + ); + + let mut all_query = Query::new(); + all_query.insert_all(); + + let mut element_iterator = + KVIterator::new(merk_to_clear.storage.raw_iter(), &all_query).unwrap(); + + // delete all nested subtrees + while let Some((key, element_value)) = + element_iterator.next_kv().unwrap_add_cost(&mut cost) + { + let element = raw_decode(&element_value).unwrap(); + if element.is_tree() { + cost_return_on_error!( + &mut cost, + self.delete( + subtree_path.clone(), + key.as_slice(), + Some(DeleteOptions { + allow_deleting_non_empty_trees: true, + deleting_non_empty_trees_returns_error: false, + ..Default::default() + }), + None + ) + ); + } + } + + // delete non subtree values + merk_to_clear.clear(); + + // propagate changes + let mut merk_cache: HashMap, Merk> = + HashMap::default(); + merk_cache.insert(subtree_path.clone(), merk_to_clear); + cost_return_on_error!( + &mut cost, + self.propagate_changes_without_transaction( + merk_cache, + subtree_path.clone(), + &batch, + ) + ); + } + + cost_return_on_error!( + &mut cost, + self.db + .commit_multi_context_batch(batch, transaction) + .map_err(Into::into) + ); + + Ok(()).wrap_with_cost(cost) + } + /// Delete element with sectional storage function pub fn delete_with_sectional_storage_function>( &self, @@ -1445,4 +1576,92 @@ mod tests { } ); } + + #[test] + fn test_subtree_clear() { + let element = Element::new_item(b"ayy".to_vec()); + + let db = make_test_grovedb(); + + // Insert some nested subtrees + db.insert( + [TEST_LEAF].as_ref(), + b"key1", + Element::empty_tree(), + None, + None, + ) + .unwrap() + .expect("successful subtree 1 insert"); + db.insert( + [TEST_LEAF, b"key1"].as_ref(), + b"key2", + Element::empty_tree(), + None, + None, + ) + .unwrap() + .expect("successful subtree 2 insert"); + + // Insert an element into subtree + db.insert( + [TEST_LEAF, b"key1", b"key2"].as_ref(), + b"key3", + element, + None, + None, + ) + .unwrap() + .expect("successful value insert"); + db.insert( + [TEST_LEAF].as_ref(), + b"key4", + Element::empty_tree(), + None, + None, + ) + .unwrap() + .expect("successful subtree 3 insert"); + + let key1_tree = db + .get([TEST_LEAF].as_ref(), b"key1", None) + .unwrap() + .unwrap(); + assert!(!matches!(key1_tree, Element::Tree(None, _))); + let key1_merk = db + .open_non_transactional_merk_at_path([TEST_LEAF, b"key1"].as_ref().into(), None) + .unwrap() + .unwrap(); + assert_ne!(key1_merk.root_hash().unwrap(), [0; 32]); + + let root_hash_before_clear = db.root_hash(None).unwrap().unwrap(); + db.clear_subtree([TEST_LEAF, b"key1"].as_ref(), None) + .unwrap() + .expect("unable to delete subtree"); + + assert!(matches!( + db.get([TEST_LEAF, b"key1"].as_ref(), b"key2", None) + .unwrap(), + Err(Error::PathKeyNotFound(_)) + )); + assert!(matches!( + db.get([TEST_LEAF, b"key1", b"key2"].as_ref(), b"key3", None) + .unwrap(), + Err(Error::PathParentLayerNotFound(_)) + )); + let key1_tree = db + .get([TEST_LEAF].as_ref(), b"key1", None) + .unwrap() + .unwrap(); + assert!(matches!(key1_tree, Element::Tree(None, _))); + + let key1_merk = db + .open_non_transactional_merk_at_path([TEST_LEAF, b"key1"].as_ref().into(), None) + .unwrap() + .unwrap(); + assert_eq!(key1_merk.root_hash().unwrap(), [0; 32]); + + let root_hash_after_clear = db.root_hash(None).unwrap().unwrap(); + assert_ne!(root_hash_before_clear, root_hash_after_clear); + } } From 061f923bde4c9c56d22e58ca9b79b70affaa8d26 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Fri, 29 Sep 2023 07:14:01 +0100 Subject: [PATCH 2/3] fmt --- grovedb/src/operations/delete/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index 9b518ad0..9c0cfed6 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -38,7 +38,6 @@ mod worst_case; #[cfg(feature = "full")] use std::collections::{BTreeSet, HashMap}; -use bincode::options; #[cfg(feature = "full")] pub use delete_up_tree::DeleteUpTreeOptions; #[cfg(feature = "full")] From 29200cd1236130878a4e5fb065aa17116c95fb7d Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 29 Sep 2023 13:50:14 +0700 Subject: [PATCH 3/3] modifications --- grovedb/src/error.rs | 4 + grovedb/src/operations/delete/mod.rs | 203 ++++++++++++++++++++------- 2 files changed, 153 insertions(+), 54 deletions(-) diff --git a/grovedb/src/error.rs b/grovedb/src/error.rs index d2db936f..d29feaa7 100644 --- a/grovedb/src/error.rs +++ b/grovedb/src/error.rs @@ -132,6 +132,10 @@ pub enum Error { /// Deleting non empty tree DeletingNonEmptyTree(&'static str), + #[error("clearing tree with subtrees not allowed error: {0}")] + /// Clearing tree with subtrees not allowed + ClearingTreeWithSubtreesNotAllowed(&'static str), + // Client allowed errors #[error("just in time element flags client error: {0}")] /// Just in time element flags client error diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index 9c0cfed6..490a4dab 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -64,6 +64,30 @@ use crate::{ }; use crate::{raw_decode, util::merk_optional_tx_path_not_empty}; +#[cfg(feature = "full")] +#[derive(Clone)] +/// Clear options +pub struct ClearOptions { + /// Check for Subtrees + pub check_for_subtrees: bool, + /// Allow deleting non empty trees if we check for subtrees + pub allow_deleting_subtrees: bool, + /// If we check for subtrees, and we don't allow deleting and there are + /// some, should we error? + pub trying_to_clear_with_subtrees_returns_error: bool, +} + +#[cfg(feature = "full")] +impl Default for ClearOptions { + fn default() -> Self { + ClearOptions { + check_for_subtrees: true, + allow_deleting_subtrees: false, + trying_to_clear_with_subtrees_returns_error: true, + } + } +} + #[cfg(feature = "full")] #[derive(Clone)] /// Delete options @@ -140,11 +164,31 @@ impl GroveDb { } /// Delete all elements in a specified subtree - pub fn clear_subtree<'b, B, P>( + /// Returns if we successfully cleared the subtree + fn clear_subtree<'b, B, P>( &self, path: P, + options: Option, transaction: TransactionArg, - ) -> CostResult<(), Error> + ) -> Result + where + B: AsRef<[u8]> + 'b, + P: Into>, + { + self.clear_subtree_with_costs(path, options, transaction) + .unwrap() + } + + /// Delete all elements in a specified subtree and get back costs + /// Warning: The costs for this operation are not yet correct, hence we + /// should keep this private for now + /// Returns if we successfully cleared the subtree + fn clear_subtree_with_costs<'b, B, P>( + &self, + path: P, + options: Option, + transaction: TransactionArg, + ) -> CostResult where B: AsRef<[u8]> + 'b, P: Into>, @@ -153,6 +197,8 @@ impl GroveDb { let mut cost = OperationCost::default(); let batch = StorageBatch::new(); + let options = options.unwrap_or_default(); + if let Some(transaction) = transaction { let mut merk_to_clear = cost_return_on_error!( &mut cost, @@ -163,36 +209,48 @@ impl GroveDb { ) ); - let mut all_query = Query::new(); - all_query.insert_all(); + if options.check_for_subtrees { + let mut all_query = Query::new(); + all_query.insert_all(); - let mut element_iterator = - KVIterator::new(merk_to_clear.storage.raw_iter(), &all_query).unwrap(); + let mut element_iterator = + KVIterator::new(merk_to_clear.storage.raw_iter(), &all_query).unwrap(); - // delete all nested subtrees - while let Some((key, element_value)) = - element_iterator.next_kv().unwrap_add_cost(&mut cost) - { - let element = raw_decode(&element_value).unwrap(); - if element.is_tree() { - cost_return_on_error!( - &mut cost, - self.delete( - subtree_path.clone(), - key.as_slice(), - Some(DeleteOptions { - allow_deleting_non_empty_trees: true, - deleting_non_empty_trees_returns_error: false, - ..Default::default() - }), - Some(transaction), - ) - ); + // delete all nested subtrees + while let Some((key, element_value)) = + element_iterator.next_kv().unwrap_add_cost(&mut cost) + { + let element = raw_decode(&element_value).unwrap(); + if element.is_tree() { + if options.allow_deleting_subtrees { + cost_return_on_error!( + &mut cost, + self.delete( + subtree_path.clone(), + key.as_slice(), + Some(DeleteOptions { + allow_deleting_non_empty_trees: true, + deleting_non_empty_trees_returns_error: false, + ..Default::default() + }), + Some(transaction), + ) + ); + } else if options.trying_to_clear_with_subtrees_returns_error { + return Err(Error::ClearingTreeWithSubtreesNotAllowed( + "options do not allow to clear this merk tree as it contains \ + subtrees", + )) + .wrap_with_cost(cost); + } else { + return Ok(false).wrap_with_cost(cost); + } + } } } // delete non subtree values - merk_to_clear.clear(); + cost_return_on_error!(&mut cost, merk_to_clear.clear().map_err(Error::MerkError)); // propagate changes let mut merk_cache: HashMap, Merk> = @@ -213,36 +271,47 @@ impl GroveDb { self.open_non_transactional_merk_at_path(subtree_path.clone(), Some(&batch)) ); - let mut all_query = Query::new(); - all_query.insert_all(); + if options.check_for_subtrees { + let mut all_query = Query::new(); + all_query.insert_all(); - let mut element_iterator = - KVIterator::new(merk_to_clear.storage.raw_iter(), &all_query).unwrap(); + let mut element_iterator = + KVIterator::new(merk_to_clear.storage.raw_iter(), &all_query).unwrap(); - // delete all nested subtrees - while let Some((key, element_value)) = - element_iterator.next_kv().unwrap_add_cost(&mut cost) - { - let element = raw_decode(&element_value).unwrap(); - if element.is_tree() { - cost_return_on_error!( - &mut cost, - self.delete( - subtree_path.clone(), - key.as_slice(), - Some(DeleteOptions { - allow_deleting_non_empty_trees: true, - deleting_non_empty_trees_returns_error: false, - ..Default::default() - }), - None - ) - ); + // delete all nested subtrees + while let Some((key, element_value)) = + element_iterator.next_kv().unwrap_add_cost(&mut cost) + { + let element = raw_decode(&element_value).unwrap(); + if options.allow_deleting_subtrees { + if element.is_tree() { + cost_return_on_error!( + &mut cost, + self.delete( + subtree_path.clone(), + key.as_slice(), + Some(DeleteOptions { + allow_deleting_non_empty_trees: true, + deleting_non_empty_trees_returns_error: false, + ..Default::default() + }), + None + ) + ); + } + } else if options.trying_to_clear_with_subtrees_returns_error { + return Err(Error::ClearingTreeWithSubtreesNotAllowed( + "options do not allow to clear this merk tree as it contains subtrees", + )) + .wrap_with_cost(cost); + } else { + return Ok(false).wrap_with_cost(cost); + } } } // delete non subtree values - merk_to_clear.clear(); + cost_return_on_error!(&mut cost, merk_to_clear.clear().map_err(Error::MerkError)); // propagate changes let mut merk_cache: HashMap, Merk> = @@ -265,7 +334,7 @@ impl GroveDb { .map_err(Into::into) ); - Ok(()).wrap_with_cost(cost) + Ok(true).wrap_with_cost(cost) } /// Delete element with sectional storage function @@ -868,7 +937,7 @@ mod tests { use pretty_assertions::assert_eq; use crate::{ - operations::delete::{delete_up_tree::DeleteUpTreeOptions, DeleteOptions}, + operations::delete::{delete_up_tree::DeleteUpTreeOptions, ClearOptions, DeleteOptions}, tests::{ common::EMPTY_PATH, make_empty_grovedb, make_test_grovedb, ANOTHER_TEST_LEAF, TEST_LEAF, }, @@ -1634,10 +1703,36 @@ mod tests { assert_ne!(key1_merk.root_hash().unwrap(), [0; 32]); let root_hash_before_clear = db.root_hash(None).unwrap().unwrap(); - db.clear_subtree([TEST_LEAF, b"key1"].as_ref(), None) - .unwrap() + db.clear_subtree([TEST_LEAF, b"key1"].as_ref(), None, None) + .expect_err("unable to delete subtree"); + + let success = db + .clear_subtree( + [TEST_LEAF, b"key1"].as_ref(), + Some(ClearOptions { + check_for_subtrees: true, + allow_deleting_subtrees: false, + trying_to_clear_with_subtrees_returns_error: false, + }), + None, + ) + .expect("expected no error"); + assert!(!success); + + let success = db + .clear_subtree( + [TEST_LEAF, b"key1"].as_ref(), + Some(ClearOptions { + check_for_subtrees: true, + allow_deleting_subtrees: true, + trying_to_clear_with_subtrees_returns_error: false, + }), + None, + ) .expect("unable to delete subtree"); + assert!(success); + assert!(matches!( db.get([TEST_LEAF, b"key1"].as_ref(), b"key2", None) .unwrap(),