diff --git a/grovedb/build.rs b/grovedb/build.rs index f5792754..1d5fa25b 100644 --- a/grovedb/build.rs +++ b/grovedb/build.rs @@ -14,8 +14,8 @@ fn main() { if !grovedbg_zip_path.exists() { let response = reqwest::blocking::get(format!( - "https://github.com/dashpay/grovedbg/releases/download/{GROVEDBG_VERSION}/\ -grovedbg-{GROVEDBG_VERSION}.zip" + "https://github.com/dashpay/grovedbg/releases/download/\ +{GROVEDBG_VERSION}/grovedbg-{GROVEDBG_VERSION}.zip" )) .expect("can't download GroveDBG artifact"); diff --git a/grovedb/src/batch/batch_structure.rs b/grovedb/src/batch/batch_structure.rs index eb00f1e8..9cfe03db 100644 --- a/grovedb/src/batch/batch_structure.rs +++ b/grovedb/src/batch/batch_structure.rs @@ -16,12 +16,12 @@ use nohash_hasher::IntMap; #[cfg(feature = "full")] use crate::{ - batch::{key_info::KeyInfo, GroveDbOp, KeyInfoPath, Op, TreeCache}, + batch::{key_info::KeyInfo, GroveOp, KeyInfoPath, QualifiedGroveDbOp, TreeCache}, Element, ElementFlags, Error, }; #[cfg(feature = "full")] -pub type OpsByPath = BTreeMap>; +pub type OpsByPath = BTreeMap>; /// Level, path, key, op #[cfg(feature = "full")] pub type OpsByLevelPath = IntMap; @@ -32,7 +32,7 @@ pub(super) struct BatchStructure { /// Operations by level path pub(super) ops_by_level_paths: OpsByLevelPath, /// This is for references - pub(super) ops_by_qualified_paths: BTreeMap>, Op>, + pub(super) ops_by_qualified_paths: BTreeMap>, GroveOp>, /// Merk trees /// Very important: the type of run mode we are in is contained in this /// cache @@ -84,7 +84,7 @@ where { /// Create batch structure from a list of ops. Returns CostResult. pub(super) fn from_ops( - ops: Vec, + ops: Vec, update_element_flags_function: F, split_remove_bytes_function: SR, merk_tree_cache: C, @@ -101,7 +101,7 @@ where /// Create batch structure from a list of ops. Returns CostResult. pub(super) fn continue_from_ops( previous_ops: Option, - ops: Vec, + ops: Vec, update_element_flags_function: F, split_remove_bytes_function: SR, mut merk_tree_cache: C, @@ -112,7 +112,7 @@ where let mut current_last_level: u32 = 0; // qualified paths meaning path + key - let mut ops_by_qualified_paths: BTreeMap>, Op> = BTreeMap::new(); + let mut ops_by_qualified_paths: BTreeMap>, GroveOp> = BTreeMap::new(); for op in ops.into_iter() { let mut path = op.path.clone(); @@ -120,7 +120,10 @@ where ops_by_qualified_paths.insert(path.to_path_consume(), op.op.clone()); let op_cost = OperationCost::default(); let op_result = match &op.op { - Op::Insert { element } | Op::Replace { element } | Op::Patch { element, .. } => { + GroveOp::InsertOnly { element } + | GroveOp::InsertOrReplace { element } + | GroveOp::Replace { element } + | GroveOp::Patch { element, .. } => { if let Element::Tree(..) = element { cost_return_on_error!(&mut cost, merk_tree_cache.insert(&op, false)); } else if let Element::SumTree(..) = element { @@ -128,10 +131,11 @@ where } Ok(()) } - Op::RefreshReference { .. } | Op::Delete | Op::DeleteTree | Op::DeleteSumTree => { - Ok(()) - } - Op::ReplaceTreeRootKey { .. } | Op::InsertTreeWithRootHash { .. } => { + GroveOp::RefreshReference { .. } + | GroveOp::Delete + | GroveOp::DeleteTree + | GroveOp::DeleteSumTree => Ok(()), + GroveOp::ReplaceTreeRootKey { .. } | GroveOp::InsertTreeWithRootHash { .. } => { Err(Error::InvalidBatchOperation( "replace and insert tree hash are internal operations only", )) @@ -146,14 +150,14 @@ where if let Some(ops_on_path) = ops_on_level.get_mut(&op.path) { ops_on_path.insert(op.key, op.op); } else { - let mut ops_on_path: BTreeMap = BTreeMap::new(); + let mut ops_on_path: BTreeMap = BTreeMap::new(); ops_on_path.insert(op.key, op.op); ops_on_level.insert(op.path.clone(), ops_on_path); } } else { - let mut ops_on_path: BTreeMap = BTreeMap::new(); + let mut ops_on_path: BTreeMap = BTreeMap::new(); ops_on_path.insert(op.key, op.op); - let mut ops_on_level: BTreeMap> = + let mut ops_on_level: BTreeMap> = BTreeMap::new(); ops_on_level.insert(op.path, ops_on_path); ops_by_level_paths.insert(level, ops_on_level); diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 2f50186b..ef64b0f4 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -26,14 +26,14 @@ use crate::Element; #[cfg(feature = "full")] use crate::{ batch::{ - key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveDbOp, KeyInfoPath, Op, - TreeCache, + key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveOp, KeyInfoPath, + QualifiedGroveDbOp, TreeCache, }, Error, GroveDb, }; #[cfg(feature = "full")] -impl Op { +impl GroveOp { /// Get the estimated average case cost of the op. Calls a lower level /// function to calculate the estimate based on the type of op. Returns /// CostResult. @@ -53,14 +53,14 @@ impl Op { } }; match self { - Op::ReplaceTreeRootKey { sum, .. } => GroveDb::average_case_merk_replace_tree( + GroveOp::ReplaceTreeRootKey { sum, .. } => GroveDb::average_case_merk_replace_tree( key, layer_element_estimates, sum.is_some(), propagate, grove_version, ), - Op::InsertTreeWithRootHash { flags, sum, .. } => { + GroveOp::InsertTreeWithRootHash { flags, sum, .. } => { GroveDb::average_case_merk_insert_tree( key, flags, @@ -70,14 +70,16 @@ impl Op { grove_version, ) } - Op::Insert { element } => GroveDb::average_case_merk_insert_element( - key, - element, - in_tree_using_sums, - propagate_if_input(), - grove_version, - ), - Op::RefreshReference { + GroveOp::InsertOrReplace { element } | GroveOp::InsertOnly { element } => { + GroveDb::average_case_merk_insert_element( + key, + element, + in_tree_using_sums, + propagate_if_input(), + grove_version, + ) + } + GroveOp::RefreshReference { reference_path_type, max_reference_hop, flags, @@ -93,14 +95,14 @@ impl Op { propagate_if_input(), grove_version, ), - Op::Replace { element } => GroveDb::average_case_merk_replace_element( + GroveOp::Replace { element } => GroveDb::average_case_merk_replace_element( key, element, in_tree_using_sums, propagate_if_input(), grove_version, ), - Op::Patch { + GroveOp::Patch { element, change_in_bytes, } => GroveDb::average_case_merk_patch_element( @@ -111,20 +113,20 @@ impl Op { propagate_if_input(), grove_version, ), - Op::Delete => GroveDb::average_case_merk_delete_element( + GroveOp::Delete => GroveDb::average_case_merk_delete_element( key, layer_element_estimates, propagate, grove_version, ), - Op::DeleteTree => GroveDb::average_case_merk_delete_tree( + GroveOp::DeleteTree => GroveDb::average_case_merk_delete_tree( key, false, layer_element_estimates, propagate, grove_version, ), - Op::DeleteSumTree => GroveDb::average_case_merk_delete_tree( + GroveOp::DeleteSumTree => GroveDb::average_case_merk_delete_tree( key, true, layer_element_estimates, @@ -165,7 +167,7 @@ impl fmt::Debug for AverageCaseTreeCacheKnownPaths { #[cfg(feature = "full")] impl TreeCache for AverageCaseTreeCacheKnownPaths { - fn insert(&mut self, op: &GroveDbOp, is_sum_tree: bool) -> CostResult<(), Error> { + fn insert(&mut self, op: &QualifiedGroveDbOp, is_sum_tree: bool) -> CostResult<(), Error> { let mut average_case_cost = OperationCost::default(); let mut inserted_path = op.path.clone(); inserted_path.push(op.key.clone()); @@ -184,8 +186,8 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { fn execute_ops_on_path( &mut self, path: &KeyInfoPath, - ops_at_path_by_key: BTreeMap, - _ops_by_qualified_paths: &BTreeMap>, Op>, + ops_at_path_by_key: BTreeMap, + _ops_by_qualified_paths: &BTreeMap>, GroveOp>, _batch_apply_options: &BatchApplyOptions, _flags_update: &mut G, _split_removal_bytes: &mut SR, @@ -309,7 +311,7 @@ mod tests { use crate::{ batch::{ estimated_costs::EstimatedCostsType::AverageCaseCostsType, key_info::KeyInfo, - GroveDbOp, KeyInfoPath, + KeyInfoPath, QualifiedGroveDbOp, }, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, GroveDb, @@ -321,7 +323,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -390,7 +392,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree_with_flags(Some(b"cat".to_vec())), @@ -457,7 +459,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::new_item(b"cat".to_vec()), @@ -530,7 +532,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -616,7 +618,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"0".to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -695,7 +697,7 @@ mod tests { #[test] fn test_batch_root_one_sum_item_replace_op_average_case_costs() { let grove_version = GroveVersion::latest(); - let ops = vec![GroveDbOp::replace_op( + let ops = vec![QualifiedGroveDbOp::replace_op( vec![vec![7]], hex::decode("46447a3b4c8939fd4cf8b610ba7da3d3f6b52b39ab2549bf91503b9b07814055") .unwrap(), @@ -774,7 +776,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 1b1d42e7..9bf9a808 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -25,14 +25,14 @@ use crate::Element; #[cfg(feature = "full")] use crate::{ batch::{ - key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveDbOp, KeyInfoPath, Op, - TreeCache, + key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveOp, KeyInfoPath, + QualifiedGroveDbOp, TreeCache, }, Error, GroveDb, }; #[cfg(feature = "full")] -impl Op { +impl GroveOp { fn worst_case_cost( &self, key: &KeyInfo, @@ -49,7 +49,7 @@ impl Op { } }; match self { - Op::ReplaceTreeRootKey { sum, .. } => GroveDb::worst_case_merk_replace_tree( + GroveOp::ReplaceTreeRootKey { sum, .. } => GroveDb::worst_case_merk_replace_tree( key, sum.is_some(), is_in_parent_sum_tree, @@ -57,22 +57,26 @@ impl Op { propagate, grove_version, ), - Op::InsertTreeWithRootHash { flags, sum, .. } => GroveDb::worst_case_merk_insert_tree( - key, - flags, - sum.is_some(), - is_in_parent_sum_tree, - propagate_if_input(), - grove_version, - ), - Op::Insert { element } => GroveDb::worst_case_merk_insert_element( - key, - element, - is_in_parent_sum_tree, - propagate_if_input(), - grove_version, - ), - Op::RefreshReference { + GroveOp::InsertTreeWithRootHash { flags, sum, .. } => { + GroveDb::worst_case_merk_insert_tree( + key, + flags, + sum.is_some(), + is_in_parent_sum_tree, + propagate_if_input(), + grove_version, + ) + } + GroveOp::InsertOrReplace { element } | GroveOp::InsertOnly { element } => { + GroveDb::worst_case_merk_insert_element( + key, + element, + is_in_parent_sum_tree, + propagate_if_input(), + grove_version, + ) + } + GroveOp::RefreshReference { reference_path_type, max_reference_hop, flags, @@ -88,14 +92,14 @@ impl Op { propagate_if_input(), grove_version, ), - Op::Replace { element } => GroveDb::worst_case_merk_replace_element( + GroveOp::Replace { element } => GroveDb::worst_case_merk_replace_element( key, element, is_in_parent_sum_tree, propagate_if_input(), grove_version, ), - Op::Patch { + GroveOp::Patch { element, change_in_bytes: _, } => GroveDb::worst_case_merk_replace_element( @@ -105,20 +109,20 @@ impl Op { propagate_if_input(), grove_version, ), - Op::Delete => GroveDb::worst_case_merk_delete_element( + GroveOp::Delete => GroveDb::worst_case_merk_delete_element( key, worst_case_layer_element_estimates, propagate, grove_version, ), - Op::DeleteTree => GroveDb::worst_case_merk_delete_tree( + GroveOp::DeleteTree => GroveDb::worst_case_merk_delete_tree( key, false, worst_case_layer_element_estimates, propagate, grove_version, ), - Op::DeleteSumTree => GroveDb::worst_case_merk_delete_tree( + GroveOp::DeleteSumTree => GroveDb::worst_case_merk_delete_tree( key, true, worst_case_layer_element_estimates, @@ -159,7 +163,7 @@ impl fmt::Debug for WorstCaseTreeCacheKnownPaths { #[cfg(feature = "full")] impl TreeCache for WorstCaseTreeCacheKnownPaths { - fn insert(&mut self, op: &GroveDbOp, _is_sum_tree: bool) -> CostResult<(), Error> { + fn insert(&mut self, op: &QualifiedGroveDbOp, _is_sum_tree: bool) -> CostResult<(), Error> { let mut worst_case_cost = OperationCost::default(); let mut inserted_path = op.path.clone(); inserted_path.push(op.key.clone()); @@ -178,8 +182,8 @@ impl TreeCache for WorstCaseTreeCacheKnownPaths { fn execute_ops_on_path( &mut self, path: &KeyInfoPath, - ops_at_path_by_key: BTreeMap, - _ops_by_qualified_paths: &BTreeMap>, Op>, + ops_at_path_by_key: BTreeMap, + _ops_by_qualified_paths: &BTreeMap>, GroveOp>, _batch_apply_options: &BatchApplyOptions, _flags_update: &mut G, _split_removal_bytes: &mut SR, @@ -273,8 +277,8 @@ mod tests { use crate::{ batch::{ - estimated_costs::EstimatedCostsType::WorstCaseCostsType, key_info::KeyInfo, GroveDbOp, - KeyInfoPath, + estimated_costs::EstimatedCostsType::WorstCaseCostsType, key_info::KeyInfo, + KeyInfoPath, QualifiedGroveDbOp, }, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, GroveDb, @@ -286,7 +290,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -341,7 +345,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree_with_flags(Some(b"cat".to_vec())), @@ -396,7 +400,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::new_item(b"cat".to_vec()), @@ -462,7 +466,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -528,7 +532,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"0".to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -592,7 +596,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), diff --git a/grovedb/src/batch/just_in_time_cost_tests.rs b/grovedb/src/batch/just_in_time_cost_tests.rs index e1fddf5c..a3fa9aec 100644 --- a/grovedb/src/batch/just_in_time_cost_tests.rs +++ b/grovedb/src/batch/just_in_time_cost_tests.rs @@ -5,11 +5,18 @@ mod tests { use std::option::Option::None; + use grovedb_costs::storage_cost::{ + removal::StorageRemovedBytes::BasicStorageRemoval, + transition::OperationStorageTransitionType, + }; use grovedb_version::version::GroveVersion; + use integer_encoding::VarInt; use crate::{ - batch::GroveDbOp, - reference_path::ReferencePathType::UpstreamFromElementHeightReference, + batch::QualifiedGroveDbOp, + reference_path::{ + ReferencePathType, ReferencePathType::UpstreamFromElementHeightReference, + }, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, }; @@ -41,17 +48,17 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"documents".to_vec()], b"key2".to_vec(), Element::new_item_with_flags(b"pizza".to_vec(), Some([0, 1].to_vec())), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"documents".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"documents".to_vec(), b"key3".to_vec()], b"key4".to_vec(), Element::new_reference(UpstreamFromElementHeightReference( @@ -175,17 +182,17 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"documents".to_vec()], b"key2".to_vec(), Element::new_item_with_flags(b"pizza".to_vec(), Some([0, 1].to_vec())), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"documents".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"documents".to_vec(), b"key3".to_vec()], b"key4".to_vec(), Element::new_reference(UpstreamFromElementHeightReference( @@ -247,7 +254,7 @@ mod tests { .get(&0) .expect("expected to have root path"); assert_eq!(ops_by_root_path.len(), 1); - let new_ops = vec![GroveDbOp::insert_op( + let new_ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"balances".to_vec()], b"person".to_vec(), Element::new_sum_item_with_flags(1000, Some([0, 1].to_vec())), @@ -311,4 +318,113 @@ mod tests { assert_ne!(apply_root_hash, apply_partial_root_hash); } + + #[test] + fn test_batch_root_one_update_tree_bigger_flags_with_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags(b"value1".to_vec(), Some(vec![0, 0])), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + // We are adding 2 bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 1])), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + Some(1), + ), + ), + ]; + + let _ = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + if new_flags[0] == 0 { + new_flags[0] = 1; + let new_flags_epoch = new_flags[1]; + new_flags[1] = old_flags.unwrap()[1]; + new_flags.push(new_flags_epoch); + new_flags.extend(cost.added_bytes.encode_var_vec()); + assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + Ok(true) + } else { + assert_eq!(new_flags[0], 1); + Ok(false) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => { + new_flags.extend(vec![1, 2]); + Ok(true) + } + _ => Ok(false), + }, + |_flags, removed_key_bytes, removed_value_bytes| { + Ok(( + BasicStorageRemoval(removed_key_bytes), + BasicStorageRemoval(removed_value_bytes), + )) + }, + Some(&tx), + grove_version, + ) + .cost; + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } } diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 7f9c119e..c4dc6bf1 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -48,10 +48,13 @@ use grovedb_costs::{ }; use grovedb_merk::{ tree::{ - kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, - value_hash, NULL_HASH, + kv::{ + ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, + KV, + }, + value_hash, TreeNode, NULL_HASH, }, - CryptoHash, Error as MerkError, Merk, MerkType, RootHashKeyAndSum, + CryptoHash, Error as MerkError, Merk, MerkType, Op, RootHashKeyAndSum, TreeFeatureType::{BasicMerkNode, SummedMerkNode}, }; use grovedb_path::SubtreePath; @@ -74,7 +77,7 @@ use crate::batch::estimated_costs::EstimatedCostsType; use crate::{ batch::{batch_structure::BatchStructure, mode::BatchRunMode}, element::{MaxReferenceHop, SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}, - operations::get::MAX_REFERENCE_HOPS, + operations::{get::MAX_REFERENCE_HOPS, proof::util::hex_to_ascii}, reference_path::{ path_from_reference_path_type, path_from_reference_qualified_path_type, ReferencePathType, }, @@ -83,7 +86,7 @@ use crate::{ /// Operations #[derive(Debug, PartialEq, Eq, Hash, Clone)] -pub enum Op { +pub enum GroveOp { /// Replace tree root key ReplaceTreeRootKey { /// Hash @@ -93,8 +96,13 @@ pub enum Op { /// Sum sum: Option, }, - /// Insert - Insert { + /// Inserts an element that is known to not yet exist + InsertOnly { + /// Element + element: Element, + }, + /// Inserts or Replaces an element + InsertOrReplace { /// Element element: Element, }, @@ -141,29 +149,30 @@ pub enum Op { DeleteSumTree, } -impl Op { +impl GroveOp { fn to_u8(&self) -> u8 { match self { - Op::DeleteTree => 0, - Op::DeleteSumTree => 1, - Op::Delete => 2, - Op::InsertTreeWithRootHash { .. } => 3, - Op::ReplaceTreeRootKey { .. } => 4, - Op::RefreshReference { .. } => 5, - Op::Replace { .. } => 6, - Op::Patch { .. } => 7, - Op::Insert { .. } => 8, + GroveOp::DeleteTree => 0, + GroveOp::DeleteSumTree => 1, + GroveOp::Delete => 2, + GroveOp::InsertTreeWithRootHash { .. } => 3, + GroveOp::ReplaceTreeRootKey { .. } => 4, + GroveOp::RefreshReference { .. } => 5, + GroveOp::Replace { .. } => 6, + GroveOp::Patch { .. } => 7, + GroveOp::InsertOrReplace { .. } => 8, + GroveOp::InsertOnly { .. } => 9, } } } -impl PartialOrd for Op { +impl PartialOrd for GroveOp { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } -impl Ord for Op { +impl Ord for GroveOp { fn cmp(&self, other: &Self) -> Ordering { self.to_u8().cmp(&other.to_u8()) } @@ -336,16 +345,16 @@ impl KeyInfoPath { /// Batch operation #[derive(Clone, PartialEq, Eq)] -pub struct GroveDbOp { +pub struct QualifiedGroveDbOp { /// Path to a subtree - subject to an operation pub path: KeyInfoPath, /// Key of an element in the subtree pub key: KeyInfo, /// Operation to perform on the key - pub op: Op, + pub op: GroveOp, } -impl fmt::Debug for GroveDbOp { +impl fmt::Debug for QualifiedGroveDbOp { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let mut path_out = Vec::new(); let path_drawer = Drawer::new(&mut path_out); @@ -355,10 +364,11 @@ impl fmt::Debug for GroveDbOp { self.key.visualize(key_drawer).unwrap(); let op_dbg = match &self.op { - Op::Insert { element } => format!("Insert {:?}", element), - Op::Replace { element } => format!("Replace {:?}", element), - Op::Patch { element, .. } => format!("Patch {:?}", element), - Op::RefreshReference { + GroveOp::InsertOrReplace { element } => format!("Insert Or Replace {:?}", element), + GroveOp::InsertOnly { element } => format!("Insert {:?}", element), + GroveOp::Replace { element } => format!("Replace {:?}", element), + GroveOp::Patch { element, .. } => format!("Patch {:?}", element), + GroveOp::RefreshReference { reference_path_type, max_reference_hop, trust_refresh_reference, @@ -369,11 +379,11 @@ impl fmt::Debug for GroveDbOp { reference_path_type, max_reference_hop, trust_refresh_reference ) } - Op::Delete => "Delete".to_string(), - Op::DeleteTree => "Delete Tree".to_string(), - Op::DeleteSumTree => "Delete Sum Tree".to_string(), - Op::ReplaceTreeRootKey { .. } => "Replace Tree Hash and Root Key".to_string(), - Op::InsertTreeWithRootHash { .. } => "Insert Tree Hash and Root Key".to_string(), + GroveOp::Delete => "Delete".to_string(), + GroveOp::DeleteTree => "Delete Tree".to_string(), + GroveOp::DeleteSumTree => "Delete Sum Tree".to_string(), + GroveOp::ReplaceTreeRootKey { .. } => "Replace Tree Hash and Root Key".to_string(), + GroveOp::InsertTreeWithRootHash { .. } => "Insert Tree Hash and Root Key".to_string(), }; f.debug_struct("GroveDbOp") @@ -384,14 +394,24 @@ impl fmt::Debug for GroveDbOp { } } -impl GroveDbOp { +impl QualifiedGroveDbOp { + /// An insert op using a known owned path and known key + pub fn insert_only_op(path: Vec>, key: Vec, element: Element) -> Self { + let path = KeyInfoPath::from_known_owned_path(path); + Self { + path, + key: KnownKey(key), + op: GroveOp::InsertOnly { element }, + } + } + /// An insert op using a known owned path and known key - pub fn insert_op(path: Vec>, key: Vec, element: Element) -> Self { + pub fn insert_or_replace_op(path: Vec>, key: Vec, element: Element) -> Self { let path = KeyInfoPath::from_known_owned_path(path); Self { path, key: KnownKey(key), - op: Op::Insert { element }, + op: GroveOp::InsertOrReplace { element }, } } @@ -400,7 +420,7 @@ impl GroveDbOp { Self { path, key, - op: Op::Insert { element }, + op: GroveOp::InsertOrReplace { element }, } } @@ -410,7 +430,7 @@ impl GroveDbOp { Self { path, key: KnownKey(key), - op: Op::Replace { element }, + op: GroveOp::Replace { element }, } } @@ -419,7 +439,7 @@ impl GroveDbOp { Self { path, key, - op: Op::Replace { element }, + op: GroveOp::Replace { element }, } } @@ -434,7 +454,7 @@ impl GroveDbOp { Self { path, key: KnownKey(key), - op: Op::Patch { + op: GroveOp::Patch { element, change_in_bytes, }, @@ -451,7 +471,7 @@ impl GroveDbOp { Self { path, key, - op: Op::Patch { + op: GroveOp::Patch { element, change_in_bytes, }, @@ -471,7 +491,7 @@ impl GroveDbOp { Self { path, key: KnownKey(key), - op: Op::RefreshReference { + op: GroveOp::RefreshReference { reference_path_type, max_reference_hop, flags, @@ -486,7 +506,7 @@ impl GroveDbOp { Self { path, key: KnownKey(key), - op: Op::Delete, + op: GroveOp::Delete, } } @@ -497,9 +517,9 @@ impl GroveDbOp { path, key: KnownKey(key), op: if is_sum_tree { - Op::DeleteSumTree + GroveOp::DeleteSumTree } else { - Op::DeleteTree + GroveOp::DeleteTree }, } } @@ -509,7 +529,7 @@ impl GroveDbOp { Self { path, key, - op: Op::Delete, + op: GroveOp::Delete, } } @@ -519,15 +539,17 @@ impl GroveDbOp { path, key, op: if is_sum_tree { - Op::DeleteSumTree + GroveOp::DeleteSumTree } else { - Op::DeleteTree + GroveOp::DeleteTree }, } } /// Verify consistency of operations - pub fn verify_consistency_of_operations(ops: &[GroveDbOp]) -> GroveDbOpConsistencyResults { + pub fn verify_consistency_of_operations( + ops: &[QualifiedGroveDbOp], + ) -> GroveDbOpConsistencyResults { let ops_len = ops.len(); // operations should not have any duplicates let mut repeated_ops = vec![]; @@ -564,7 +586,7 @@ impl GroveDbOp { None } }) - .collect::>(); + .collect::>(); if !doubled_ops.is_empty() { doubled_ops.push(op.op.clone()); same_path_key_ops.push((op.path.clone(), op.key.clone(), doubled_ops)); @@ -574,21 +596,23 @@ impl GroveDbOp { let inserts = ops .iter() .filter_map(|current_op| match current_op.op { - Op::Insert { .. } | Op::Replace { .. } => Some(current_op.clone()), + GroveOp::InsertOrReplace { .. } | GroveOp::Replace { .. } => { + Some(current_op.clone()) + } _ => None, }) - .collect::>(); + .collect::>(); let deletes = ops .iter() .filter_map(|current_op| { - if let Op::Delete = current_op.op { + if let GroveOp::Delete = current_op.op { Some(current_op.clone()) } else { None } }) - .collect::>(); + .collect::>(); let mut insert_ops_below_deleted_ops = vec![]; @@ -610,7 +634,7 @@ impl GroveDbOp { None } }) - .collect::>(); + .collect::>(); if !inserts_with_deleted_ops_above.is_empty() { insert_ops_below_deleted_ops .push((deleted_op.clone(), inserts_with_deleted_ops_above)); @@ -628,10 +652,13 @@ impl GroveDbOp { /// Results of a consistency check on an operation batch #[derive(Debug)] pub struct GroveDbOpConsistencyResults { - repeated_ops: Vec<(GroveDbOp, u16)>, // the u16 is count - same_path_key_ops: Vec<(KeyInfoPath, KeyInfo, Vec)>, - insert_ops_below_deleted_ops: Vec<(GroveDbOp, Vec)>, /* the deleted op first, - * then inserts under */ + /// Repeated Ops, the second u16 element represents the count + repeated_ops: Vec<(QualifiedGroveDbOp, u16)>, + /// The same path key ops + same_path_key_ops: Vec<(KeyInfoPath, KeyInfo, Vec)>, + /// This shows issues when we delete a tree but insert under the deleted + /// tree Deleted ops are first, with inserts under them in a tree + insert_ops_below_deleted_ops: Vec<(QualifiedGroveDbOp, Vec)>, } impl GroveDbOpConsistencyResults { @@ -656,7 +683,7 @@ impl fmt::Debug for TreeCacheMerkByPath { } trait TreeCache { - fn insert(&mut self, op: &GroveDbOp, is_sum_tree: bool) -> CostResult<(), Error>; + fn insert(&mut self, op: &QualifiedGroveDbOp, is_sum_tree: bool) -> CostResult<(), Error>; fn get_batch_run_mode(&self) -> BatchRunMode; @@ -664,8 +691,8 @@ trait TreeCache { fn execute_ops_on_path( &mut self, path: &KeyInfoPath, - ops_at_path_by_key: BTreeMap, - ops_by_qualified_paths: &BTreeMap>, Op>, + ops_at_path_by_key: BTreeMap, + ops_by_qualified_paths: &BTreeMap>, GroveOp>, batch_apply_options: &BatchApplyOptions, flags_update: &mut G, split_removal_bytes: &mut SR, @@ -725,22 +752,34 @@ where /// deserializing the referenced element. /// * `Error::InvalidBatchOperation`: If the referenced element points to a /// tree being updated. - fn process_reference<'a>( + fn process_reference<'a, G, SR>( &'a mut self, qualified_path: &[Vec], - ops_by_qualified_paths: &'a BTreeMap>, Op>, + ops_by_qualified_paths: &'a BTreeMap>, GroveOp>, recursions_allowed: u8, intermediate_reference_info: Option<&'a ReferencePathType>, + flags_update: &mut G, + split_removal_bytes: &mut SR, grove_version: &GroveVersion, - ) -> CostResult { + ) -> CostResult + where + G: FnMut(&StorageCost, Option, &mut ElementFlags) -> Result, + SR: FnMut( + &mut ElementFlags, + u32, + u32, + ) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, + { let mut cost = OperationCost::default(); let (key, reference_path) = qualified_path.split_last().unwrap(); // already checked - let reference_merk_wrapped = self - .merks - .remove(reference_path) - .map(|x| Ok(x).wrap_with_cost(Default::default())) - .unwrap_or_else(|| (self.get_merk_fn)(reference_path, false)); - let merk = cost_return_on_error!(&mut cost, reference_merk_wrapped); + + let merk = match self.merks.entry(reference_path.to_vec()) { + HashMapEntry::Occupied(o) => o.into_mut(), + HashMapEntry::Vacant(v) => v.insert(cost_return_on_error!( + &mut cost, + (self.get_merk_fn)(reference_path, false) + )), + }; // Here the element being referenced doesn't change in the same batch // and the max hop count is 1, meaning it should point directly to the base @@ -786,6 +825,8 @@ where path.as_slice(), ops_by_qualified_paths, recursions_allowed - 1, + flags_update, + split_removal_bytes, grove_version, ) } else { @@ -793,33 +834,82 @@ where // but the hop count is greater than 1, we can't just take the value hash from // the referenced element as an element further down in the chain might still // change in the batch. - let referenced_element = cost_return_on_error!( + self.process_reference_with_hop_count_greater_than_one( + key, + reference_path, + qualified_path, + ops_by_qualified_paths, + recursions_allowed, + flags_update, + split_removal_bytes, + grove_version, + ) + } + } + + /// Retrieves and deserializes the referenced element from the Merk tree. + /// + /// This function is responsible for fetching the referenced element using + /// the provided key and reference path, deserializing it into an + /// `Element`. It handles potential errors that can occur during these + /// operations, such as missing references or corrupted data. + /// + /// # Arguments + /// + /// * `key` - The key associated with the referenced element within the Merk + /// tree. + /// * `reference_path` - The path to the referenced element, used to locate + /// it in the Merk tree. + /// * `grove_version` - The current version of the GroveDB being used for + /// serialization and deserialization operations. + /// + /// # Returns + /// + /// * `Ok((Element, Vec, bool))` - Returns the deserialized `Element` + /// and the serialized counterpart if the retrieval and deserialization + /// are successful, wrapped in the associated cost. Also returns if the + /// merk of the element is a sum tree as a bool. + /// * `Err(Error)` - Returns an error if any issue occurs during the + /// retrieval or deserialization of the referenced element. + /// + /// # Errors + /// + /// This function may return the following errors: + /// + /// * `Error::MissingReference` - If the referenced element is missing from + /// the Merk tree. + /// * `Error::CorruptedData` - If the referenced element cannot be + /// deserialized due to corrupted data. + fn get_and_deserialize_referenced_element<'a>( + &'a mut self, + key: &[u8], + reference_path: &[Vec], + grove_version: &GroveVersion, + ) -> CostResult, bool)>, Error> { + let mut cost = OperationCost::default(); + + let merk = match self.merks.entry(reference_path.to_vec()) { + HashMapEntry::Occupied(o) => o.into_mut(), + HashMapEntry::Vacant(v) => v.insert(cost_return_on_error!( &mut cost, - merk.get( - key.as_ref(), - true, - Some(Element::value_defined_cost_for_serialized_value), - grove_version - ) - .map_err(|e| Error::CorruptedData(e.to_string())) - ); + (self.get_merk_fn)(reference_path, false) + )), + }; - let referenced_element = cost_return_on_error_no_add!( - &cost, - referenced_element.ok_or({ - let reference_string = reference_path - .iter() - .map(hex::encode) - .collect::>() - .join("/"); - Error::MissingReference(format!( - "reference to path:`{}` key:`{}` in batch is missing", - reference_string, - hex::encode(key) - )) - }) - ); + let referenced_element = cost_return_on_error!( + &mut cost, + merk.get( + key.as_ref(), + true, + Some(Element::value_defined_cost_for_serialized_value), + grove_version + ) + .map_err(|e| Error::CorruptedData(e.to_string())) + ); + let is_sum_tree = merk.is_sum_tree; + + if let Some(referenced_element) = referenced_element { let element = cost_return_on_error_no_add!( &cost, Element::deserialize(referenced_element.as_slice(), grove_version).map_err(|_| { @@ -827,30 +917,122 @@ where }) ); - match element { - Element::Item(..) | Element::SumItem(..) => { - let serialized = - cost_return_on_error_no_add!(&cost, element.serialize(grove_version)); - let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); - Ok(val_hash).wrap_with_cost(cost) - } - Element::Reference(path, ..) => { - let path = cost_return_on_error_no_add!( - &cost, - path_from_reference_qualified_path_type(path, qualified_path) - ); - self.follow_reference_get_value_hash( - path.as_slice(), - ops_by_qualified_paths, - recursions_allowed - 1, - grove_version, - ) - } - Element::Tree(..) | Element::SumTree(..) => Err(Error::InvalidBatchOperation( - "references can not point to trees being updated", - )) - .wrap_with_cost(cost), + Ok(Some((element, referenced_element, is_sum_tree))).wrap_with_cost(cost) + } else { + Ok(None).wrap_with_cost(cost) + } + } + + /// Processes a reference with a hop count greater than one, handling the + /// retrieval and further processing of the referenced element. + /// + /// This function is used when the hop count is greater than 1, meaning that + /// the reference points to another element that may also be a reference. + /// It handles retrieving the referenced element, deserializing it, and + /// determining the appropriate action based on the type of the element. + /// + /// # Arguments + /// + /// * `key` - The key corresponding to the referenced element in the current + /// Merk tree. + /// * `reference_path` - The path to the referenced element within the + /// current batch of operations. + /// * `qualified_path` - The fully qualified path to the reference, already + /// validated as a valid path. + /// * `ops_by_qualified_paths` - A map of qualified paths to their + /// corresponding batch operations. Used to track and manage updates + /// within the batch. + /// * `recursions_allowed` - The maximum allowed hop count to reach the + /// final target element. Each recursive call reduces this by one. + /// * `flags_update` - A mutable closure that handles updating element flags + /// during the processing of the reference. + /// * `split_removal_bytes` - A mutable closure that handles splitting and + /// managing the removal of bytes during the processing of the reference. + /// * `grove_version` - The current version of the GroveDB being used for + /// serialization and deserialization operations. + /// + /// # Returns + /// + /// * `Ok(CryptoHash)` - Returns the crypto hash of the referenced element + /// if successful, wrapped in the associated cost. + /// * `Err(Error)` - Returns an error if there is an issue with the + /// operation, such as a missing reference, corrupted data, or an invalid + /// batch operation. + /// + /// # Errors + /// + /// This function will return `Err(Error)` if any issues are encountered + /// during the processing of the reference. Possible errors include: + /// + /// * `Error::MissingReference` - If a direct or indirect reference to the + /// target element is missing in the batch. + /// * `Error::CorruptedData` - If there is an issue while retrieving or + /// deserializing the referenced element. + /// * `Error::InvalidBatchOperation` - If the referenced element points to a + /// tree being updated, which is not allowed. + fn process_reference_with_hop_count_greater_than_one<'a, G, SR>( + &'a mut self, + key: &[u8], + reference_path: &[Vec], + qualified_path: &[Vec], + ops_by_qualified_paths: &'a BTreeMap>, GroveOp>, + recursions_allowed: u8, + flags_update: &mut G, + split_removal_bytes: &mut SR, + grove_version: &GroveVersion, + ) -> CostResult + where + G: FnMut(&StorageCost, Option, &mut ElementFlags) -> Result, + SR: FnMut( + &mut ElementFlags, + u32, + u32, + ) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, + { + let mut cost = OperationCost::default(); + + let Some((element, ..)) = cost_return_on_error!( + &mut cost, + self.get_and_deserialize_referenced_element(key, reference_path, grove_version) + ) else { + let reference_string = reference_path + .iter() + .map(hex::encode) + .collect::>() + .join("/"); + return Err(Error::MissingReference(format!( + "reference to path:`{}` key:`{}` in batch is missing", + reference_string, + hex::encode(key) + ))) + .wrap_with_cost(cost); + }; + + match element { + Element::Item(..) | Element::SumItem(..) => { + let serialized = + cost_return_on_error_no_add!(&cost, element.serialize(grove_version)); + let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) } + Element::Reference(path, ..) => { + let path = cost_return_on_error_no_add!( + &cost, + path_from_reference_qualified_path_type(path, qualified_path) + ); + self.follow_reference_get_value_hash( + path.as_slice(), + ops_by_qualified_paths, + recursions_allowed - 1, + flags_update, + split_removal_bytes, + grove_version, + ) + } + Element::Tree(..) | Element::SumTree(..) => Err(Error::InvalidBatchOperation( + "references can not point to trees being updated", + )) + .wrap_with_cost(cost), } } @@ -864,35 +1046,126 @@ where /// insert ref_3 and another operation to change something in the /// reference chain in the same batch. /// All these has to be taken into account. - fn follow_reference_get_value_hash<'a>( + fn follow_reference_get_value_hash<'a, G, SR>( &'a mut self, qualified_path: &[Vec], - ops_by_qualified_paths: &'a BTreeMap>, Op>, + ops_by_qualified_paths: &'a BTreeMap>, GroveOp>, recursions_allowed: u8, + flags_update: &mut G, + split_removal_bytes: &mut SR, grove_version: &GroveVersion, - ) -> CostResult { + ) -> CostResult + where + G: FnMut(&StorageCost, Option, &mut ElementFlags) -> Result, + SR: FnMut( + &mut ElementFlags, + u32, + u32, + ) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, + { let mut cost = OperationCost::default(); if recursions_allowed == 0 { return Err(Error::ReferenceLimit).wrap_with_cost(cost); } // If the element being referenced changes in the same batch // we need to set the value_hash based on the new change and not the old state. + + // However the operation might either be merged or unmerged, if it is unmerged + // we need to merge it with the state first if let Some(op) = ops_by_qualified_paths.get(qualified_path) { // the path is being modified, inserted or deleted in the batch of operations match op { - Op::ReplaceTreeRootKey { .. } | Op::InsertTreeWithRootHash { .. } => Err( + GroveOp::ReplaceTreeRootKey { .. } | GroveOp::InsertTreeWithRootHash { .. } => Err( Error::InvalidBatchOperation("references can not point to trees being updated"), ) .wrap_with_cost(cost), - Op::Insert { element } | Op::Replace { element } | Op::Patch { element, .. } => { + GroveOp::InsertOrReplace { element } + | GroveOp::Replace { element } + | GroveOp::Patch { element, .. } => { match element { Element::Item(..) | Element::SumItem(..) => { let serialized = cost_return_on_error_no_add!( &cost, element.serialize(grove_version) ); - let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); - Ok(val_hash).wrap_with_cost(cost) + if element.get_flags().is_none() { + // There are no storage flags, we can just hash new element + let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) + } else { + let mut new_element = element.clone(); + let new_flags = new_element.get_flags_mut().as_mut().unwrap(); + + // it can be unmerged, let's get the value on disk + let (key, reference_path) = qualified_path.split_last().unwrap(); + let serialized_element_result = cost_return_on_error!( + &mut cost, + self.get_and_deserialize_referenced_element( + key, + reference_path, + grove_version + ) + ); + if let Some((old_element, old_serialized_element, is_in_sum_tree)) = + serialized_element_result + { + let maybe_old_flags = old_element.get_flags_owned(); + + let old_storage_cost = + KV::node_byte_cost_size_for_key_and_raw_value_lengths( + key.len() as u32, + old_serialized_element.len() as u32, + is_in_sum_tree, + ); + let new_storage_cost = + KV::node_byte_cost_size_for_key_and_raw_value_lengths( + key.len() as u32, + serialized.len() as u32, + is_in_sum_tree, + ); + + // There are storage flags + let storage_costs = TreeNode::storage_cost_for_update( + new_storage_cost, + old_storage_cost, + ); + + let changed = cost_return_on_error_no_add!( + &cost, + (flags_update)(&storage_costs, maybe_old_flags, new_flags) + .map_err(|e| match e { + Error::JustInTimeElementFlagsClientError(_) => { + MerkError::ClientCorruptionError(e.to_string()) + .into() + } + _ => MerkError::ClientCorruptionError( + "non client error".to_string(), + ) + .into(), + }) + ); + if changed { + // There are no storage flags, we can just hash new element + let serialized = cost_return_on_error_no_add!( + &cost, + new_element.serialize(grove_version) + ); + let val_hash = + value_hash(&serialized).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) + } else { + // There are no storage flags, we can just hash new element + + let val_hash = + value_hash(&serialized).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) + } + } else { + let val_hash = + value_hash(&serialized).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) + } + } } Element::Reference(path, ..) => { let path = cost_return_on_error_no_add!( @@ -906,6 +1179,8 @@ where path.as_slice(), ops_by_qualified_paths, recursions_allowed - 1, + flags_update, + split_removal_bytes, grove_version, ) } @@ -917,7 +1192,33 @@ where } } } - Op::RefreshReference { + GroveOp::InsertOnly { element } => match element { + Element::Item(..) | Element::SumItem(..) => { + let serialized = + cost_return_on_error_no_add!(&cost, element.serialize(grove_version)); + let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) + } + Element::Reference(path, ..) => { + let path = cost_return_on_error_no_add!( + &cost, + path_from_reference_qualified_path_type(path.clone(), qualified_path) + ); + self.follow_reference_get_value_hash( + path.as_slice(), + ops_by_qualified_paths, + recursions_allowed - 1, + flags_update, + split_removal_bytes, + grove_version, + ) + } + Element::Tree(..) | Element::SumTree(..) => Err(Error::InvalidBatchOperation( + "references can not point to trees being updated", + )) + .wrap_with_cost(cost), + }, + GroveOp::RefreshReference { reference_path_type, trust_refresh_reference, .. @@ -933,10 +1234,12 @@ where ops_by_qualified_paths, recursions_allowed, reference_info, + flags_update, + split_removal_bytes, grove_version, ) } - Op::Delete | Op::DeleteTree | Op::DeleteSumTree => { + GroveOp::Delete | GroveOp::DeleteTree | GroveOp::DeleteSumTree => { Err(Error::InvalidBatchOperation( "references can not point to something currently being deleted", )) @@ -949,6 +1252,8 @@ where ops_by_qualified_paths, recursions_allowed, None, + flags_update, + split_removal_bytes, grove_version, ) } @@ -966,7 +1271,7 @@ where F: FnMut(&[Vec], bool) -> CostResult, Error>, S: StorageContext<'db>, { - fn insert(&mut self, op: &GroveDbOp, is_sum_tree: bool) -> CostResult<(), Error> { + fn insert(&mut self, op: &QualifiedGroveDbOp, is_sum_tree: bool) -> CostResult<(), Error> { let mut cost = OperationCost::default(); let mut inserted_path = op.path.to_path(); @@ -988,12 +1293,15 @@ where ) -> CostResult<(), Error> { let mut cost = OperationCost::default(); let base_path = vec![]; - let merk_wrapped = self - .merks - .remove(&base_path) - .map(|x| Ok(x).wrap_with_cost(Default::default())) - .unwrap_or_else(|| (self.get_merk_fn)(&[], false)); - let mut merk = cost_return_on_error!(&mut cost, merk_wrapped); + + let merk = match self.merks.entry(base_path.clone()) { + HashMapEntry::Occupied(o) => o.into_mut(), + HashMapEntry::Vacant(v) => v.insert(cost_return_on_error!( + &mut cost, + (self.get_merk_fn)(&base_path, false) + )), + }; + merk.set_base_root_key(root_key) .add_cost(cost) .map_err(|_| Error::InternalError("unable to set base root key".to_string())) @@ -1002,8 +1310,8 @@ where fn execute_ops_on_path( &mut self, path: &KeyInfoPath, - ops_at_path_by_key: BTreeMap, - ops_by_qualified_paths: &BTreeMap>, Op>, + ops_at_path_by_key: BTreeMap, + ops_by_qualified_paths: &BTreeMap>, GroveOp>, batch_apply_options: &BatchApplyOptions, flags_update: &mut G, split_removal_bytes: &mut SR, @@ -1014,121 +1322,130 @@ where let p = path.to_path(); let path = &p; - let merk_wrapped = self - .merks - .remove(path) - .map(|x| Ok(x).wrap_with_cost(Default::default())) - .unwrap_or_else(|| (self.get_merk_fn)(path, false)); - let mut merk = cost_return_on_error!(&mut cost, merk_wrapped); - let is_sum_tree = merk.is_sum_tree; + // This also populates Merk trees cache + let is_sum_tree = { + let merk = match self.merks.entry(path.to_vec()) { + HashMapEntry::Occupied(o) => o.into_mut(), + HashMapEntry::Vacant(v) => v.insert(cost_return_on_error!( + &mut cost, + (self.get_merk_fn)(path, false) + )), + }; + merk.is_sum_tree + }; - let mut batch_operations: Vec<(Vec, _)> = vec![]; + let mut batch_operations: Vec<(Vec, Op)> = vec![]; for (key_info, op) in ops_at_path_by_key.into_iter() { match op { - Op::Insert { element } | Op::Replace { element } | Op::Patch { element, .. } => { - match &element { - Element::Reference(path_reference, element_max_reference_hop, _) => { - let merk_feature_type = cost_return_on_error!( - &mut cost, - element - .get_feature_type(is_sum_tree) - .wrap_with_cost(OperationCost::default()) - ); - let path_reference = cost_return_on_error!( - &mut cost, - path_from_reference_path_type( - path_reference.clone(), - path, - Some(key_info.as_slice()) - ) + GroveOp::InsertOnly { element } + | GroveOp::InsertOrReplace { element } + | GroveOp::Replace { element } + | GroveOp::Patch { element, .. } => match &element { + Element::Reference(path_reference, element_max_reference_hop, _) => { + let merk_feature_type = cost_return_on_error!( + &mut cost, + element + .get_feature_type(is_sum_tree) .wrap_with_cost(OperationCost::default()) - ); - if path_reference.is_empty() { - return Err(Error::InvalidBatchOperation( - "attempting to insert an empty reference", - )) - .wrap_with_cost(cost); - } + ); + let path_reference = cost_return_on_error!( + &mut cost, + path_from_reference_path_type( + path_reference.clone(), + path, + Some(key_info.as_slice()) + ) + .wrap_with_cost(OperationCost::default()) + ); + if path_reference.is_empty() { + return Err(Error::InvalidBatchOperation( + "attempting to insert an empty reference", + )) + .wrap_with_cost(cost); + } - let referenced_element_value_hash = cost_return_on_error!( - &mut cost, - self.follow_reference_get_value_hash( - path_reference.as_slice(), - ops_by_qualified_paths, - element_max_reference_hop.unwrap_or(MAX_REFERENCE_HOPS as u8), - grove_version, - ) - ); + let referenced_element_value_hash = cost_return_on_error!( + &mut cost, + self.follow_reference_get_value_hash( + path_reference.as_slice(), + ops_by_qualified_paths, + element_max_reference_hop.unwrap_or(MAX_REFERENCE_HOPS as u8), + flags_update, + split_removal_bytes, + grove_version, + ) + ); - cost_return_on_error!( + cost_return_on_error!( + &mut cost, + element.insert_reference_into_batch_operations( + key_info.get_key_clone(), + referenced_element_value_hash, + &mut batch_operations, + merk_feature_type, + grove_version, + ) + ); + } + Element::Tree(..) | Element::SumTree(..) => { + let merk_feature_type = cost_return_on_error!( + &mut cost, + element + .get_feature_type(is_sum_tree) + .wrap_with_cost(OperationCost::default()) + ); + cost_return_on_error!( + &mut cost, + element.insert_subtree_into_batch_operations( + key_info.get_key_clone(), + NULL_HASH, + false, + &mut batch_operations, + merk_feature_type, + grove_version, + ) + ); + } + Element::Item(..) | Element::SumItem(..) => { + let merk_feature_type = cost_return_on_error!( + &mut cost, + element + .get_feature_type(is_sum_tree) + .wrap_with_cost(OperationCost::default()) + ); + if batch_apply_options.validate_insertion_does_not_override { + let merk = self.merks.get_mut(path).expect("the Merk is cached"); + + let inserted = cost_return_on_error!( &mut cost, - element.insert_reference_into_batch_operations( - key_info.get_key_clone(), - referenced_element_value_hash, + element.insert_if_not_exists_into_batch_operations( + merk, + key_info.get_key(), &mut batch_operations, merk_feature_type, grove_version, ) ); - } - Element::Tree(..) | Element::SumTree(..) => { - let merk_feature_type = cost_return_on_error!( - &mut cost, - element - .get_feature_type(is_sum_tree) - .wrap_with_cost(OperationCost::default()) - ); + if !inserted { + return Err(Error::InvalidBatchOperation( + "attempting to overwrite a tree", + )) + .wrap_with_cost(cost); + } + } else { cost_return_on_error!( &mut cost, - element.insert_subtree_into_batch_operations( - key_info.get_key_clone(), - NULL_HASH, - false, + element.insert_into_batch_operations( + key_info.get_key(), &mut batch_operations, merk_feature_type, grove_version, ) ); } - Element::Item(..) | Element::SumItem(..) => { - let merk_feature_type = cost_return_on_error!( - &mut cost, - element - .get_feature_type(is_sum_tree) - .wrap_with_cost(OperationCost::default()) - ); - if batch_apply_options.validate_insertion_does_not_override { - let inserted = cost_return_on_error!( - &mut cost, - element.insert_if_not_exists_into_batch_operations( - &mut merk, - key_info.get_key(), - &mut batch_operations, - merk_feature_type, - grove_version, - ) - ); - if !inserted { - return Err(Error::InvalidBatchOperation( - "attempting to overwrite a tree", - )) - .wrap_with_cost(cost); - } - } else { - cost_return_on_error!( - &mut cost, - element.insert_into_batch_operations( - key_info.get_key(), - &mut batch_operations, - merk_feature_type, - grove_version, - ) - ); - } - } } - } - Op::RefreshReference { + }, + GroveOp::RefreshReference { reference_path_type, max_reference_hop, flags, @@ -1140,6 +1457,7 @@ where let element = if trust_refresh_reference { Element::Reference(reference_path_type, max_reference_hop, flags) } else { + let merk = self.merks.get(path).expect("the Merk is cached"); let value = cost_return_on_error!( &mut cost, merk.get( @@ -1199,6 +1517,8 @@ where path_reference.as_slice(), ops_by_qualified_paths, max_reference_hop.unwrap_or(MAX_REFERENCE_HOPS as u8), + flags_update, + split_removal_bytes, grove_version ) ); @@ -1214,7 +1534,7 @@ where ) ); } - Op::Delete => { + GroveOp::Delete => { cost_return_on_error!( &mut cost, Element::delete_into_batch_operations( @@ -1227,7 +1547,7 @@ where ) ); } - Op::DeleteTree => { + GroveOp::DeleteTree => { cost_return_on_error!( &mut cost, Element::delete_into_batch_operations( @@ -1239,7 +1559,7 @@ where ) ); } - Op::DeleteSumTree => { + GroveOp::DeleteSumTree => { cost_return_on_error!( &mut cost, Element::delete_into_batch_operations( @@ -1251,11 +1571,12 @@ where ) ); } - Op::ReplaceTreeRootKey { + GroveOp::ReplaceTreeRootKey { hash, root_key, sum, } => { + let merk = self.merks.get(path).expect("the Merk is cached"); cost_return_on_error!( &mut cost, GroveDb::update_tree_item_preserve_flag_into_batch_operations( @@ -1269,7 +1590,7 @@ where ) ); } - Op::InsertTreeWithRootHash { + GroveOp::InsertTreeWithRootHash { hash, root_key, flags, @@ -1298,6 +1619,9 @@ where } } } + + let merk = self.merks.get_mut(path).expect("the Merk is cached"); + cost_return_on_error!( &mut cost, merk.apply_unchecked::<_, Vec, _, _, _, _>( @@ -1391,8 +1715,7 @@ where .root_hash_key_and_sum() .add_cost(cost) .map_err(Error::MerkError); - // We need to reinsert the merk - self.merks.insert(path.clone(), merk); + r } @@ -1501,16 +1824,19 @@ impl GroveDb { { match ops_on_path.entry(key.clone()) { Entry::Vacant(vacant_entry) => { - vacant_entry.insert(Op::ReplaceTreeRootKey { - hash: root_hash, - root_key: calculated_root_key, - sum: sum_value, - }); + vacant_entry.insert( + GroveOp::ReplaceTreeRootKey { + hash: root_hash, + root_key: calculated_root_key, + sum: sum_value, + } + .into(), + ); } Entry::Occupied(occupied_entry) => { let mutable_occupied_entry = occupied_entry.into_mut(); match mutable_occupied_entry { - Op::ReplaceTreeRootKey { + GroveOp::ReplaceTreeRootKey { hash, root_key, sum, @@ -1519,33 +1845,36 @@ impl GroveDb { *root_key = calculated_root_key; *sum = sum_value; } - Op::InsertTreeWithRootHash { .. } => { + GroveOp::InsertTreeWithRootHash { .. } => { return Err(Error::CorruptedCodeExecution( "we can not do this operation twice", )) .wrap_with_cost(cost); } - Op::Insert { element } - | Op::Replace { element } - | Op::Patch { element, .. } => { + GroveOp::InsertOrReplace { element } + | GroveOp::InsertOnly { element } + | GroveOp::Replace { element } + | GroveOp::Patch { element, .. } => { if let Element::Tree(_, flags) = element { *mutable_occupied_entry = - Op::InsertTreeWithRootHash { + GroveOp::InsertTreeWithRootHash { hash: root_hash, root_key: calculated_root_key, flags: flags.clone(), sum: None, - }; + } + .into(); } else if let Element::SumTree(.., flags) = element { *mutable_occupied_entry = - Op::InsertTreeWithRootHash { + GroveOp::InsertTreeWithRootHash { hash: root_hash, root_key: calculated_root_key, flags: flags.clone(), sum: sum_value, - }; + } + .into(); } else { return Err(Error::InvalidBatchOperation( "insertion of element under a non tree", @@ -1553,14 +1882,16 @@ impl GroveDb { .wrap_with_cost(cost); } } - Op::RefreshReference { .. } => { + GroveOp::RefreshReference { .. } => { return Err(Error::InvalidBatchOperation( "insertion of element under a refreshed \ reference", )) .wrap_with_cost(cost); } - Op::Delete | Op::DeleteTree | Op::DeleteSumTree => { + GroveOp::Delete + | GroveOp::DeleteTree + | GroveOp::DeleteSumTree => { if calculated_root_key.is_some() { return Err(Error::InvalidBatchOperation( "modification of tree when it will be \ @@ -1573,10 +1904,11 @@ impl GroveDb { } } } else { - let mut ops_on_path: BTreeMap = BTreeMap::new(); + let mut ops_on_path: BTreeMap = + BTreeMap::new(); ops_on_path.insert( key.clone(), - Op::ReplaceTreeRootKey { + GroveOp::ReplaceTreeRootKey { hash: root_hash, root_key: calculated_root_key, sum: sum_value, @@ -1585,17 +1917,20 @@ impl GroveDb { ops_at_level_above.insert(parent_path, ops_on_path); } } else { - let mut ops_on_path: BTreeMap = BTreeMap::new(); + let mut ops_on_path: BTreeMap = BTreeMap::new(); ops_on_path.insert( key.clone(), - Op::ReplaceTreeRootKey { + GroveOp::ReplaceTreeRootKey { hash: root_hash, root_key: calculated_root_key, sum: sum_value, - }, + } + .into(), ); - let mut ops_on_level: BTreeMap> = - BTreeMap::new(); + let mut ops_on_level: BTreeMap< + KeyInfoPath, + BTreeMap, + > = BTreeMap::new(); ops_on_level.insert(KeyInfoPath(parent_path.to_vec()), ops_on_path); ops_by_level_paths.insert(current_level - 1, ops_on_level); } @@ -1617,7 +1952,7 @@ impl GroveDb { /// Then return the list of leftover operations fn apply_body<'db, S: StorageContext<'db>>( &self, - ops: Vec, + ops: Vec, batch_apply_options: Option, update_element_flags_function: impl FnMut( &StorageCost, @@ -1662,7 +1997,7 @@ impl GroveDb { fn continue_partial_apply_body<'db, S: StorageContext<'db>>( &self, previous_leftover_operations: Option, - additional_ops: Vec, + additional_ops: Vec, batch_apply_options: Option, update_element_flags_function: impl FnMut( &StorageCost, @@ -1708,7 +2043,7 @@ impl GroveDb { /// Applies operations on GroveDB without batching pub fn apply_operations_without_batching( &self, - ops: Vec, + ops: Vec, options: Option, transaction: TransactionArg, grove_version: &GroveVersion, @@ -1723,7 +2058,7 @@ impl GroveDb { let mut cost = OperationCost::default(); for op in ops.into_iter() { match op.op { - Op::Insert { element } | Op::Replace { element } => { + GroveOp::InsertOrReplace { element } | GroveOp::Replace { element } => { // TODO: paths in batches is something to think about let path_slices: Vec<&[u8]> = op.path.iterator().map(|p| p.as_slice()).collect(); @@ -1739,7 +2074,7 @@ impl GroveDb { ) ); } - Op::Delete => { + GroveOp::Delete => { let path_slices: Vec<&[u8]> = op.path.iterator().map(|p| p.as_slice()).collect(); cost_return_on_error!( @@ -1762,7 +2097,7 @@ impl GroveDb { /// Applies batch on GroveDB pub fn apply_batch( &self, - ops: Vec, + ops: Vec, batch_apply_options: Option, transaction: TransactionArg, grove_version: &GroveVersion, @@ -1789,12 +2124,12 @@ impl GroveDb { /// Applies batch on GroveDB pub fn apply_partial_batch( &self, - ops: Vec, + ops: Vec, batch_apply_options: Option, cost_based_add_on_operations: impl FnMut( &OperationCost, &Option, - ) -> Result, Error>, + ) -> Result, Error>, transaction: TransactionArg, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -1858,8 +2193,14 @@ impl GroveDb { Element::get_from_storage(&parent_storage, parent_key, grove_version).map_err( |_| { Error::InvalidPath(format!( - "could not get key for parent of subtree for batch at path {}", - parent_path.to_vec().into_iter().map(hex::encode).join("/") + "could not get key for parent of subtree for batch at path [{}] \ + for key {}", + parent_path + .to_vec() + .into_iter() + .map(|v| hex_to_ascii(&v)) + .join("/"), + hex_to_ascii(parent_key) )) } ) @@ -1969,7 +2310,7 @@ impl GroveDb { /// Applies batch of operations on GroveDB pub fn apply_batch_with_element_flags_update( &self, - ops: Vec, + ops: Vec, batch_apply_options: Option, update_element_flags_function: impl FnMut( &StorageCost, @@ -2009,7 +2350,7 @@ impl GroveDb { .unwrap_or(true); if check_batch_operation_consistency { - let consistency_result = GroveDbOp::verify_consistency_of_operations(&ops); + let consistency_result = QualifiedGroveDbOp::verify_consistency_of_operations(&ops); if !consistency_result.is_empty() { return Err(Error::InvalidBatchOperation( "batch operations fail consistency checks", @@ -2060,6 +2401,21 @@ impl GroveDb { .commit_multi_context_batch(storage_batch, Some(tx)) .map_err(|e| e.into()) ); + + // Keep this commented for easy debugging in the future. + // let issues = self + // .visualize_verify_grovedb(Some(tx), true, + // &Default::default()) .unwrap(); + // if issues.len() > 0 { + // println!( + // "tx_issues: {}", + // issues + // .iter() + // .map(|(hash, (a, b, c))| format!("{}: {} {} {}", + // hash, a, b, c)) .collect::>() + // .join(" | ") + // ); + // } } else { cost_return_on_error!( &mut cost, @@ -2087,6 +2443,21 @@ impl GroveDb { .commit_multi_context_batch(storage_batch, None) .map_err(|e| e.into()) ); + + // Keep this commented for easy debugging in the future. + // let issues = self + // .visualize_verify_grovedb(None, true, &Default::default()) + // .unwrap(); + // if issues.len() > 0 { + // println!( + // "non_tx_issues: {}", + // issues + // .iter() + // .map(|(hash, (a, b, c))| format!("{}: {} {} {}", + // hash, a, b, c)) .collect::>() + // .join(" | ") + // ); + // } } Ok(()).wrap_with_cost(cost) } @@ -2097,7 +2468,7 @@ impl GroveDb { /// If it is not set we default to pausing at the root tree pub fn apply_partial_batch_with_element_flags_update( &self, - ops: Vec, + ops: Vec, batch_apply_options: Option, mut update_element_flags_function: impl FnMut( &StorageCost, @@ -2115,7 +2486,7 @@ impl GroveDb { mut add_on_operations: impl FnMut( &OperationCost, &Option, - ) -> Result, Error>, + ) -> Result, Error>, transaction: TransactionArg, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -2145,7 +2516,7 @@ impl GroveDb { !batch_apply_options.disable_operation_consistency_check; if check_batch_operation_consistency { - let consistency_result = GroveDbOp::verify_consistency_of_operations(&ops); + let consistency_result = QualifiedGroveDbOp::verify_consistency_of_operations(&ops); if !consistency_result.is_empty() { return Err(Error::InvalidBatchOperation( "batch operations fail consistency checks", @@ -2346,7 +2717,7 @@ impl GroveDb { /// ops pub fn estimated_case_operations_for_batch( estimated_costs_type: EstimatedCostsType, - ops: Vec, + ops: Vec, batch_apply_options: Option, update_element_flags_function: impl FnMut( &StorageCost, @@ -2447,28 +2818,32 @@ mod tests { let element = Element::new_item(b"ayy".to_vec()); let element2 = Element::new_item(b"ayy2".to_vec()); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element.clone(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"key2".to_vec(), element2.clone(), @@ -2528,8 +2903,16 @@ mod tests { // No two operations should be the same let ops = vec![ - GroveDbOp::insert_op(vec![b"a".to_vec()], b"b".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op(vec![b"a".to_vec()], b"b".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"a".to_vec()], + b"b".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"a".to_vec()], + b"b".to_vec(), + Element::empty_tree(), + ), ]; assert!(matches!( db.apply_batch(ops, None, None, grove_version).unwrap(), @@ -2540,12 +2923,16 @@ mod tests { // Can't perform 2 or more operations on the same node let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"a".to_vec()], b"b".to_vec(), Element::new_item(vec![1]), ), - GroveDbOp::insert_op(vec![b"a".to_vec()], b"b".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"a".to_vec()], + b"b".to_vec(), + Element::empty_tree(), + ), ]; assert!(matches!( db.apply_batch(ops, None, None, grove_version).unwrap(), @@ -2556,12 +2943,12 @@ mod tests { // Can't insert under a deleted path let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"b".to_vec(), Element::new_item(vec![1]), ), - GroveDbOp::delete_op(vec![], TEST_LEAF.to_vec()), + QualifiedGroveDbOp::delete_op(vec![], TEST_LEAF.to_vec()), ]; assert!(matches!( db.apply_batch(ops, None, None, grove_version).unwrap(), @@ -2572,12 +2959,12 @@ mod tests { // Should allow invalid operations pass when disable option is set to true let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"b".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"b".to_vec(), Element::empty_tree(), @@ -2622,28 +3009,32 @@ mod tests { let element = Element::new_item(b"ayy".to_vec()); let element2 = Element::new_item(b"ayy2".to_vec()); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element.clone(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"key2".to_vec(), element2.clone(), @@ -2721,28 +3112,28 @@ mod tests { let tx = db.start_transaction(); // let's start by inserting a tree structure let ops = vec![ - GroveDbOp::insert_op(vec![], b"1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op(vec![], b"1".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"1".to_vec()], b"my_contract".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"1".to_vec(), b"my_contract".to_vec()], b"0".to_vec(), Element::new_item(b"this is the contract".to_vec()), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"1".to_vec(), b"my_contract".to_vec()], b"1".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"1".to_vec(), b"my_contract".to_vec(), b"1".to_vec()], b"person".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2752,7 +3143,7 @@ mod tests { b"0".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2781,7 +3172,7 @@ mod tests { // now let's add an item let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2795,7 +3186,7 @@ mod tests { some_element_flags.clone(), ), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2806,7 +3197,7 @@ mod tests { b"my apples are safe".to_vec(), Element::empty_tree_with_flags(some_element_flags.clone()), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2818,7 +3209,7 @@ mod tests { b"0".to_vec(), Element::empty_tree_with_flags(some_element_flags.clone()), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2855,7 +3246,7 @@ mod tests { // now let's add an item let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2866,7 +3257,7 @@ mod tests { b"wisdom".to_vec(), Element::new_item_with_flags(b"Wisdom Ogwu".to_vec(), some_element_flags.clone()), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2877,7 +3268,7 @@ mod tests { b"canteloupe!".to_vec(), Element::empty_tree_with_flags(some_element_flags.clone()), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2889,7 +3280,7 @@ mod tests { b"0".to_vec(), Element::empty_tree_with_flags(some_element_flags.clone()), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2930,60 +3321,60 @@ mod tests { .expect("successful batch apply"); } - fn grove_db_ops_for_contract_insert() -> Vec { + fn grove_db_ops_for_contract_insert() -> Vec { let mut grove_db_ops = vec![]; - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![], b"contract".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec()], [0u8].to_vec(), Element::new_item(b"serialized_contract".to_vec()), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec()], [1u8].to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec()], b"domain".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"domain".to_vec()], [0u8].to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"domain".to_vec()], b"normalized_domain_label".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"domain".to_vec()], b"unique_records".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"domain".to_vec()], b"alias_records".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec()], b"preorder".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"preorder".to_vec()], [0u8].to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"preorder".to_vec()], b"salted_domain".to_vec(), Element::empty_tree(), @@ -2992,10 +3383,10 @@ mod tests { grove_db_ops } - fn grove_db_ops_for_contract_document_insert() -> Vec { + fn grove_db_ops_for_contract_document_insert() -> Vec { let mut grove_db_ops = vec![]; - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![ b"contract".to_vec(), [1u8].to_vec(), @@ -3006,7 +3397,7 @@ mod tests { Element::new_item(b"serialized_domain".to_vec()), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![ b"contract".to_vec(), [1u8].to_vec(), @@ -3017,7 +3408,7 @@ mod tests { Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![ b"contract".to_vec(), [1u8].to_vec(), @@ -3029,7 +3420,7 @@ mod tests { Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![ b"contract".to_vec(), [1u8].to_vec(), @@ -3042,7 +3433,7 @@ mod tests { Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![ b"contract".to_vec(), [1u8].to_vec(), @@ -3165,13 +3556,17 @@ mod tests { let db = make_test_grovedb(grove_version); let element = Element::new_item(b"ayy".to_vec()); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element, ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), @@ -3193,23 +3588,27 @@ mod tests { let db = make_test_grovedb(grove_version); let element = Element::new_item(b"ayy".to_vec()); let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"key2".to_vec(), element.clone(), ), - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element, ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), @@ -3257,17 +3656,17 @@ mod tests { .expect("cannot insert a subtree"); let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element, ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::delete_op(vec![b"key1".to_vec()], b"key2".to_vec()), + QualifiedGroveDbOp::delete_op(vec![b"key1".to_vec()], b"key2".to_vec()), ]; assert!(db .apply_batch(ops, None, None, grove_version) @@ -3281,23 +3680,27 @@ mod tests { let db = make_test_grovedb(grove_version); let element = Element::new_item(b"ayy".to_vec()); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element, ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), ), - GroveDbOp::delete_op(vec![b"key1".to_vec()], b"key2".to_vec()), + QualifiedGroveDbOp::delete_op(vec![b"key1".to_vec()], b"key2".to_vec()), ]; db.apply_batch(ops, None, None, grove_version) .unwrap() @@ -3340,7 +3743,7 @@ mod tests { .expect("cannot insert value"); // Insertion into scalar is invalid - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"invalid".to_vec()], b"key1".to_vec(), element.clone(), @@ -3351,7 +3754,7 @@ mod tests { .is_err()); // Insertion into a tree is correct - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"valid".to_vec()], b"key1".to_vec(), element.clone(), @@ -3396,8 +3799,8 @@ mod tests { // TEST_LEAF can not be overwritten let ops = vec![ - GroveDbOp::insert_op(vec![], TEST_LEAF.to_vec(), element2), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op(vec![], TEST_LEAF.to_vec(), element2), + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key_subtree".to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -3423,8 +3826,8 @@ mod tests { // TEST_LEAF will be deleted so you can not insert underneath it let ops = vec![ - GroveDbOp::delete_op(vec![], TEST_LEAF.to_vec()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::delete_op(vec![], TEST_LEAF.to_vec()), + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -3439,8 +3842,8 @@ mod tests { // We are testing with the batch apply option // validate_tree_insertion_does_not_override set to true let ops = vec![ - GroveDbOp::delete_op(vec![], TEST_LEAF.to_vec()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::delete_op(vec![], TEST_LEAF.to_vec()), + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -3470,12 +3873,12 @@ mod tests { let grove_version = GroveVersion::latest(); let db = make_test_grovedb(grove_version); let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![], TEST_LEAF.to_vec(), Element::new_item(b"ayy".to_vec()), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -3526,7 +3929,7 @@ mod tests { ) .unwrap() .expect("cannot insert an item"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::new_item(b"ayy2".to_vec()), @@ -3590,23 +3993,27 @@ mod tests { let element2 = Element::new_item(b"ayy2".to_vec()); let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element.clone(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op(vec![TEST_LEAF.to_vec()], b"key".to_vec(), element2.clone()), - GroveDbOp::delete_op(vec![ANOTHER_TEST_LEAF.to_vec()], b"key1".to_vec()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec()], + b"key".to_vec(), + element2.clone(), + ), + QualifiedGroveDbOp::delete_op(vec![ANOTHER_TEST_LEAF.to_vec()], b"key1".to_vec()), ]; db.apply_batch(ops, None, None, grove_version) .unwrap() @@ -3692,7 +4099,7 @@ mod tests { } let element = Element::new_item(b"ayy".to_vec()); - let batch = vec![GroveDbOp::insert_op( + let batch = vec![QualifiedGroveDbOp::insert_or_replace_op( acc_path.clone(), b"key".to_vec(), element.clone(), @@ -3701,7 +4108,11 @@ mod tests { .unwrap() .expect("cannot apply batch"); - let batch = vec![GroveDbOp::insert_op(acc_path, b"key".to_vec(), element)]; + let batch = vec![QualifiedGroveDbOp::insert_or_replace_op( + acc_path, + b"key".to_vec(), + element, + )]; db.apply_batch(batch, None, None, grove_version) .unwrap() .expect("cannot apply same batch twice"); @@ -3730,7 +4141,7 @@ mod tests { let root_hash = db.root_hash(None, grove_version).unwrap().unwrap(); let element = Element::new_item(b"ayy".to_vec()); - let batch = vec![GroveDbOp::insert_op( + let batch = vec![QualifiedGroveDbOp::insert_or_replace_op( acc_path.clone(), b"key".to_vec(), element, @@ -3750,7 +4161,7 @@ mod tests { let grove_version = GroveVersion::latest(); // insert reference that points to non-existent item let db = make_test_grovedb(grove_version); - let batch = vec![GroveDbOp::insert_op( + let batch = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ @@ -3767,7 +4178,7 @@ mod tests { let db = make_test_grovedb(grove_version); let elem = Element::new_item(b"ayy".to_vec()); let batch = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ @@ -3775,7 +4186,7 @@ mod tests { b"invalid_path".to_vec(), ])), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"invalid_path".to_vec(), elem.clone(), @@ -3808,7 +4219,7 @@ mod tests { let db = make_test_grovedb(grove_version); let elem = Element::new_item(b"ayy".to_vec()); let batch = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key2".to_vec(), Element::new_reference_with_hops( @@ -3819,7 +4230,7 @@ mod tests { Some(1), ), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::new_reference_with_hops( @@ -3830,7 +4241,11 @@ mod tests { Some(1), ), ), - GroveDbOp::insert_op(vec![TEST_LEAF.to_vec()], b"invalid_path".to_vec(), elem), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec()], + b"invalid_path".to_vec(), + elem, + ), ]; assert!(matches!( db.apply_batch(batch, None, None, grove_version).unwrap(), diff --git a/grovedb/src/batch/multi_insert_cost_tests.rs b/grovedb/src/batch/multi_insert_cost_tests.rs index ad171d6d..8a1200e6 100644 --- a/grovedb/src/batch/multi_insert_cost_tests.rs +++ b/grovedb/src/batch/multi_insert_cost_tests.rs @@ -5,13 +5,18 @@ mod tests { use std::{ops::Add, option::Option::None}; use grovedb_costs::{ - storage_cost::{removal::StorageRemovedBytes::NoStorageRemoval, StorageCost}, + storage_cost::{ + removal::StorageRemovedBytes::{BasicStorageRemoval, NoStorageRemoval}, + transition::OperationStorageTransitionType, + StorageCost, + }, OperationCost, }; use grovedb_version::version::GroveVersion; + use integer_encoding::VarInt; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, reference_path::ReferencePathType::{SiblingReference, UpstreamFromElementHeightReference}, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, @@ -46,8 +51,16 @@ mod tests { let non_batch_cost = non_batch_cost_1.add(non_batch_cost_2); tx.rollback().expect("expected to rollback"); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op(vec![], b"key2".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key2".to_vec(), + Element::empty_tree(), + ), ]; let cost = db.apply_batch(ops, None, Some(&tx), grove_version).cost; assert_eq!( @@ -97,13 +110,17 @@ mod tests { let non_batch_cost = non_batch_cost_1.add(non_batch_cost_2).add(non_batch_cost_3); tx.rollback().expect("expected to rollback"); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key2".to_vec(), Element::new_item_with_flags(b"pizza".to_vec(), Some([0, 1].to_vec())), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key3".to_vec(), Element::new_reference(SiblingReference(b"key2".to_vec())), @@ -173,18 +190,22 @@ mod tests { .add(non_batch_cost_4); tx.rollback().expect("expected to rollback"); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::new_item_with_flags(b"pizza".to_vec(), Some([0, 1].to_vec())), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key3".to_vec()], b"key4".to_vec(), Element::new_reference(UpstreamFromElementHeightReference( @@ -212,8 +233,16 @@ mod tests { let tx = db.start_transaction(); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op(vec![], b"key2".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key2".to_vec(), + Element::empty_tree(), + ), ]; let cost_result = db.apply_batch(ops, None, Some(&tx), grove_version); cost_result.value.expect("expected to execute batch"); @@ -268,8 +297,12 @@ mod tests { let tx = db.start_transaction(); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), @@ -322,4 +355,111 @@ mod tests { } ); } + + #[test] + fn test_batch_insert_item_in_also_inserted_sub_tree_with_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"tree2".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec(), b"tree2".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value".to_vec(), Some(vec![0, 1])), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"tree".to_vec(), b"tree2".to_vec()], + b"keyref".to_vec(), + Element::new_reference_with_flags( + SiblingReference(b"key1".to_vec()), + Some(vec![0, 1]), + ), + ), + ]; + + let cost = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + if new_flags[0] == 0 { + new_flags[0] = 1; + let new_flags_epoch = new_flags[1]; + new_flags[1] = old_flags.unwrap()[1]; + new_flags.push(new_flags_epoch); + new_flags.extend(cost.added_bytes.encode_var_vec()); + assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + Ok(true) + } else { + assert_eq!(new_flags[0], 1); + Ok(false) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => { + new_flags.extend(vec![1, 2]); + Ok(true) + } + _ => Ok(false), + }, + |_flags, removed_key_bytes, removed_value_bytes| { + Ok(( + BasicStorageRemoval(removed_key_bytes), + BasicStorageRemoval(removed_value_bytes), + )) + }, + Some(&tx), + grove_version, + ) + .cost_as_result() + .expect("expect no error"); + + // Hash node calls + + // Seek Count + + assert_eq!( + cost, + OperationCost { + seek_count: 8, // todo: verify this + storage_cost: StorageCost { + added_bytes: 430, + replaced_bytes: 78, // todo: verify this + removed_bytes: NoStorageRemoval + }, + storage_loaded_bytes: 152, // todo: verify this + hash_node_calls: 22, // todo: verify this + } + ); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } } diff --git a/grovedb/src/batch/single_deletion_cost_tests.rs b/grovedb/src/batch/single_deletion_cost_tests.rs index fac9682f..e2b3d9a9 100644 --- a/grovedb/src/batch/single_deletion_cost_tests.rs +++ b/grovedb/src/batch/single_deletion_cost_tests.rs @@ -11,7 +11,7 @@ mod tests { use intmap::IntMap; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, }; @@ -72,7 +72,11 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![], + b"key1".to_vec(), + false, + )]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() @@ -137,7 +141,7 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_op(vec![], b"key1".to_vec())]; + let ops = vec![QualifiedGroveDbOp::delete_op(vec![], b"key1".to_vec())]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() @@ -212,7 +216,11 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); - let ops = vec![GroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![], + b"key1".to_vec(), + false, + )]; let batch_cost = db .apply_batch(ops, None, None, grove_version) .cost_as_result() @@ -288,7 +296,7 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); - let ops = vec![GroveDbOp::delete_op(vec![], b"key1".to_vec())]; + let ops = vec![QualifiedGroveDbOp::delete_op(vec![], b"key1".to_vec())]; let batch_cost = db .apply_batch(ops, None, None, grove_version) .cost_as_result() @@ -357,7 +365,11 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![], + b"key1".to_vec(), + false, + )]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() @@ -452,7 +464,11 @@ mod tests { )); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![], + b"key1".to_vec(), + false, + )]; let batch_cost = db .apply_batch_with_element_flags_update( ops, @@ -543,7 +559,7 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_op(vec![], b"key1".to_vec())]; + let ops = vec![QualifiedGroveDbOp::delete_op(vec![], b"key1".to_vec())]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() @@ -623,7 +639,11 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); - let ops = vec![GroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![], + b"key1".to_vec(), + false, + )]; let batch_cost = db .apply_batch(ops, None, None, grove_version) .cost_as_result() @@ -699,7 +719,7 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); - let ops = vec![GroveDbOp::delete_op(vec![], b"key1".to_vec())]; + let ops = vec![QualifiedGroveDbOp::delete_op(vec![], b"key1".to_vec())]; let batch_cost = db .apply_batch(ops, None, None, grove_version) .cost_as_result() diff --git a/grovedb/src/batch/single_insert_cost_tests.rs b/grovedb/src/batch/single_insert_cost_tests.rs index c025fb27..2a269159 100644 --- a/grovedb/src/batch/single_insert_cost_tests.rs +++ b/grovedb/src/batch/single_insert_cost_tests.rs @@ -20,7 +20,8 @@ mod tests { use intmap::IntMap; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, + reference_path::ReferencePathType::SiblingReference, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, }; @@ -42,7 +43,7 @@ mod tests { ) .cost; tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -57,7 +58,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -122,7 +123,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::new_item(b"cat".to_vec()), @@ -200,7 +201,7 @@ mod tests { assert_eq!(cost.storage_cost.added_bytes, 143); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -293,7 +294,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -375,7 +376,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"0".to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -453,7 +454,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::new_item([0u8; 59].to_vec()), @@ -516,7 +517,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::new_item([0u8; 60].to_vec()), @@ -573,6 +574,136 @@ mod tests { ); } + #[test] + fn test_batch_root_one_insert_with_flags_cost_right_below_value_required_cost_of_2() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::new_item_with_flags([0u8; 56].to_vec(), Some(vec![0, 0])), + )]; + let cost_result = db.apply_batch(ops, None, Some(&tx), grove_version); + cost_result.value.expect("expected to execute batch"); + let cost = cost_result.cost; + // Explanation for 243 storage_written_bytes + + // Key -> 37 bytes + // 32 bytes for the key prefix + // 4 bytes for the key + // 1 byte for key_size (required space for 36) + + // Value -> 128 + // 1 for the flag options + // 1 for flags size + // 2 for flag bytes + // 1 for the enum type + // 1 for the value size + // 56 bytes + // 32 for node hash + // 32 for value hash + // 1 for basic merk + // 1 byte for the value_size (required space for 127) + + // Parent Hook -> 40 + // Key Bytes 4 + // Hash Size 32 + // Key Length 1 + // Child Heights 2 + // Sum 1 + // Total 37 + 128 + 40 = 205 + + // Hash node calls + // 2 for the node hash + // 1 for the value hash + // 1 kv_digest_to_kv_hash + + // Seek Count + // 1 to load from root tree + // 1 to insert + // 1 to update root tree + assert_eq!( + cost, + OperationCost { + seek_count: 3, + storage_cost: StorageCost { + added_bytes: 205, + replaced_bytes: 0, + removed_bytes: NoStorageRemoval, + }, + storage_loaded_bytes: 0, + hash_node_calls: 4, + } + ); + } + + #[test] + fn test_batch_root_one_insert_with_flags_cost_right_above_value_required_cost_of_2() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::new_item_with_flags([0u8; 57].to_vec(), Some(vec![0, 0])), + )]; + let cost_result = db.apply_batch(ops, None, Some(&tx), grove_version); + cost_result.value.expect("expected to execute batch"); + let cost = cost_result.cost; + // Explanation for 243 storage_written_bytes + + // Key -> 37 bytes + // 32 bytes for the key prefix + // 4 bytes for the key + // 1 byte for key_size (required space for 36) + + // Value -> 130 + // 1 for the flag option + // 1 for flags size + // 2 for flag bytes + // 1 for the enum type + // 1 for the value size + // 60 bytes + // 32 for node hash + // 32 for value hash + // 1 for basic merk + // 2 byte for the value_size (required space for 128) + + // Parent Hook -> 40 + // Key Bytes 4 + // Hash Size 32 + // Key Length 1 + // Child Heights 2 + // Sum 1 + // Total 37 + 130 + 40 = 207 + + // Hash node calls + // 2 for the node hash + // 1 for the value hash (just under) + // 1 kv_digest_to_kv_hash + + // Seek Count + // 1 to load from root tree + // 1 to insert + // 1 to update root tree + assert_eq!( + cost, + OperationCost { + seek_count: 3, + storage_cost: StorageCost { + added_bytes: 207, + replaced_bytes: 0, + removed_bytes: NoStorageRemoval, + }, + storage_loaded_bytes: 0, + hash_node_calls: 4, + } + ); + } + #[test] fn test_batch_root_one_update_item_bigger_cost_no_flags() { let grove_version = GroveVersion::latest(); @@ -601,7 +732,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_item_with_flags(b"value100".to_vec(), Some(vec![1])), @@ -667,7 +798,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 1])), @@ -728,6 +859,243 @@ mod tests { ); } + #[test] + fn test_batch_root_one_update_item_bigger_cost_with_refresh_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags(b"value1".to_vec(), Some(vec![0, 0])), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"keyref", + Element::new_reference_with_flags(SiblingReference(b"key1".to_vec()), Some(vec![0, 0])), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + // We are adding 2 bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 1])), + ), + QualifiedGroveDbOp::replace_op( + vec![b"tree".to_vec()], + b"keyref".to_vec(), + Element::new_reference_with_flags( + SiblingReference(b"key1".to_vec()), + Some(vec![0, 1]), + ), + ), + ]; + + let cost = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + if new_flags[0] == 0 { + new_flags[0] = 1; + let new_flags_epoch = new_flags[1]; + new_flags[1] = old_flags.unwrap()[1]; + new_flags.push(new_flags_epoch); + new_flags.extend(cost.added_bytes.encode_var_vec()); + assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + Ok(true) + } else { + assert_eq!(new_flags[0], 1); + Ok(false) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => { + new_flags.extend(vec![1, 2]); + Ok(true) + } + _ => Ok(false), + }, + |_flags, removed_key_bytes, removed_value_bytes| { + Ok(( + BasicStorageRemoval(removed_key_bytes), + BasicStorageRemoval(removed_value_bytes), + )) + }, + Some(&tx), + grove_version, + ) + .cost; + + // Hash node calls + + // Seek Count + + assert_eq!( + cost, + OperationCost { + seek_count: 9, // todo: verify this + storage_cost: StorageCost { + added_bytes: 4, + replaced_bytes: 316, // todo: verify this + removed_bytes: NoStorageRemoval + }, + storage_loaded_bytes: 357, // todo: verify this + hash_node_calls: 16, // todo: verify this + } + ); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } + + #[test] + fn test_batch_root_one_update_item_bigger_cost_with_insert_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags(b"value1".to_vec(), Some(vec![0, 0])), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + // We are adding 2 bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 1])), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"tree".to_vec()], + b"keyref".to_vec(), + Element::new_reference_with_flags( + SiblingReference(b"key1".to_vec()), + Some(vec![0, 1]), + ), + ), + ]; + + let cost = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + if new_flags[0] == 0 { + new_flags[0] = 1; + let new_flags_epoch = new_flags[1]; + new_flags[1] = old_flags.unwrap()[1]; + new_flags.push(new_flags_epoch); + new_flags.extend(cost.added_bytes.encode_var_vec()); + assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + Ok(true) + } else { + assert_eq!(new_flags[0], 1); + Ok(false) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => { + new_flags.extend(vec![1, 2]); + Ok(true) + } + _ => Ok(false), + }, + |_flags, removed_key_bytes, removed_value_bytes| { + Ok(( + BasicStorageRemoval(removed_key_bytes), + BasicStorageRemoval(removed_value_bytes), + )) + }, + Some(&tx), + grove_version, + ) + .cost; + + // Hash node calls + + // Seek Count + + assert_eq!( + cost, + OperationCost { + seek_count: 8, // todo: verify this + storage_cost: StorageCost { + added_bytes: 163, + replaced_bytes: 196, // todo: verify this + removed_bytes: NoStorageRemoval + }, + storage_loaded_bytes: 236, // todo: verify this + hash_node_calls: 16, // todo: verify this + } + ); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } + #[test] fn test_batch_root_one_update_item_smaller_cost_no_flags() { let grove_version = GroveVersion::latest(); @@ -756,7 +1124,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_item_with_flags(b"value".to_vec(), Some(vec![1])), @@ -821,7 +1189,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_item_with_flags(b"value".to_vec(), Some(vec![0, 1])), @@ -883,6 +1251,131 @@ mod tests { ); } + #[test] + fn test_batch_root_one_update_item_smaller_cost_with_refresh_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags(b"value1".to_vec(), Some(vec![0, 0])), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"keyref", + Element::new_reference_with_flags(SiblingReference(b"key1".to_vec()), Some(vec![0, 0])), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + // We are adding 2 bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value".to_vec(), Some(vec![0, 1])), + ), + QualifiedGroveDbOp::replace_op( + vec![b"tree".to_vec()], + b"keyref".to_vec(), + Element::new_reference_with_flags( + SiblingReference(b"key1".to_vec()), + Some(vec![0, 1]), + ), + ), + ]; + + let cost = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + if new_flags[0] == 0 { + new_flags[0] = 1; + let new_flags_epoch = new_flags[1]; + new_flags[1] = old_flags.unwrap()[1]; + new_flags.push(new_flags_epoch); + new_flags.extend(cost.added_bytes.encode_var_vec()); + assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + Ok(true) + } else { + assert_eq!(new_flags[0], 1); + Ok(false) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => Ok(true), + _ => Ok(false), + }, + |_flags, _removed_key_bytes, removed_value_bytes| { + let mut removed_bytes = StorageRemovalPerEpochByIdentifier::default(); + // we are removing 1 byte from epoch 0 for an identity + let mut removed_bytes_for_identity = IntMap::new(); + removed_bytes_for_identity.insert(0, removed_value_bytes); + removed_bytes.insert(Identifier::default(), removed_bytes_for_identity); + Ok((NoStorageRemoval, SectionedStorageRemoval(removed_bytes))) + }, + Some(&tx), + grove_version, + ) + .cost; + + let mut removed_bytes = StorageRemovalPerEpochByIdentifier::default(); + // we are removing 1 byte from epoch 0 for an identity + let mut removed_bytes_for_identity = IntMap::new(); + removed_bytes_for_identity.insert(0, 1); + removed_bytes.insert(Identifier::default(), removed_bytes_for_identity); + + assert_eq!( + cost, + OperationCost { + seek_count: 9, // todo: verify this + storage_cost: StorageCost { + added_bytes: 0, + replaced_bytes: 315, // todo: verify this + removed_bytes: SectionedStorageRemoval(removed_bytes) + }, + storage_loaded_bytes: 357, // todo: verify this + hash_node_calls: 16, // todo: verify this + } + ); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } + #[test] fn test_batch_root_one_update_tree_bigger_flags_cost() { let grove_version = GroveVersion::latest(); @@ -911,7 +1404,7 @@ mod tests { .expect("expected to insert item"); // We are adding 1 byte to the flags - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_tree_with_flags(None, Some(vec![0, 1, 1])), @@ -965,4 +1458,199 @@ mod tests { } ); } + + #[test] + fn test_batch_root_one_update_cost_right_above_value_required_cost_of_2_with_refresh_reference() + { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::new_item_with_flags([0u8; 56].to_vec(), Some(vec![0, 0])), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"keyref".to_vec(), + Element::new_reference(SiblingReference(b"key1".to_vec())), + ), + ]; + + let cost_result = db.apply_batch(ops, None, Some(&tx), grove_version); + cost_result.value.expect("expected to execute batch"); + // We are adding 2 bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::new_item_with_flags([0u8; 57].to_vec(), Some(vec![0, 1])), + ), + QualifiedGroveDbOp::replace_op( + vec![], + b"keyref".to_vec(), + Element::new_reference(SiblingReference(b"key1".to_vec())), + ), + ]; + + let cost = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + if new_flags[0] == 0 { + new_flags[0] = 1; + let new_flags_epoch = new_flags[1]; + new_flags[1] = old_flags.unwrap()[1]; + new_flags.push(new_flags_epoch); + new_flags.extend(cost.added_bytes.encode_var_vec()); + assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + Ok(true) + } else { + assert_eq!(new_flags[0], 1); + Ok(false) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => { + new_flags.extend(vec![1, 2]); + Ok(true) + } + _ => Ok(false), + }, + |_flags, removed_key_bytes, removed_value_bytes| { + Ok(( + BasicStorageRemoval(removed_key_bytes), + BasicStorageRemoval(removed_value_bytes), + )) + }, + Some(&tx), + grove_version, + ) + .cost; + + assert_eq!( + cost, + OperationCost { + seek_count: 7, // todo: verify this + storage_cost: StorageCost { + added_bytes: 4, + replaced_bytes: 285, // todo: verify this + removed_bytes: NoStorageRemoval + }, + storage_loaded_bytes: 380, // todo: verify this + hash_node_calls: 12, // todo: verify this + } + ); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } + + #[test] + fn test_batch_root_one_update_cost_right_above_value_required_cost_of_2_with_insert_reference() + { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::new_item_with_flags([0u8; 56].to_vec(), Some(vec![0, 0])), + )]; + + let cost_result = db.apply_batch(ops, None, Some(&tx), grove_version); + cost_result.value.expect("expected to execute batch"); + // We are adding 2 bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::new_item_with_flags([0u8; 57].to_vec(), Some(vec![0, 1])), + ), + QualifiedGroveDbOp::insert_only_op( + vec![], + b"keyref".to_vec(), + Element::new_reference(SiblingReference(b"key1".to_vec())), + ), + ]; + + let cost = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + if new_flags[0] == 0 { + new_flags[0] = 1; + let new_flags_epoch = new_flags[1]; + new_flags[1] = old_flags.unwrap()[1]; + new_flags.push(new_flags_epoch); + new_flags.extend(cost.added_bytes.encode_var_vec()); + assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + Ok(true) + } else { + assert_eq!(new_flags[0], 1); + Ok(false) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => { + new_flags.extend(vec![1, 2]); + Ok(true) + } + _ => Ok(false), + }, + |_flags, removed_key_bytes, removed_value_bytes| { + Ok(( + BasicStorageRemoval(removed_key_bytes), + BasicStorageRemoval(removed_value_bytes), + )) + }, + Some(&tx), + grove_version, + ) + .cost; + + assert_eq!( + cost, + OperationCost { + seek_count: 5, // todo: verify this + storage_cost: StorageCost { + added_bytes: 160, + replaced_bytes: 168, // todo: verify this + removed_bytes: NoStorageRemoval + }, + storage_loaded_bytes: 133, // todo: verify this + hash_node_calls: 12, // todo: verify this + } + ); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } } diff --git a/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs b/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs index bf5637d0..c0be9a08 100644 --- a/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs +++ b/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs @@ -5,7 +5,7 @@ mod tests { use grovedb_version::version::GroveVersion; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, }; @@ -43,7 +43,11 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![], + b"key1".to_vec(), + false, + )]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() @@ -101,7 +105,7 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_op( + let ops = vec![QualifiedGroveDbOp::delete_op( vec![b"sum_tree".to_vec()], b"key1".to_vec(), )]; @@ -146,7 +150,11 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![], + b"key1".to_vec(), + false, + )]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() diff --git a/grovedb/src/batch/single_sum_item_insert_cost_tests.rs b/grovedb/src/batch/single_sum_item_insert_cost_tests.rs index 0ba3da44..d58e7327 100644 --- a/grovedb/src/batch/single_sum_item_insert_cost_tests.rs +++ b/grovedb/src/batch/single_sum_item_insert_cost_tests.rs @@ -9,7 +9,7 @@ mod tests { use grovedb_version::version::GroveVersion; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, }; @@ -42,7 +42,7 @@ mod tests { ) .cost; tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"sum_tree".to_vec()], b"key1".to_vec(), Element::new_sum_item(150), @@ -57,7 +57,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_sum_tree(), @@ -134,7 +134,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_sum_tree(), @@ -220,7 +220,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_sum_tree(), @@ -306,7 +306,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"0".to_vec()], b"key1".to_vec(), Element::empty_sum_tree(), @@ -396,7 +396,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"0".to_vec()], b"key1".to_vec(), Element::empty_sum_tree(), @@ -487,7 +487,7 @@ mod tests { .unwrap() .expect("expected to insert sum tree"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"sum_tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(15, Some([0; 42].to_vec())), @@ -562,7 +562,7 @@ mod tests { .unwrap() .expect("expected to insert sum tree"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"sum_tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(15, Some([0; 43].to_vec())), @@ -648,7 +648,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(100000, None), @@ -714,7 +714,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(100000, Some(vec![1])), @@ -780,7 +780,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(5, None), @@ -846,7 +846,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(5, Some(vec![1])), diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 96c2d0de..feef0721 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -212,6 +212,7 @@ use tokio::net::ToSocketAddrs; use crate::element::helpers::raw_decode; #[cfg(any(feature = "full", feature = "verify"))] pub use crate::error::Error; +use crate::operations::proof::util::hex_to_ascii; #[cfg(feature = "full")] use crate::util::{root_merk_optional_tx, storage_context_optional_tx}; #[cfg(feature = "full")] @@ -899,11 +900,13 @@ impl GroveDb { /// Method to visualize hash mismatch after verification pub fn visualize_verify_grovedb( &self, + transaction: TransactionArg, verify_references: bool, + allow_cache: bool, grove_version: &GroveVersion, ) -> Result, Error> { Ok(self - .verify_grovedb(None, verify_references, grove_version)? + .verify_grovedb(transaction, verify_references, allow_cache, grove_version)? .iter() .map(|(path, (root_hash, expected, actual))| { ( @@ -927,6 +930,7 @@ impl GroveDb { &self, transaction: TransactionArg, verify_references: bool, + allow_cache: bool, grove_version: &GroveVersion, ) -> Result>, (CryptoHash, CryptoHash, CryptoHash)>, Error> { if let Some(transaction) = transaction { @@ -944,6 +948,7 @@ impl GroveDb { None, transaction, verify_references, + allow_cache, grove_version, ) } else { @@ -955,6 +960,7 @@ impl GroveDb { &SubtreePath::empty(), None, verify_references, + allow_cache, grove_version, ) } @@ -968,12 +974,12 @@ impl GroveDb { path: &SubtreePath, batch: Option<&'db StorageBatch>, verify_references: bool, + allow_cache: bool, grove_version: &GroveVersion, ) -> Result>, (CryptoHash, CryptoHash, CryptoHash)>, Error> { let mut all_query = Query::new(); all_query.insert_all(); - let _in_sum_tree = merk.is_sum_tree; let mut issues = HashMap::new(); let mut element_iterator = KVIterator::new(merk.storage.raw_iter(), &all_query).unwrap(); @@ -984,15 +990,17 @@ impl GroveDb { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, - true, + allow_cache, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) .unwrap() .map_err(MerkError)? - .ok_or(Error::CorruptedData( - "expected merk to contain value at key".to_string(), - ))?; + .ok_or(Error::CorruptedData(format!( + "expected merk to contain value at key {} for {}", + hex_to_ascii(&key), + element.type_str() + )))?; let new_path = path.derive_owned_with_child(key); let new_path_ref = SubtreePath::from(&new_path); @@ -1019,6 +1027,7 @@ impl GroveDb { &new_path_ref, batch, verify_references, + true, grove_version, )?); } @@ -1026,15 +1035,17 @@ impl GroveDb { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, - true, + allow_cache, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) .unwrap() .map_err(MerkError)? - .ok_or(Error::CorruptedData( - "expected merk to contain value at key".to_string(), - ))?; + .ok_or(Error::CorruptedData(format!( + "expected merk to contain value at key {} for {}", + hex_to_ascii(&key), + element.type_str() + )))?; let actual_value_hash = value_hash(&kv_value).unwrap(); if actual_value_hash != element_value_hash { issues.insert( @@ -1053,18 +1064,19 @@ impl GroveDb { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, - true, + allow_cache, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) .unwrap() .map_err(MerkError)? - .ok_or(Error::CorruptedData( - "expected merk to contain value at key".to_string(), - ))?; + .ok_or(Error::CorruptedData(format!( + "expected merk to contain value at key {} for reference", + hex_to_ascii(&key) + )))?; let referenced_value_hash = { - let mut full_path = path_from_reference_path_type( + let full_path = path_from_reference_path_type( reference_path.clone(), &path.to_vec(), Some(&key), @@ -1072,7 +1084,7 @@ impl GroveDb { let item = self .follow_reference( (full_path.as_slice()).into(), - true, + allow_cache, None, grove_version, ) @@ -1106,12 +1118,12 @@ impl GroveDb { batch: Option<&'db StorageBatch>, transaction: &Transaction, verify_references: bool, + allow_cache: bool, grove_version: &GroveVersion, ) -> Result>, (CryptoHash, CryptoHash, CryptoHash)>, Error> { let mut all_query = Query::new(); all_query.insert_all(); - let _in_sum_tree = merk.is_sum_tree; let mut issues = HashMap::new(); let mut element_iterator = KVIterator::new(merk.storage.raw_iter(), &all_query).unwrap(); @@ -1122,15 +1134,17 @@ impl GroveDb { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, - true, + allow_cache, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) .unwrap() .map_err(MerkError)? - .ok_or(Error::CorruptedData( - "expected merk to contain value at key".to_string(), - ))?; + .ok_or(Error::CorruptedData(format!( + "expected merk to contain value at key {} for {}", + hex_to_ascii(&key), + element.type_str() + )))?; let new_path = path.derive_owned_with_child(key); let new_path_ref = SubtreePath::from(&new_path); @@ -1159,6 +1173,7 @@ impl GroveDb { batch, transaction, verify_references, + true, grove_version, )?); } @@ -1166,15 +1181,17 @@ impl GroveDb { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, - true, + allow_cache, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) .unwrap() .map_err(MerkError)? - .ok_or(Error::CorruptedData( - "expected merk to contain value at key".to_string(), - ))?; + .ok_or(Error::CorruptedData(format!( + "expected merk to contain value at key {} for {}", + hex_to_ascii(&key), + element.type_str() + )))?; let actual_value_hash = value_hash(&kv_value).unwrap(); if actual_value_hash != element_value_hash { issues.insert( @@ -1193,18 +1210,19 @@ impl GroveDb { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, - true, + allow_cache, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) .unwrap() .map_err(MerkError)? - .ok_or(Error::CorruptedData( - "expected merk to contain value at key".to_string(), - ))?; + .ok_or(Error::CorruptedData(format!( + "expected merk to contain value at key {} for reference", + hex_to_ascii(&key) + )))?; let referenced_value_hash = { - let mut full_path = path_from_reference_path_type( + let full_path = path_from_reference_path_type( reference_path.clone(), &path.to_vec(), Some(&key), @@ -1212,7 +1230,7 @@ impl GroveDb { let item = self .follow_reference( (full_path.as_slice()).into(), - true, + allow_cache, Some(transaction), grove_version, ) diff --git a/grovedb/src/operations/delete/average_case.rs b/grovedb/src/operations/delete/average_case.rs index 3ed1abd1..986f2b90 100644 --- a/grovedb/src/operations/delete/average_case.rs +++ b/grovedb/src/operations/delete/average_case.rs @@ -17,7 +17,7 @@ use grovedb_version::{ use intmap::IntMap; use crate::{ - batch::{key_info::KeyInfo, GroveDbOp, KeyInfoPath}, + batch::{key_info::KeyInfo, KeyInfoPath, QualifiedGroveDbOp}, Error, GroveDb, }; @@ -34,7 +34,7 @@ impl GroveDb { validate: bool, estimated_layer_info: IntMap, grove_version: &GroveVersion, - ) -> CostResult, Error> { + ) -> CostResult, Error> { check_grovedb_v0_with_cost!( "average_case_delete_operations_for_delete_up_tree_while_empty", grove_version @@ -145,7 +145,7 @@ impl GroveDb { except_keys_count: u16, estimated_key_element_size: EstimatedKeyAndElementSize, grove_version: &GroveVersion, - ) -> CostResult { + ) -> CostResult { check_grovedb_v0_with_cost!( "average_case_delete_operation_for_delete", grove_version @@ -188,6 +188,10 @@ impl GroveDb { estimated_key_element_size.0 + HASH_LENGTH_U32, ); - Ok(GroveDbOp::delete_estimated_op(path.clone(), key.clone())).wrap_with_cost(cost) + Ok(QualifiedGroveDbOp::delete_estimated_op( + path.clone(), + key.clone(), + )) + .wrap_with_cost(cost) } } diff --git a/grovedb/src/operations/delete/delete_up_tree.rs b/grovedb/src/operations/delete/delete_up_tree.rs index dd331b69..7ecfce83 100644 --- a/grovedb/src/operations/delete/delete_up_tree.rs +++ b/grovedb/src/operations/delete/delete_up_tree.rs @@ -11,7 +11,7 @@ use grovedb_version::{ }; use crate::{ - batch::GroveDbOp, operations::delete::DeleteOptions, ElementFlags, Error, GroveDb, + batch::QualifiedGroveDbOp, operations::delete::DeleteOptions, ElementFlags, Error, GroveDb, TransactionArg, }; @@ -122,7 +122,7 @@ impl GroveDb { .delete_up_tree_while_empty_with_sectional_storage ); let mut cost = OperationCost::default(); - let mut batch_operations: Vec = Vec::new(); + let mut batch_operations: Vec = Vec::new(); let maybe_ops = cost_return_on_error!( &mut cost, @@ -170,10 +170,10 @@ impl GroveDb { key: &[u8], options: &DeleteUpTreeOptions, is_known_to_be_subtree_with_sum: Option<(bool, bool)>, - mut current_batch_operations: Vec, + mut current_batch_operations: Vec, transaction: TransactionArg, grove_version: &GroveVersion, - ) -> CostResult, Error> { + ) -> CostResult, Error> { check_grovedb_v0_with_cost!( "delete", grove_version @@ -202,10 +202,10 @@ impl GroveDb { key: &[u8], options: &DeleteUpTreeOptions, is_known_to_be_subtree_with_sum: Option<(bool, bool)>, - current_batch_operations: &mut Vec, + current_batch_operations: &mut Vec, transaction: TransactionArg, grove_version: &GroveVersion, - ) -> CostResult>, Error> { + ) -> CostResult>, Error> { check_grovedb_v0_with_cost!( "delete", grove_version diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index 31d96b85..9244c60b 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -33,7 +33,7 @@ use grovedb_version::{ #[cfg(feature = "full")] use crate::{ - batch::{GroveDbOp, Op}, + batch::{GroveOp, QualifiedGroveDbOp}, util::storage_context_with_parent_optional_tx, Element, ElementFlags, Error, GroveDb, Transaction, TransactionArg, }; @@ -512,10 +512,10 @@ impl GroveDb { key: &[u8], options: &DeleteOptions, is_known_to_be_subtree_with_sum: Option<(bool, bool)>, - current_batch_operations: &[GroveDbOp], + current_batch_operations: &[QualifiedGroveDbOp], transaction: TransactionArg, grove_version: &GroveVersion, - ) -> CostResult, Error> { + ) -> CostResult, Error> { check_grovedb_v0_with_cost!( "delete_operation_for_delete_internal", grove_version @@ -565,7 +565,7 @@ impl GroveDb { let batch_deleted_keys = current_batch_operations .iter() .filter_map(|op| match op.op { - Op::Delete | Op::DeleteTree | Op::DeleteSumTree => { + GroveOp::Delete | GroveOp::DeleteTree | GroveOp::DeleteSumTree => { // todo: to_path clones (best to figure out how to compare without // cloning) if op.path.to_path() == subtree_merk_path_vec { @@ -595,7 +595,7 @@ impl GroveDb { // If there is any current batch operation that is inserting something in this // tree then it is not empty either is_empty &= !current_batch_operations.iter().any(|op| match op.op { - Op::Delete | Op::DeleteTree | Op::DeleteSumTree => false, + GroveOp::Delete | GroveOp::DeleteTree | GroveOp::DeleteSumTree => false, // todo: fix for to_path (it clones) _ => op.path.to_path() == subtree_merk_path_vec, }); @@ -610,7 +610,7 @@ impl GroveDb { Ok(None) } } else if is_empty { - Ok(Some(GroveDbOp::delete_tree_op( + Ok(Some(QualifiedGroveDbOp::delete_tree_op( path.to_vec(), key.to_vec(), is_subtree_with_sum, @@ -622,7 +622,11 @@ impl GroveDb { }; result.wrap_with_cost(cost) } else { - Ok(Some(GroveDbOp::delete_op(path.to_vec(), key.to_vec()))).wrap_with_cost(cost) + Ok(Some(QualifiedGroveDbOp::delete_op( + path.to_vec(), + key.to_vec(), + ))) + .wrap_with_cost(cost) } } } diff --git a/grovedb/src/operations/delete/worst_case.rs b/grovedb/src/operations/delete/worst_case.rs index b2a50bb2..8533cde5 100644 --- a/grovedb/src/operations/delete/worst_case.rs +++ b/grovedb/src/operations/delete/worst_case.rs @@ -13,7 +13,7 @@ use grovedb_version::{ use intmap::IntMap; use crate::{ - batch::{key_info::KeyInfo, GroveDbOp, KeyInfoPath}, + batch::{key_info::KeyInfo, KeyInfoPath, QualifiedGroveDbOp}, element::SUM_TREE_COST_SIZE, Error, GroveDb, }; @@ -29,7 +29,7 @@ impl GroveDb { intermediate_tree_info: IntMap<(bool, u32)>, max_element_size: u32, grove_version: &GroveVersion, - ) -> CostResult, Error> { + ) -> CostResult, Error> { check_grovedb_v0_with_cost!( "delete", grove_version @@ -127,7 +127,7 @@ impl GroveDb { except_keys_count: u16, max_element_size: u32, grove_version: &GroveVersion, - ) -> CostResult { + ) -> CostResult { check_grovedb_v0_with_cost!( "worst_case_delete_operation_for_delete", grove_version @@ -165,6 +165,10 @@ impl GroveDb { // in the worst case this is a tree add_worst_case_cost_for_is_empty_tree_except(&mut cost, except_keys_count); - Ok(GroveDbOp::delete_estimated_op(path.clone(), key.clone())).wrap_with_cost(cost) + Ok(QualifiedGroveDbOp::delete_estimated_op( + path.clone(), + key.clone(), + )) + .wrap_with_cost(cost) } } diff --git a/grovedb/src/operations/insert/mod.rs b/grovedb/src/operations/insert/mod.rs index 87f0f97b..af7629ae 100644 --- a/grovedb/src/operations/insert/mod.rs +++ b/grovedb/src/operations/insert/mod.rs @@ -7,7 +7,6 @@ use std::{collections::HashMap, option::Option::None}; use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::tree::value_hash; #[cfg(feature = "full")] use grovedb_merk::{tree::NULL_HASH, Merk, MerkOptions}; use grovedb_path::SubtreePath; diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index f6b7b6d4..41263669 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -758,7 +758,7 @@ pub fn make_deep_tree_with_sum_trees(grove_version: &GroveVersion) -> TempGroveD } mod tests { - use batch::GroveDbOp; + use batch::QualifiedGroveDbOp; use grovedb_merk::proofs::query::SubqueryBranch; use super::*; @@ -3990,7 +3990,12 @@ mod tests { )); // `verify_grovedb` must identify issues - assert!(db.verify_grovedb(None, true, grove_version).unwrap().len() > 0); + assert!( + db.verify_grovedb(None, true, false, grove_version) + .unwrap() + .len() + > 0 + ); } #[test] @@ -4043,7 +4048,7 @@ mod tests { .unwrap(); assert!(db - .verify_grovedb(None, true, grove_version) + .verify_grovedb(None, true, false, grove_version) .unwrap() .is_empty()); @@ -4060,7 +4065,7 @@ mod tests { .unwrap(); assert!(!db - .verify_grovedb(None, true, grove_version) + .verify_grovedb(None, true, false, grove_version) .unwrap() .is_empty()); } @@ -4071,22 +4076,22 @@ mod tests { let db = make_test_grovedb(grove_version); let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"value".to_vec(), Element::new_item(b"hello".to_vec()), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"refc".to_vec(), Element::new_reference(ReferencePathType::SiblingReference(b"value".to_vec())), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"refb".to_vec(), Element::new_reference(ReferencePathType::SiblingReference(b"refc".to_vec())), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"refa".to_vec(), Element::new_reference(ReferencePathType::SiblingReference(b"refb".to_vec())), @@ -4098,7 +4103,7 @@ mod tests { .unwrap(); assert!(db - .verify_grovedb(None, true, grove_version) + .verify_grovedb(None, true, false, grove_version) .unwrap() .is_empty()); @@ -4115,7 +4120,7 @@ mod tests { .unwrap(); assert!(!db - .verify_grovedb(None, true, grove_version) + .verify_grovedb(None, true, false, grove_version) .unwrap() .is_empty()); } diff --git a/grovedb/src/tests/query_tests.rs b/grovedb/src/tests/query_tests.rs index 48c358c6..4ca273fe 100644 --- a/grovedb/src/tests/query_tests.rs +++ b/grovedb/src/tests/query_tests.rs @@ -7,7 +7,7 @@ mod tests { use tempfile::TempDir; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, query_result_type::{ PathKeyOptionalElementTrio, QueryResultElement::PathKeyElementTrioResultItem, QueryResultElements, QueryResultType, @@ -1595,33 +1595,37 @@ mod tests { 77, 252, 86, 99, 107, 197, 226, 188, 54, 239, 64, 17, 37, ]; - let batch = vec![GroveDbOp::insert_op(vec![], vec![1], Element::empty_tree())]; + let batch = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![], + vec![1], + Element::empty_tree(), + )]; db.apply_batch(batch, None, None, grove_version) .unwrap() .expect("should apply batch"); let batch = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![vec![1]], tree_name_slice.to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![vec![1], tree_name_slice.to_vec()], b"\0".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![vec![1], tree_name_slice.to_vec()], vec![1], Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![vec![1], tree_name_slice.to_vec(), vec![1]], b"person".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), @@ -1631,7 +1635,7 @@ mod tests { b"\0".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), @@ -1647,7 +1651,7 @@ mod tests { .expect("should apply batch"); let batch = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), @@ -1658,7 +1662,7 @@ mod tests { b"person_id_1".to_vec(), Element::new_item(vec![50]), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), @@ -1669,7 +1673,7 @@ mod tests { b"cammi".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), @@ -1681,7 +1685,7 @@ mod tests { b"\0".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), diff --git a/grovedb/src/tests/sum_tree_tests.rs b/grovedb/src/tests/sum_tree_tests.rs index 92df7d73..b255f653 100644 --- a/grovedb/src/tests/sum_tree_tests.rs +++ b/grovedb/src/tests/sum_tree_tests.rs @@ -9,7 +9,7 @@ use grovedb_storage::StorageBatch; use grovedb_version::version::GroveVersion; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, reference_path::ReferencePathType, tests::{make_test_grovedb, TEST_LEAF}, Element, Error, GroveDb, PathQuery, @@ -779,17 +779,17 @@ fn test_sum_tree_with_batches() { let grove_version = GroveVersion::latest(); let db = make_test_grovedb(grove_version); let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_sum_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"a".to_vec(), Element::new_item(vec![214]), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"b".to_vec(), Element::new_sum_item(10), @@ -835,7 +835,7 @@ fn test_sum_tree_with_batches() { )); // Create new batch to use existing tree - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"c".to_vec(), Element::new_sum_item(10), @@ -871,42 +871,42 @@ fn test_sum_tree_with_batches() { // Add a new sum tree with its own sum items, should affect sum of original // tree let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"d".to_vec(), Element::empty_sum_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"d".to_vec()], b"first".to_vec(), Element::new_sum_item(4), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"d".to_vec()], b"second".to_vec(), Element::new_item(vec![4]), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"e".to_vec(), Element::empty_sum_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], b"first".to_vec(), Element::new_sum_item(12), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], b"second".to_vec(), Element::new_item(vec![4]), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], b"third".to_vec(), Element::empty_sum_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ TEST_LEAF.to_vec(), b"key1".to_vec(), @@ -916,7 +916,7 @@ fn test_sum_tree_with_batches() { b"a".to_vec(), Element::new_sum_item(5), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ TEST_LEAF.to_vec(), b"key1".to_vec(), diff --git a/merk/benches/merk.rs b/merk/benches/merk.rs index e2d55219..62bc6ebb 100644 --- a/merk/benches/merk.rs +++ b/merk/benches/merk.rs @@ -38,10 +38,12 @@ use grovedb_merk::{ }; use grovedb_path::SubtreePath; use grovedb_storage::{rocksdb_storage::test_utils::TempStorage, Storage}; +use grovedb_version::version::GroveVersion; use rand::prelude::*; /// 1 million gets in 2k batches pub fn get(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; let num_batches = initial_size / batch_size; @@ -95,6 +97,7 @@ pub fn get(c: &mut Criterion) { /// 1 million sequential inserts in 2k batches pub fn insert_1m_2k_seq(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -136,6 +139,7 @@ pub fn insert_1m_2k_seq(c: &mut Criterion) { /// 1 million random inserts in 2k batches pub fn insert_1m_2k_rand(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -177,6 +181,7 @@ pub fn insert_1m_2k_rand(c: &mut Criterion) { /// 1 million sequential updates in 2k batches pub fn update_1m_2k_seq(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -237,6 +242,7 @@ pub fn update_1m_2k_seq(c: &mut Criterion) { /// 1 million random updates in 2k batches pub fn update_1m_2k_rand(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -297,6 +303,7 @@ pub fn update_1m_2k_rand(c: &mut Criterion) { /// 1 million random deletes in 2k batches pub fn delete_1m_2k_rand(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -382,6 +389,7 @@ pub fn delete_1m_2k_rand(c: &mut Criterion) { /// 1 million random proofs in 2k batches pub fn prove_1m_2k_rand(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -434,6 +442,7 @@ pub fn prove_1m_2k_rand(c: &mut Criterion) { /// Build 1 million trunk chunks in 2k batches, random pub fn build_trunk_chunk_1m_2k_rand(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -477,6 +486,7 @@ pub fn build_trunk_chunk_1m_2k_rand(c: &mut Criterion) { /// Chunk producer random 1 million pub fn chunkproducer_rand_1m_1_rand(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -518,6 +528,7 @@ pub fn chunkproducer_rand_1m_1_rand(c: &mut Criterion) { /// Chunk iter 1 million pub fn chunk_iter_1m_1(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -565,6 +576,7 @@ pub fn chunk_iter_1m_1(c: &mut Criterion) { /// Restore merk of size 500 pub fn restore_500_1(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let merk_size = 500; let mut merk = TempMerk::new(grove_version); diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index 6b2710b6..dc0cbaf2 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -158,16 +158,7 @@ impl TreeNode { self.inner.kv.feature_type.is_sum_feature() } - /// Compare current value byte cost with old cost and return - /// current value byte cost with updated `KeyValueStorageCost` - pub fn kv_with_parent_hook_size_and_storage_cost_from_old_cost( - &self, - current_value_byte_cost: u32, - old_cost: u32, - ) -> Result<(u32, KeyValueStorageCost), Error> { - let key_storage_cost = StorageCost { - ..Default::default() - }; + pub fn storage_cost_for_update(current_value_byte_cost: u32, old_cost: u32) -> StorageCost { let mut value_storage_cost = StorageCost { ..Default::default() }; @@ -190,6 +181,20 @@ impl TreeNode { value_storage_cost.added_bytes += current_value_byte_cost - old_cost; } } + value_storage_cost + } + + /// Compare current value byte cost with old cost and return + /// current value byte cost with updated `KeyValueStorageCost` + pub fn kv_with_parent_hook_size_and_storage_cost_from_old_cost( + &self, + current_value_byte_cost: u32, + old_cost: u32, + ) -> Result<(u32, KeyValueStorageCost), Error> { + let key_storage_cost = StorageCost { + ..Default::default() + }; + let value_storage_cost = Self::storage_cost_for_update(current_value_byte_cost, old_cost); let key_value_storage_cost = KeyValueStorageCost { key_storage_cost, // the key storage cost is added later