diff --git a/crates/subspace-archiving/src/archiver.rs b/crates/subspace-archiving/src/archiver.rs index a1967e9925..dccec16038 100644 --- a/crates/subspace-archiving/src/archiver.rs +++ b/crates/subspace-archiving/src/archiver.rs @@ -193,16 +193,16 @@ impl NewArchivedSegment { .segment_piece_indexes_source_first(); // Iterate through the object mapping vector for each piece - object_mapping - .into_iter() - .zip(piece_indexes) - .flat_map(|(piece_mappings, piece_index)| { + object_mapping.into_iter().zip(piece_indexes).flat_map( + move |(piece_mappings, piece_index)| { // And then through each individual object mapping in the piece + let piece_mappings = piece_mappings.objects().to_vec(); + piece_mappings - .objects .into_iter() .map(move |piece_object| GlobalObject::new(piece_index, &piece_object)) - }) + }, + ) } } @@ -316,11 +316,10 @@ impl Archiver { // Take part of the encoded block that wasn't archived yet and push to the // buffer and block continuation object_mapping - .objects + .objects_mut() .retain_mut(|block_object: &mut BlockObject| { - let current_offset = block_object.offset(); - if current_offset >= archived_block_bytes { - block_object.set_offset(current_offset - archived_block_bytes); + if block_object.offset >= archived_block_bytes { + block_object.offset -= archived_block_bytes; true } else { false @@ -509,13 +508,12 @@ impl Archiver { bytes.truncate(split_point); - let continuation_object_mapping = BlockObjectMapping { + let continuation_object_mapping = BlockObjectMapping::V0 { objects: object_mapping - .objects + .objects_mut() .extract_if(|block_object: &mut BlockObject| { - let current_offset = block_object.offset(); - if current_offset >= split_point as u32 { - block_object.set_offset(current_offset - split_point as u32); + if block_object.offset >= split_point as u32 { + block_object.offset -= split_point as u32; true } else { false @@ -553,13 +551,12 @@ impl Archiver { bytes.truncate(split_point); - let continuation_object_mapping = BlockObjectMapping { + let continuation_object_mapping = BlockObjectMapping::V0 { objects: object_mapping - .objects + .objects_mut() .extract_if(|block_object: &mut BlockObject| { - let current_offset = block_object.offset(); - if current_offset >= split_point as u32 { - block_object.set_offset(current_offset - split_point as u32); + if block_object.offset >= split_point as u32 { + block_object.offset -= split_point as u32; true } else { false @@ -640,12 +637,12 @@ impl Archiver { bytes, object_mapping, } => { - for block_object in &object_mapping.objects { + for block_object in object_mapping.objects() { // `+1` corresponds to `SegmentItem::X {}` enum variant encoding let offset_in_segment = base_offset_in_segment + 1 + Compact::compact_len(&(bytes.len() as u32)) - + block_object.offset() as usize; + + block_object.offset as usize; let raw_piece_offset = (offset_in_segment % RawRecord::SIZE).try_into().expect( "Offset within piece should always fit in 32-bit integer; qed", @@ -653,8 +650,8 @@ impl Archiver { if let Some(piece_object_mapping) = corrected_object_mapping .get_mut(offset_in_segment / RawRecord::SIZE) { - piece_object_mapping.objects.push(PieceObject::V0 { - hash: block_object.hash(), + piece_object_mapping.objects_mut().push(PieceObject { + hash: block_object.hash, offset: raw_piece_offset, }); } diff --git a/crates/subspace-archiving/tests/integration/archiver.rs b/crates/subspace-archiving/tests/integration/archiver.rs index f6b6423bc5..0c49450c6e 100644 --- a/crates/subspace-archiving/tests/integration/archiver.rs +++ b/crates/subspace-archiving/tests/integration/archiver.rs @@ -52,8 +52,8 @@ fn compare_block_objects_to_piece_objects<'a>( block_objects.zip(piece_objects).for_each( |((block, block_object_mapping), (piece, piece_object_mapping))| { assert_eq!( - extract_data_from_source_record(piece.record(), piece_object_mapping.offset()), - extract_data(block, block_object_mapping.offset()) + extract_data_from_source_record(piece.record(), piece_object_mapping.offset), + extract_data(block, block_object_mapping.offset) ); }, ); @@ -81,13 +81,13 @@ fn archiver() { .as_mut() .write_all(&Compact(128_u64).encode()) .unwrap(); - let object_mapping = BlockObjectMapping { + let object_mapping = BlockObjectMapping::V0 { objects: vec![ - BlockObject::V0 { + BlockObject { hash: Blake3Hash::default(), offset: 0u32, }, - BlockObject::V0 { + BlockObject { hash: Blake3Hash::default(), offset: RecordedHistorySegment::SIZE as u32 / 3, }, @@ -117,17 +117,17 @@ fn archiver() { .as_mut() .write_all(&Compact(100_u64).encode()) .unwrap(); - let object_mapping = BlockObjectMapping { + let object_mapping = BlockObjectMapping::V0 { objects: vec![ - BlockObject::V0 { + BlockObject { hash: Blake3Hash::default(), offset: RecordedHistorySegment::SIZE as u32 / 6, }, - BlockObject::V0 { + BlockObject { hash: Blake3Hash::default(), offset: RecordedHistorySegment::SIZE as u32 / 5, }, - BlockObject::V0 { + BlockObject { hash: Blake3Hash::default(), offset: RecordedHistorySegment::SIZE as u32 / 3 * 2 - 200, }, @@ -170,19 +170,19 @@ fn archiver() { first_archived_segment .object_mapping .iter() - .filter(|object_mapping| !object_mapping.objects.is_empty()) + .filter(|object_mapping| !object_mapping.objects().is_empty()) .count(), 4 ); { let block_objects = iter::repeat(block_0.as_ref()) - .zip(&block_0_object_mapping.objects) - .chain(iter::repeat(block_1.as_ref()).zip(block_1_object_mapping.objects.iter())); + .zip(block_0_object_mapping.objects()) + .chain(iter::repeat(block_1.as_ref()).zip(block_1_object_mapping.objects())); let piece_objects = first_archived_segment .pieces .source_pieces() .zip(&first_archived_segment.object_mapping) - .flat_map(|(piece, object_mapping)| iter::repeat(piece).zip(&object_mapping.objects)); + .flat_map(|(piece, object_mapping)| iter::repeat(piece).zip(object_mapping.objects())); compare_block_objects_to_piece_objects(block_objects, piece_objects); } @@ -249,7 +249,7 @@ fn archiver() { archived_segments[0] .object_mapping .iter() - .filter(|object_mapping| !object_mapping.objects.is_empty()) + .filter(|object_mapping| !object_mapping.objects().is_empty()) .count(), 1 ); @@ -262,18 +262,18 @@ fn archiver() { archived_segments[1] .object_mapping .iter() - .filter(|object_mapping| !object_mapping.objects.is_empty()) + .filter(|object_mapping| !object_mapping.objects().is_empty()) .count(), 0 ); { let block_objects = - iter::repeat(block_1.as_ref()).zip(block_1_object_mapping.objects.iter().skip(2)); + iter::repeat(block_1.as_ref()).zip(block_1_object_mapping.objects().iter().skip(2)); let piece_objects = archived_segments[0] .pieces .source_pieces() .zip(&archived_segments[0].object_mapping) - .flat_map(|(piece, object_mapping)| iter::repeat(piece).zip(&object_mapping.objects)); + .flat_map(|(piece, object_mapping)| iter::repeat(piece).zip(object_mapping.objects())); compare_block_objects_to_piece_objects(block_objects, piece_objects); } @@ -538,8 +538,8 @@ fn spill_over_edge_case() { // subtracting with overflow when trying to slice internal bytes of the segment item let archived_segments = archiver.add_block( vec![0u8; RecordedHistorySegment::SIZE], - BlockObjectMapping { - objects: vec![BlockObject::V0 { + BlockObjectMapping::V0 { + objects: vec![BlockObject { hash: Blake3Hash::default(), offset: 0, }], @@ -552,7 +552,7 @@ fn spill_over_edge_case() { archived_segments[0] .object_mapping .iter() - .filter(|o| !o.objects.is_empty()) + .filter(|o| !o.objects().is_empty()) .count(), 0 ); @@ -560,7 +560,7 @@ fn spill_over_edge_case() { archived_segments[1] .object_mapping .iter() - .filter(|o| !o.objects.is_empty()) + .filter(|o| !o.objects().is_empty()) .count(), 1 ); @@ -589,7 +589,7 @@ fn object_on_the_edge_of_segment() { .unwrap(); let mut second_block = vec![0u8; RecordedHistorySegment::SIZE * 2]; - let object_mapping = BlockObject::V0 { + let object_mapping = BlockObject { hash: Blake3Hash::default(), // Offset is designed to fall exactly on the edge of the segment offset: RecordedHistorySegment::SIZE as u32 @@ -624,7 +624,7 @@ fn object_on_the_edge_of_segment() { }; let mapped_bytes = rand::random::<[u8; 32]>().to_vec().encode(); // Write mapped bytes at expected offset in source data - second_block[object_mapping.offset() as usize..][..mapped_bytes.len()] + second_block[object_mapping.offset as usize..][..mapped_bytes.len()] .copy_from_slice(&mapped_bytes); // First ensure that any smaller offset will get translated into the first archived segment, @@ -632,10 +632,10 @@ fn object_on_the_edge_of_segment() { { let archived_segments = archiver.clone().add_block( second_block.clone(), - BlockObjectMapping { - objects: vec![BlockObject::V0 { - hash: object_mapping.hash(), - offset: object_mapping.offset() - 1, + BlockObjectMapping::V0 { + objects: vec![BlockObject { + hash: object_mapping.hash, + offset: object_mapping.offset - 1, }], }, true, @@ -646,7 +646,7 @@ fn object_on_the_edge_of_segment() { archived_segments[0] .object_mapping .iter() - .filter(|o| !o.objects.is_empty()) + .filter(|o| !o.objects().is_empty()) .count(), 1 ); @@ -654,7 +654,7 @@ fn object_on_the_edge_of_segment() { let archived_segments = archiver.add_block( second_block, - BlockObjectMapping { + BlockObjectMapping::V0 { objects: vec![object_mapping], }, true, @@ -665,7 +665,7 @@ fn object_on_the_edge_of_segment() { archived_segments[0] .object_mapping .iter() - .filter(|o| !o.objects.is_empty()) + .filter(|o| !o.objects().is_empty()) .count(), 0 ); @@ -674,11 +674,11 @@ fn object_on_the_edge_of_segment() { archived_segments[1] .object_mapping .iter() - .filter(|o| !o.objects.is_empty()) + .filter(|o| !o.objects().is_empty()) .count(), 1 ); - assert_eq!(archived_segments[1].object_mapping[0].objects.len(), 1); + assert_eq!(archived_segments[1].object_mapping[0].objects().len(), 1); // Ensure bytes are mapped correctly assert_eq!( @@ -687,7 +687,7 @@ fn object_on_the_edge_of_segment() { .to_raw_record_chunks() .flatten() .copied() - .skip(archived_segments[1].object_mapping[0].objects[0].offset() as usize) + .skip(archived_segments[1].object_mapping[0].objects()[0].offset as usize) .take(mapped_bytes.len()) .collect::>(), mapped_bytes diff --git a/crates/subspace-core-primitives/src/objects.rs b/crates/subspace-core-primitives/src/objects.rs index c57e9a2197..5b4b7da5b8 100644 --- a/crates/subspace-core-primitives/src/objects.rs +++ b/crates/subspace-core-primitives/src/objects.rs @@ -36,95 +36,116 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Copy, Clone, PartialEq, Eq, Ord, PartialOrd, Hash, Encode, Decode, TypeInfo)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] -pub enum BlockObject { +pub struct BlockObject { + /// Object hash + #[cfg_attr(feature = "serde", serde(with = "hex"))] + pub hash: Blake3Hash, + /// Offset of object in the encoded block. + pub offset: u32, +} + +/// Mapping of objects stored inside of the block +#[derive(Debug, Clone, PartialEq, Eq, Ord, PartialOrd, Hash, Encode, Decode, TypeInfo)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] +#[cfg_attr(feature = "serde", serde(rename_all_fields = "camelCase"))] +pub enum BlockObjectMapping { /// V0 of object mapping data structure - // TODO: move the enum and accessor method to BlockObjectMapping #[codec(index = 0)] - #[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] V0 { - /// Object hash - #[cfg_attr(feature = "serde", serde(with = "hex"))] - hash: Blake3Hash, - /// Offset of object in the encoded block. - offset: u32, + /// Objects stored inside of the block + objects: Vec, }, } -impl BlockObject { - /// Object hash - pub fn hash(&self) -> Blake3Hash { - match self { - Self::V0 { hash, .. } => *hash, +impl Default for BlockObjectMapping { + fn default() -> Self { + Self::V0 { + objects: Vec::new(), + } + } +} + +impl BlockObjectMapping { + /// Returns a newly created BlockObjectMapping from a list of object mappings + #[inline] + pub fn from_objects(objects: impl IntoIterator) -> Self { + Self::V0 { + objects: objects.into_iter().collect(), } } - /// Offset of object in the encoded block. - pub fn offset(&self) -> u32 { + /// Returns the object mappings + pub fn objects(&self) -> &[BlockObject] { match self { - Self::V0 { offset, .. } => *offset, + Self::V0 { objects, .. } => objects, } } - /// Sets new offset. - pub fn set_offset(&mut self, new_offset: u32) { + /// Returns the object mappings as a mutable slice + pub fn objects_mut(&mut self) -> &mut Vec { match self { - Self::V0 { offset, .. } => { - *offset = new_offset; - } + Self::V0 { objects, .. } => objects, } } } -/// Mapping of objects stored inside of the block -#[derive(Debug, Default, Clone, PartialEq, Eq, Ord, PartialOrd, Hash, Encode, Decode, TypeInfo)] +/// Object stored inside of the piece +#[derive(Debug, Copy, Clone, PartialEq, Eq, Ord, PartialOrd, Hash, Encode, Decode, TypeInfo)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] -pub struct BlockObjectMapping { - /// Objects stored inside of the block - pub objects: Vec, +pub struct PieceObject { + /// Object hash + #[cfg_attr(feature = "serde", serde(with = "hex"))] + pub hash: Blake3Hash, + /// Raw record offset of the object in that piece, for use with `Record::to_raw_record_bytes` + pub offset: u32, } -/// Object stored inside of the piece -#[derive(Debug, Copy, Clone, PartialEq, Eq, Ord, PartialOrd, Hash, Encode, Decode, TypeInfo)] +/// Mapping of objects stored inside of the piece +#[derive(Debug, Clone, PartialEq, Eq, Ord, PartialOrd, Hash, Encode, Decode, TypeInfo)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] -pub enum PieceObject { +#[cfg_attr(feature = "serde", serde(rename_all_fields = "camelCase"))] +pub enum PieceObjectMapping { /// V0 of object mapping data structure - // TODO: move the enum and accessor method to PieceObjectMapping #[codec(index = 0)] - #[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] V0 { - /// Object hash - #[cfg_attr(feature = "serde", serde(with = "hex"))] - hash: Blake3Hash, - /// Raw record offset of the object in that piece, for use with `Record::to_raw_record_bytes` - offset: u32, + /// Objects stored inside of the piece + objects: Vec, }, } -impl PieceObject { - /// Object hash - pub fn hash(&self) -> Blake3Hash { - match self { - Self::V0 { hash, .. } => *hash, +impl Default for PieceObjectMapping { + fn default() -> Self { + Self::V0 { + objects: Vec::new(), } } +} - /// Raw record offset of the object in that piece, for use with `Record::to_raw_record_bytes` - pub fn offset(&self) -> u32 { +impl PieceObjectMapping { + /// Returns a newly created PieceObjectMapping from a list of object mappings + #[inline] + pub fn from_objects(objects: impl IntoIterator) -> Self { + Self::V0 { + objects: objects.into_iter().collect(), + } + } + + /// Returns the object mappings as a read-only slice + pub fn objects(&self) -> &[PieceObject] { match self { - Self::V0 { offset, .. } => *offset, + Self::V0 { objects, .. } => objects, } } -} -/// Mapping of objects stored inside of the piece -#[derive(Debug, Default, Clone, PartialEq, Eq, Ord, PartialOrd, Hash, Encode, Decode, TypeInfo)] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] -pub struct PieceObjectMapping { - /// Objects stored inside of the piece - pub objects: Vec, + /// Returns the object mappings as a mutable slice + pub fn objects_mut(&mut self) -> &mut Vec { + match self { + Self::V0 { objects, .. } => objects, + } + } } /// Object stored in the history of the blockchain @@ -164,9 +185,9 @@ impl GlobalObject { /// Returns a newly created GlobalObject from a piece index and object. pub fn new(piece_index: PieceIndex, piece_object: &PieceObject) -> Self { Self { - hash: piece_object.hash().into(), + hash: piece_object.hash.into(), piece_index, - offset: piece_object.offset(), + offset: piece_object.offset, } } } @@ -213,4 +234,11 @@ impl GlobalObjectMapping { Self::V0 { objects, .. } => objects, } } + + /// Returns the object mappings as a mutable slice + pub fn objects_mut(&mut self) -> &mut Vec { + match self { + Self::V0 { objects, .. } => objects, + } + } } diff --git a/crates/subspace-runtime/src/object_mapping.rs b/crates/subspace-runtime/src/object_mapping.rs index 9ab1a13071..76daaeabcf 100644 --- a/crates/subspace-runtime/src/object_mapping.rs +++ b/crates/subspace-runtime/src/object_mapping.rs @@ -92,7 +92,7 @@ pub(crate) fn extract_call_block_object_mapping>( match call { // Extract the actual object mappings. RuntimeCall::System(frame_system::Call::remark { remark }) => { - objects.push(BlockObject::V0 { + objects.push(BlockObject { hash: crypto::blake3_hash(remark), // Add frame_system::Call enum variant to the base offset. offset: base_offset + 1, @@ -136,7 +136,7 @@ pub(crate) fn extract_block_object_mapping( extract_call_block_object_mapping( base_extrinsic_offset as u32, - &mut block_object_mapping.objects, + block_object_mapping.objects_mut(), &extrinsic.function, MAX_OBJECT_MAPPING_RECURSION_DEPTH, &mut successful_calls, diff --git a/test/subspace-test-runtime/src/lib.rs b/test/subspace-test-runtime/src/lib.rs index 8a15246cb3..2b14e15f53 100644 --- a/test/subspace-test-runtime/src/lib.rs +++ b/test/subspace-test-runtime/src/lib.rs @@ -93,8 +93,8 @@ use sp_version::RuntimeVersion; use static_assertions::const_assert; use subspace_core_primitives::objects::{BlockObject, BlockObjectMapping}; use subspace_core_primitives::{ - HistorySize, Piece, Randomness, SegmentCommitment, SegmentHeader, SegmentIndex, SlotNumber, - SolutionRange, U256, + crypto, HistorySize, Piece, Randomness, SegmentCommitment, SegmentHeader, SegmentIndex, + SlotNumber, SolutionRange, U256, }; use subspace_runtime_primitives::{ AccountId, Balance, BlockNumber, FindBlockRewardAddress, Hash, Moment, Nonce, Signature, @@ -1020,17 +1020,28 @@ fn extract_call_block_object_mapping>( recursion_depth_left: u16, successful_calls: &mut Peekable, ) { - // Add enum variant to the base offset. + // Add RuntimeCall enum variant to the base offset. base_offset += 1; - if let RuntimeCall::Utility(call) = call { - extract_utility_block_object_mapping( + match call { + // Extract the actual object mappings. + RuntimeCall::System(frame_system::Call::remark { remark }) => { + objects.push(BlockObject { + hash: crypto::blake3_hash(remark), + // Add frame_system::Call enum variant to the base offset. + offset: base_offset + 1, + }); + } + // Recursively extract object mappings for the call. + RuntimeCall::Utility(call) => extract_utility_block_object_mapping( base_offset, objects, call, recursion_depth_left, successful_calls, - ); + ), + // Other calls don't contain object mappings. + _ => {} } } @@ -1056,7 +1067,7 @@ fn extract_block_object_mapping(block: Block, successful_calls: Vec) -> Bl extract_call_block_object_mapping( base_extrinsic_offset as u32, - &mut block_object_mapping.objects, + block_object_mapping.objects_mut(), &extrinsic.function, MAX_OBJECT_MAPPING_RECURSION_DEPTH, &mut successful_calls,