Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix for mmap file creation #84

Merged
merged 2 commits into from
Jul 25, 2024
Merged

fix for mmap file creation #84

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a2c5105
fix for mmap file creation
ewoolsey Jul 25, 2024
0c42a52
Capacity tests for mmap vec
ewoolsey Jul 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,4 @@ Cargo.lock
snarkfiles_tmp
semaphore_files
.idea
lcov.info
22 changes: 22 additions & 0 deletions src/cascading_merkle_tree.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,11 @@ where
/// Validates all elements of the storage, ensuring that they
/// correspond to a valid tree.
pub fn validate(&self) -> Result<()> {
debug_assert_eq!(
self.root,
self.compute_from_storage_tip(0),
"Root hash does not match recomputed root hash"
);
self.storage.validate(&self.empty_value)
}

Expand Down Expand Up @@ -985,6 +990,23 @@ mod tests {
}
}

#[test]
fn test_extend_from_slice_2() {
for increment in 1..20 {
let mut tree = CascadingMerkleTree::<PoseidonHash>::new(vec![], 30, &Field::ZERO);
let mut vec = vec![];
for _ in 0..20 {
let slice = (0..increment)
.map(|_| Field::from(rand::random::<usize>()))
.collect::<Vec<_>>();
tree.extend_from_slice(&slice);
vec.extend_from_slice(&slice);
tree.validate().unwrap();
assert_eq!(tree.leaves().collect::<Vec<_>>(), vec);
}
}
}

#[test]
fn test_vec_realloc_speed() {
let empty = 0;
Expand Down
208 changes: 204 additions & 4 deletions src/generic_storage/mmap_vec.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,8 @@
/// Notably this means that there can exist no other mutable mappings to the
/// same file in this process or any other
pub unsafe fn create(file: File) -> color_eyre::Result<Self> {
let initial_byte_len = META_SIZE;

file.set_len(initial_byte_len as u64)
file.set_len(0)?;
file.set_len(META_SIZE as u64)
.context("Failed to resize underlying file")?;

let mut s = Self::restore(file)?;
Expand All @@ -63,7 +62,12 @@
/// # Safety
/// Same requirements as `restore`
pub unsafe fn restore_from_path(file_path: impl AsRef<Path>) -> color_eyre::Result<Self> {
let file = OpenOptions::new().read(true).write(true).open(file_path)?;
let file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
Dzejkop marked this conversation as resolved.
Show resolved Hide resolved
.open(file_path)?;

Self::restore(file)
}
Expand All @@ -78,7 +82,7 @@
/// Notably this means that there can exist no other mutable mappings to the
/// same file in this process or any other
pub unsafe fn restore(file: File) -> color_eyre::Result<Self> {
const { assert!(std::mem::size_of::<T>() != 0) };

Check failure on line 85 in src/generic_storage/mmap_vec.rs

View workflow job for this annotation

GitHub Actions / Test 

inline-const is experimental

let mut byte_len = file.metadata()?.len() as usize;

Expand Down Expand Up @@ -245,6 +249,202 @@

use super::*;

#[test]
#[allow(clippy::manual_bits)]
fn test_capacity_push() {
let f = tempfile::NamedTempFile::new().unwrap();
let file_path = f.path().to_owned();

let mut storage: MmapVec<u32> = unsafe { MmapVec::create(f.reopen().unwrap()).unwrap() };
assert_eq!(storage.capacity, 0);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len(),
META_SIZE as u64
);

storage.push(0);
assert_eq!(storage.capacity, 1);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len() as usize,
size_of::<u32>() + META_SIZE
);

storage.push(0);
assert_eq!(storage.capacity, 2);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len() as usize,
size_of::<u32>() * 2 + META_SIZE
);

storage.push(0);
assert_eq!(storage.capacity, 4);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len() as usize,
size_of::<u32>() * 4 + META_SIZE
);

storage.push(0);
assert_eq!(storage.capacity, 4);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len() as usize,
size_of::<u32>() * 4 + META_SIZE
);

storage.push(0);
assert_eq!(storage.capacity, 8);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len() as usize,
size_of::<u32>() * 8 + META_SIZE
);
}

#[test]
#[allow(clippy::manual_bits)]
fn test_capacity_extend() {
let f = tempfile::NamedTempFile::new().unwrap();
let file_path = f.path().to_owned();

let mut storage: MmapVec<u32> = unsafe { MmapVec::create(f.reopen().unwrap()).unwrap() };
assert_eq!(storage.capacity, 0);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len(),
META_SIZE as u64
);

storage.extend_from_slice(&[0, 0]);
assert_eq!(storage.capacity, 2);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len() as usize,
size_of::<u32>() * 2 + META_SIZE
);

storage.extend_from_slice(&[0, 0, 0]);
assert_eq!(storage.capacity, 8);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len() as usize,
size_of::<u32>() * 8 + META_SIZE
);

storage.extend_from_slice(&[0]);
assert_eq!(storage.capacity, 8);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len() as usize,
size_of::<u32>() * 8 + META_SIZE
);
}

#[test]
#[allow(clippy::manual_bits)]
fn test_capacity_create() {
let f = tempfile::NamedTempFile::new().unwrap();
let file_path = f.path().to_owned();

let mut storage: MmapVec<u32> = unsafe { MmapVec::create(f.reopen().unwrap()).unwrap() };
assert_eq!(storage.capacity, 0);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len() as usize,
META_SIZE
);

storage.extend_from_slice(&[0, 0, 0, 0, 0]);
assert_eq!(storage.capacity, 8);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len() as usize,
size_of::<u32>() * 8 + META_SIZE
);

let storage: MmapVec<u32> = unsafe { MmapVec::create(f.reopen().unwrap()).unwrap() };
assert_eq!(storage.capacity, 0);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len() as usize,
META_SIZE
);
}

#[test]
#[allow(clippy::manual_bits)]
fn test_capacity_create_from_path() {
let f = tempfile::NamedTempFile::new().unwrap();
let file_path = f.path().to_owned();

let mut storage: MmapVec<u32> = unsafe { MmapVec::create(f.reopen().unwrap()).unwrap() };
assert_eq!(storage.capacity, 0);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len() as usize,
META_SIZE
);

storage.extend_from_slice(&[0, 0, 0, 0, 0]);
assert_eq!(storage.capacity, 8);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len() as usize,
size_of::<u32>() * 8 + META_SIZE
);

let storage: MmapVec<u32> = unsafe { MmapVec::create_from_path(&file_path).unwrap() };
assert_eq!(storage.capacity, 0);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len() as usize,
META_SIZE
);
}

#[test]
#[allow(clippy::manual_bits)]
fn test_capacity_restore() {
let f = tempfile::NamedTempFile::new().unwrap();
let file_path = f.path().to_owned();

let mut storage: MmapVec<u32> = unsafe { MmapVec::create(f.reopen().unwrap()).unwrap() };
assert_eq!(storage.capacity, 0);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len() as usize,
META_SIZE
);

storage.extend_from_slice(&[0, 0, 0, 0, 0]);
assert_eq!(storage.capacity, 8);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len() as usize,
size_of::<u32>() * 8 + META_SIZE
);

let storage: MmapVec<u32> = unsafe { MmapVec::restore(f.reopen().unwrap()).unwrap() };
assert_eq!(storage.capacity, 8);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len() as usize,
size_of::<u32>() * 8 + META_SIZE
);
}
Dzejkop marked this conversation as resolved.
Show resolved Hide resolved

#[test]
#[allow(clippy::manual_bits)]
fn test_capacity_restore_from_path() {
let f = tempfile::NamedTempFile::new().unwrap();
let file_path = f.path().to_owned();

let mut storage: MmapVec<u32> = unsafe { MmapVec::create(f.reopen().unwrap()).unwrap() };
assert_eq!(storage.capacity, 0);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len() as usize,
META_SIZE
);

storage.extend_from_slice(&[0, 0, 0, 0, 0]);
assert_eq!(storage.capacity, 8);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len() as usize,
size_of::<u32>() * 8 + META_SIZE
);

let storage: MmapVec<u32> = unsafe { MmapVec::restore_from_path(&file_path).unwrap() };
assert_eq!(storage.capacity, 8);
assert_eq!(
std::fs::metadata(&file_path).unwrap().len() as usize,
size_of::<u32>() * 8 + META_SIZE
);
}

#[test]
fn test_mmap_vec() {
let f = tempfile::tempfile().unwrap();
Expand Down
Loading