diff --git a/.gitmodules b/.gitmodules index e946c61f..da01402e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "lmdb-master-sys/lmdb"] path = lmdb-master-sys/lmdb - url = https://github.com/LMDB/lmdb - branch = mdb.master + url = https://github.com/meilisearch/lmdb.git + branch = allow-nested-rtxn-from-wtxn diff --git a/heed/Cargo.toml b/heed/Cargo.toml index f707a86c..57943669 100644 --- a/heed/Cargo.toml +++ b/heed/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "heed" -version = "0.20.4" +version = "0.20.5" authors = ["Kerollmops "] description = "A fully typed LMDB wrapper with minimum overhead" license = "MIT" @@ -23,8 +23,11 @@ serde = { version = "1.0.203", features = ["derive"], optional = true } synchronoise = "1.0.1" [dev-dependencies] -serde = { version = "1.0.203", features = ["derive"] } -tempfile = "3.10.1" +rand = "0.9.0" +rayon = "1.10.0" +roaring = "0.10.10" +serde = { version = "1.0.217", features = ["derive"] } +tempfile = "3.15.0" [target.'cfg(windows)'.dependencies] url = "2.5.2" @@ -61,6 +64,10 @@ arbitrary_precision = ["heed-types/arbitrary_precision"] raw_value = ["heed-types/raw_value"] unbounded_depth = ["heed-types/unbounded_depth"] +asan = ["lmdb-master-sys/asan"] +fuzzer = ["lmdb-master-sys/fuzzer"] +fuzzer-no-link = ["lmdb-master-sys/fuzzer-no-link"] + # Whether to tell LMDB to use POSIX semaphores during compilation # (instead of the default, which are System V semaphores). # POSIX semaphores are required for Apple's App Sandbox on iOS & macOS, @@ -70,6 +77,18 @@ unbounded_depth = ["heed-types/unbounded_depth"] # posix-sem = ["lmdb-master-sys/posix-sem"] +# Prints debugging information on stderr. +debug-simple = ["lmdb-master-sys/debug-simple"] +# Add copious tracing. It enables the debug-simple feature. +debug-copious-tracing = ["lmdb-master-sys/debug-copious-tracing"] +# Add dumps of all IDLs read from and written to the database +# (used for free space management). It enables the debug-simple +# and debug-dump-idls features. +debug-dump-idls = ["lmdb-master-sys/debug-dump-idls"] +# Do an audit after each commit. It enables the debug-simple, +# debug-copious-tracing, and debug-dump-idls features. +debug-audit-commit = ["lmdb-master-sys/debug-audit-commit"] + # These features configure the MDB_IDL_LOGN macro, which determines # the size of the free and dirty page lists (and thus the amount of memory # allocated when opening an LMDB environment in read-write mode). @@ -110,6 +129,10 @@ mdb_idl_logn_16 = ["lmdb-master-sys/mdb_idl_logn_16"] # computers then you need to keep your keys within the smaller 1982 byte limit. longer-keys = ["lmdb-master-sys/longer-keys"] +[[example]] +name = "nested-rtxns" +required-features = ["read-txn-no-tls"] + [[example]] name = "rmp-serde" required-features = ["serde-rmp"] diff --git a/heed/examples/nested-rtxns.rs b/heed/examples/nested-rtxns.rs new file mode 100644 index 00000000..eda59354 --- /dev/null +++ b/heed/examples/nested-rtxns.rs @@ -0,0 +1,77 @@ +use heed::types::*; +use heed::{Database, EnvOpenOptions}; +use rand::prelude::*; +use rayon::prelude::*; +use roaring::RoaringBitmap; + +fn main() -> Result<(), Box> { + let dir = tempfile::tempdir()?; + let env = unsafe { + EnvOpenOptions::new() + .map_size(2 * 1024 * 1024 * 1024) // 2 GiB + .open(dir.path())? + }; + + // opening a write transaction + let mut wtxn = env.write_txn()?; + // we will open the default unnamed database + let db: Database, Bytes> = env.create_database(&mut wtxn, None)?; + + let mut buffer = Vec::new(); + for i in 0..1000 { + let mut rng = StdRng::seed_from_u64(i as u64); + let max = rng.random_range(10_000..=100_000); + let roaring = RoaringBitmap::from_sorted_iter(0..max)?; + buffer.clear(); + roaring.serialize_into(&mut buffer)?; + db.put(&mut wtxn, &i, &buffer)?; + } + + // opening multiple read-only transactions + // to check if those values are now available + // without committing beforehand + let rtxns = (0..1000).map(|_| env.nested_read_txn(&wtxn)).collect::>>()?; + + rtxns.into_par_iter().enumerate().for_each(|(i, rtxn)| { + let mut rng = StdRng::seed_from_u64(i as u64); + let max = rng.random_range(10_000..=100_000); + let roaring = RoaringBitmap::from_sorted_iter(0..max).unwrap(); + + let mut buffer = Vec::new(); + roaring.serialize_into(&mut buffer).unwrap(); + + let i = i as u32; + let ret = db.get(&rtxn, &i).unwrap(); + assert_eq!(ret, Some(&buffer[..])); + }); + + for i in 1000..10_000 { + let mut rng = StdRng::seed_from_u64(i as u64); + let max = rng.random_range(10_000..=100_000); + let roaring = RoaringBitmap::from_sorted_iter(0..max)?; + buffer.clear(); + roaring.serialize_into(&mut buffer)?; + db.put(&mut wtxn, &i, &buffer)?; + } + + // opening multiple read-only transactions + // to check if those values are now available + // without committing beforehand + let rtxns = + (1000..10_000).map(|_| env.nested_read_txn(&wtxn)).collect::>>()?; + + rtxns.into_par_iter().enumerate().for_each(|(i, rtxn)| { + let mut rng = StdRng::seed_from_u64(i as u64); + let max = rng.random_range(10_000..=100_000); + let roaring = RoaringBitmap::from_sorted_iter(0..max).unwrap(); + + let mut buffer = Vec::new(); + roaring.serialize_into(&mut buffer).unwrap(); + + let i = i as u32; + let ret = db.get(&rtxn, &i).unwrap(); + assert_eq!(ret, Some(&buffer[..])); + }); + + Ok(()) +} diff --git a/heed/src/env.rs b/heed/src/env.rs index 99ecf54a..118c8470 100644 --- a/heed/src/env.rs +++ b/heed/src/env.rs @@ -781,6 +781,52 @@ impl Env { RoTxn::new(self) } + /// Create a nested transaction with read only access for use with the environment. + /// + /// The new transaction will be a nested transaction, with the transaction indicated by parent + /// as its parent. Transactions may be nested to any level. + /// + /// ``` + /// use std::fs; + /// use std::path::Path; + /// use heed::{EnvOpenOptions, Database}; + /// use heed::types::*; + /// + /// # fn main() -> Result<(), Box> { + /// let dir = tempfile::tempdir()?; + /// let env = unsafe { + /// EnvOpenOptions::new() + /// .map_size(2 * 1024 * 1024) // 2 MiB + /// .open(dir.path())? + /// }; + /// + /// // we will open the default unnamed database + /// let mut wtxn = env.write_txn()?; + /// let db: Database, U32> = env.create_database(&mut wtxn, None)?; + /// + /// // opening a write transaction + /// for i in 0..1000 { + /// db.put(&mut wtxn, &i, &i)?; + /// } + /// + /// // opening multiple read-only transactions + /// // to check if those values are now available + /// // without committing beforehand + /// let rtxns = (0..1000).map(|_| env.nested_read_txn(&wtxn)).collect::>>()?; + /// + /// for (i, rtxn) in rtxns.iter().enumerate() { + /// let i = i as u32; + /// let ret = db.get(&rtxn, &i)?; + /// assert_eq!(ret, Some(i)); + /// } + /// + /// # Ok(()) } + /// ``` + #[cfg(feature = "read-txn-no-tls")] + pub fn nested_read_txn<'p>(&'p self, parent: &'p RwTxn) -> Result> { + RoTxn::nested(self, parent) + } + /// Create a transaction with read-only access for use with the environment. /// Contrary to [`Self::read_txn`], this version **owns** the environment, which /// means you won't be able to close the environment while this transaction is alive. diff --git a/heed/src/txn.rs b/heed/src/txn.rs index b8c18ae9..9b68d7bd 100644 --- a/heed/src/txn.rs +++ b/heed/src/txn.rs @@ -61,6 +61,22 @@ impl<'e> RoTxn<'e> { Ok(RoTxn { txn, env: Cow::Owned(env) }) } + #[cfg(feature = "read-txn-no-tls")] + pub(crate) fn nested<'p>(env: &'p Env, parent: &'p RwTxn) -> Result> { + let mut txn: *mut ffi::MDB_txn = ptr::null_mut(); + let parent_ptr: *mut ffi::MDB_txn = parent.txn.txn; + + unsafe { + // Note that we open a write transaction here and this is the (current) + // ugly way to trick LMDB and let me create multiple write txn. + mdb_result(ffi::mdb_txn_begin(env.env_mut_ptr(), parent_ptr, 0, &mut txn))? + }; + + // Note that we wrap the write txn into a RoTxn so it's + // safe as the user cannot do any modification with it. + Ok(RoTxn { txn, env: Cow::Borrowed(env) }) + } + pub(crate) fn env_mut_ptr(&self) -> *mut ffi::MDB_env { self.env.env_mut_ptr() } diff --git a/lmdb-master-sys/Cargo.toml b/lmdb-master-sys/Cargo.toml index c68b6fed..beb9c7b4 100644 --- a/lmdb-master-sys/Cargo.toml +++ b/lmdb-master-sys/Cargo.toml @@ -27,7 +27,9 @@ doctest = false libc = "0.2.155" [build-dependencies] -bindgen = { version = "0.69.4", default-features = false, optional = true, features = ["runtime"] } +bindgen = { version = "0.69.4", default-features = false, optional = true, features = [ + "runtime", +] } cc = "1.0.104" doxygen-rs = "0.4.2" @@ -41,6 +43,18 @@ fuzzer = [] fuzzer-no-link = [] posix-sem = [] +# Prints debugging information on stderr. +debug-simple = [] +# Add copious tracing. It enables the debug-simple feature. +debug-copious-tracing = [] +# Add dumps of all IDLs read from and written to the database +# (used for free space management). It enables the debug-simple +# and debug-dump-idls features. +debug-dump-idls = [] +# Do an audit after each commit. It enables the debug-simple, +# debug-copious-tracing, and debug-dump-idls features. +debug-audit-commit = [] + # These features configure the MDB_IDL_LOGN macro, which determines # the size of the free and dirty page lists (and thus the amount of memory # allocated when opening an LMDB environment in read-write mode). diff --git a/lmdb-master-sys/build.rs b/lmdb-master-sys/build.rs index 9bff3df5..8d2d52c9 100644 --- a/lmdb-master-sys/build.rs +++ b/lmdb-master-sys/build.rs @@ -102,6 +102,32 @@ const MDB_IDL_LOGN: u8 = 15; ))] const MDB_IDL_LOGN: u8 = 16; +#[cfg(not(any( + feature = "debug-simple", + feature = "debug-copious-tracing", + feature = "debug-dump-idls", + feature = "debug-audit-commit" +)))] +const MDB_DEBUG: Option = None; +#[cfg(all( + feature = "debug-simple", + not(any( + feature = "debug-copious-tracing", + feature = "debug-dump-idls", + feature = "debug-audit-commit" + )) +))] +const MDB_DEBUG: Option = Some(0); +#[cfg(all( + feature = "debug-copious-tracing", + not(any(feature = "debug-dump-idls", feature = "debug-audit-commit")) +))] +const MDB_DEBUG: Option = Some(1); +#[cfg(all(feature = "debug-dump-idls", not(feature = "debug-audit-commit")))] +const MDB_DEBUG: Option = Some(2); +#[cfg(feature = "debug-audit-commit")] +const MDB_DEBUG: Option = Some(3); + macro_rules! warn { ($message:expr) => { println!("cargo:warning={}", $message); @@ -133,12 +159,17 @@ fn main() { .flag_if_supported("-Wbad-function-cast") .flag_if_supported("-Wuninitialized"); + if let Some(debug) = MDB_DEBUG { + builder.define("MDB_DEBUG", Some(debug.to_string().as_str())); + } + if cfg!(feature = "posix-sem") { builder.define("MDB_USE_POSIX_SEM", None); } if cfg!(feature = "asan") { builder.flag("-fsanitize=address"); + builder.flag("-static-libasan"); } if cfg!(feature = "fuzzer") { diff --git a/lmdb-master-sys/lmdb b/lmdb-master-sys/lmdb index 88d0531d..d9089b9e 160000 --- a/lmdb-master-sys/lmdb +++ b/lmdb-master-sys/lmdb @@ -1 +1 @@ -Subproject commit 88d0531d3ec3a592cdd63ca77647d31c568c24bc +Subproject commit d9089b9eb5cbf63123edabd323cea0f2dc71cd88