Skip to content

Commit

Permalink
Yair/storage db documentation (#825)
Browse files Browse the repository at this point in the history
* Documentation for the db module in storage crate

* Rename Result -> DbResult
  • Loading branch information
yair-starkware authored Jul 13, 2023
1 parent 93d9cdd commit 427dd8e
Showing 1 changed file with 51 additions and 26 deletions.
77 changes: 51 additions & 26 deletions crates/papyrus_storage/src/db/mod.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,20 @@
//! Basic structs for interacting with the db.
//!
//! Low database layer for interaction with libmdbx. The API is supposedly generic enough to easily
//! replace the database library with other Berkley-like database implementations.
//!
//! Assumptions:
//! - The database is transactional with full ACID semantics.
//! - The keys are always sorted and range lookups are supported.
//!
//! Guarantees:
//! - The serialization is consistent across code versions (though, not necessarily across
//! machines).

#[cfg(test)]
mod db_test;
// TODO(yair): Make the serialization module pub(crate).
#[doc(hidden)]
pub mod serialization;

use std::borrow::Cow;
Expand All @@ -14,12 +29,6 @@ use starknet_api::core::ChainId;

use crate::db::serialization::{StorageSerde, StorageSerdeEx};

// Low database layer for interaction with libmdbx. The API is supposedly generic enough to easily
// replace the database library with other Berkley-like database implementations.
//
// Assumptions:
// The serialization is consistent across code versions (though, not necessarily across machines).

// Maximum number of Sub-Databases.
const MAX_DBS: usize = 26;

Expand All @@ -30,24 +39,33 @@ type Environment = libmdbx::Environment<EnvironmentKind>;
type DbKeyType<'env> = Cow<'env, [u8]>;
type DbValueType<'env> = Cow<'env, [u8]>;

/// The configuration of the database.
#[derive(Clone, Serialize, Deserialize)]
pub struct DbConfig {
/// The path prefix of the database files. The final path is the path prefix followed by the
/// chain id.
pub path_prefix: PathBuf,
/// The [chain id](https://docs.rs/starknet_api/latest/starknet_api/core/struct.ChainId.html) of the Starknet network.
pub chain_id: ChainId,
/// The minimum size of the database.
pub min_size: usize,
/// The maximum size of the database.
pub max_size: usize,
/// The growth step of the database.
pub growth_step: isize,
}

impl DbConfig {
fn path(&self) -> PathBuf {
/// Returns the path of the database (path prefix, followed by the chain id).
pub fn path(&self) -> PathBuf {
self.path_prefix.join(self.chain_id.0.as_str())
}
}

/// A single table statistics.
#[derive(Serialize, Deserialize, Debug)]
pub struct DbTableStats {
/// The name of the table.
pub database: String,
pub branch_pages: usize,
pub depth: u32,
Expand All @@ -57,21 +75,25 @@ pub struct DbTableStats {
pub page_size: u32,
}

/// An error that can occur when interacting with the database.
#[derive(thiserror::Error, Debug)]
pub enum DbError {
/// An error that occurred in the database library.
#[error(transparent)]
Inner(#[from] libmdbx::Error),
#[error("Deserialization failed.")]
/// An error that occurred during deserialization.
InnerDeserialization,
/// An error that occurred during serialization.
#[error("Serialization failed.")]
Serialization,
}
type Result<V> = result::Result<V, DbError>;
type DbResult<V> = result::Result<V, DbError>;

/// Opens an MDBX environment and returns a reader and a writer to it.
/// Tries to open an MDBX environment and returns a reader and a writer to it.
/// There is a single non clonable writer instance, to make sure there is only one write transaction
/// at any given moment.
pub(crate) fn open_env(config: DbConfig) -> Result<(DbReader, DbWriter)> {
pub(crate) fn open_env(config: DbConfig) -> DbResult<(DbReader, DbWriter)> {
let env = Arc::new(
Environment::new()
.set_geometry(Geometry {
Expand All @@ -95,12 +117,12 @@ pub(crate) struct DbWriter {
}

impl DbReader {
pub(crate) fn begin_ro_txn(&self) -> Result<DbReadTransaction<'_>> {
pub(crate) fn begin_ro_txn(&self) -> DbResult<DbReadTransaction<'_>> {
Ok(DbReadTransaction { txn: self.env.begin_ro_txn()? })
}

/// Returns statistics about a specific table in the database.
pub(crate) fn get_table_stats(&self, name: &str) -> Result<DbTableStats> {
pub(crate) fn get_table_stats(&self, name: &str) -> DbResult<DbTableStats> {
let db_txn = self.begin_ro_txn()?;
let database = db_txn.txn.open_db(Some(name))?;
let stat = db_txn.txn.db_stat(&database)?;
Expand All @@ -119,14 +141,14 @@ impl DbReader {
type DbReadTransaction<'env> = DbTransaction<'env, RO>;

impl DbWriter {
pub(crate) fn begin_rw_txn(&mut self) -> Result<DbWriteTransaction<'_>> {
pub(crate) fn begin_rw_txn(&mut self) -> DbResult<DbWriteTransaction<'_>> {
Ok(DbWriteTransaction { txn: self.env.begin_rw_txn()? })
}

pub(crate) fn create_table<K: StorageSerde, V: StorageSerde>(
&mut self,
name: &'static str,
) -> Result<TableIdentifier<K, V>> {
) -> DbResult<TableIdentifier<K, V>> {
let txn = self.env.begin_rw_txn()?;
txn.create_db(Some(name), DatabaseFlags::empty())?;
txn.commit()?;
Expand All @@ -137,12 +159,13 @@ impl DbWriter {
type DbWriteTransaction<'env> = DbTransaction<'env, RW>;

impl<'a> DbWriteTransaction<'a> {
pub(crate) fn commit(self) -> Result<()> {
pub(crate) fn commit(self) -> DbResult<()> {
self.txn.commit()?;
Ok(())
}
}

#[doc(hidden)]
// Transaction wrappers.
pub trait TransactionKind {
type Internal: libmdbx::TransactionKind;
Expand All @@ -156,19 +179,19 @@ impl<'a, Mode: TransactionKind> DbTransaction<'a, Mode> {
pub fn open_table<'env, K: StorageSerde, V: StorageSerde>(
&'env self,
table_id: &TableIdentifier<K, V>,
) -> Result<TableHandle<'env, K, V>> {
) -> DbResult<TableHandle<'env, K, V>> {
let database = self.txn.open_db(Some(table_id.name))?;
Ok(TableHandle { database, _key_type: PhantomData {}, _value_type: PhantomData {} })
}
}

pub struct TableIdentifier<K: StorageSerde, V: StorageSerde> {
pub(crate) struct TableIdentifier<K: StorageSerde, V: StorageSerde> {
name: &'static str,
_key_type: PhantomData<K>,
_value_type: PhantomData<V>,
}

pub struct TableHandle<'env, K: StorageSerde, V: StorageSerde> {
pub(crate) struct TableHandle<'env, K: StorageSerde, V: StorageSerde> {
database: libmdbx::Database<'env>,
_key_type: PhantomData<K>,
_value_type: PhantomData<V>,
Expand All @@ -178,7 +201,7 @@ impl<'env, 'txn, K: StorageSerde, V: StorageSerde> TableHandle<'env, K, V> {
pub(crate) fn cursor<Mode: TransactionKind>(
&'env self,
txn: &'txn DbTransaction<'env, Mode>,
) -> Result<DbCursor<'txn, Mode, K, V>> {
) -> DbResult<DbCursor<'txn, Mode, K, V>> {
let cursor = txn.txn.cursor(&self.database)?;
Ok(DbCursor { cursor, _key_type: PhantomData {}, _value_type: PhantomData {} })
}
Expand All @@ -187,7 +210,7 @@ impl<'env, 'txn, K: StorageSerde, V: StorageSerde> TableHandle<'env, K, V> {
&'env self,
txn: &'env DbTransaction<'env, Mode>,
key: &K,
) -> Result<Option<V>> {
) -> DbResult<Option<V>> {
// TODO: Support zero-copy. This might require a return type of Cow<'env, ValueType>.
let bin_key = key.serialize()?;
let Some(bytes) = txn.txn.get::<Cow<'env, [u8]>>(&self.database, &bin_key)? else {
Expand All @@ -202,7 +225,7 @@ impl<'env, 'txn, K: StorageSerde, V: StorageSerde> TableHandle<'env, K, V> {
txn: &DbTransaction<'env, RW>,
key: &K,
value: &V,
) -> Result<()> {
) -> DbResult<()> {
let data = value.serialize()?;
let bin_key = key.serialize()?;
txn.txn.put(&self.database, bin_key, data, WriteFlags::UPSERT)?;
Expand All @@ -214,15 +237,15 @@ impl<'env, 'txn, K: StorageSerde, V: StorageSerde> TableHandle<'env, K, V> {
txn: &DbTransaction<'env, RW>,
key: &K,
value: &V,
) -> Result<()> {
) -> DbResult<()> {
let data = value.serialize()?;
let bin_key = key.serialize()?;
txn.txn.put(&self.database, bin_key, data, WriteFlags::NO_OVERWRITE)?;
Ok(())
}

#[allow(dead_code)]
pub(crate) fn delete(&'env self, txn: &DbTransaction<'env, RW>, key: &K) -> Result<()> {
pub(crate) fn delete(&'env self, txn: &DbTransaction<'env, RW>, key: &K) -> DbResult<()> {
let bin_key = key.serialize()?;
txn.txn.del(&self.database, bin_key, None)?;
Ok(())
Expand All @@ -236,7 +259,7 @@ pub(crate) struct DbCursor<'txn, Mode: TransactionKind, K: StorageSerde, V: Stor
}

impl<'txn, Mode: TransactionKind, K: StorageSerde, V: StorageSerde> DbCursor<'txn, Mode, K, V> {
pub(crate) fn prev(&mut self) -> Result<Option<(K, V)>> {
pub(crate) fn prev(&mut self) -> DbResult<Option<(K, V)>> {
let prev_cursor_res = self.cursor.prev::<DbKeyType<'_>, DbValueType<'_>>()?;
match prev_cursor_res {
None => Ok(None),
Expand All @@ -251,7 +274,7 @@ impl<'txn, Mode: TransactionKind, K: StorageSerde, V: StorageSerde> DbCursor<'tx
}

#[allow(clippy::should_implement_trait)]
pub(crate) fn next(&mut self) -> Result<Option<(K, V)>> {
pub(crate) fn next(&mut self) -> DbResult<Option<(K, V)>> {
let prev_cursor_res = self.cursor.next::<DbKeyType<'_>, DbValueType<'_>>()?;
match prev_cursor_res {
None => Ok(None),
Expand All @@ -266,7 +289,7 @@ impl<'txn, Mode: TransactionKind, K: StorageSerde, V: StorageSerde> DbCursor<'tx
}

/// Position at first key greater than or equal to specified key.
pub(crate) fn lower_bound(&mut self, key: &K) -> Result<Option<(K, V)>> {
pub(crate) fn lower_bound(&mut self, key: &K) -> DbResult<Option<(K, V)>> {
let key_bytes = key.serialize()?;
let prev_cursor_res =
self.cursor.set_range::<DbKeyType<'_>, DbValueType<'_>>(&key_bytes)?;
Expand All @@ -283,12 +306,14 @@ impl<'txn, Mode: TransactionKind, K: StorageSerde, V: StorageSerde> DbCursor<'tx
}
}

#[doc(hidden)]
pub struct RO {}

impl TransactionKind for RO {
type Internal = libmdbx::RO;
}

#[doc(hidden)]
pub struct RW {}

impl TransactionKind for RW {
Expand Down

0 comments on commit 427dd8e

Please sign in to comment.