Use rust-rocksdb 0.13 [ECR-3848] (exonum#1638)

LICENSE

@@ -186,7 +186,7 @@ APPENDIX: How to apply the Apache License to your work.
    same "printed page" as the copyright notice for easier
    identification within third-party archives.
 
-Copyright 2019 Exonum Team
+Copyright [yyyy] [name of copyright owner]
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -198,4 +198,4 @@ Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
-limitations under the License.
+limitations under the License.

components/merkledb/Cargo.toml

@@ -18,14 +18,15 @@ description = "Persistent storage implementation based on RocksDB which provides
 bincode = "1.1"
 byteorder = "1.3"
 chrono = "0.4.6"
+crossbeam = "0.7.3"
 enum-primitive-derive = "0.1"
 exonum-crypto = { path = "../crypto", version = "0.13.0-rc.2", features = ["with-serde"]}
 exonum-proto = { path = "../proto", version = "0.13.0-rc.2", optional = true }
 failure = "0.1"
 hex = "0.4"
 leb128 = "0.2"
 num-traits = "0.2"
-rocksdb = { version = "0.12.3", default-features = false }
+rocksdb = { version = "0.13", default-features = false }
 rust_decimal = "1.0, <1.1.0" # The 1.1.0 version requires Rust 1.39+
 serde = "1.0"
 serde_derive = "1.0"

components/merkledb/src/backends/rocksdb.rs

@@ -16,12 +16,12 @@
 
 pub use rocksdb::{BlockBasedOptions as RocksBlockOptions, WriteOptions as RocksDBWriteOptions};
 
-use std::{fmt, iter::Peekable, mem, path::Path, sync::Arc};
-
+use crossbeam::sync::{ShardedLock, ShardedLockReadGuard};
 use rocksdb::{
     self, checkpoint::Checkpoint, ColumnFamily, DBIterator, Options as RocksDbOptions, WriteBatch,
 };
 use smallvec::SmallVec;
+use std::{fmt, iter::Peekable, mem, path::Path, sync::Arc};
 
 use crate::{
     db::{check_database, Change},
@@ -39,7 +39,7 @@ const ID_SIZE: usize = mem::size_of::<u64>();
 /// This structure is required to potentially adapt the interface to
 /// use different databases.
 pub struct RocksDB {
-    db: Arc<rocksdb::DB>,
+    db: Arc<ShardedLock<rocksdb::DB>>,
     options: DbOptions,
 }
 
@@ -62,7 +62,7 @@ impl From<&DbOptions> for RocksDbOptions {
 /// A snapshot of a `RocksDB`.
 pub struct RocksDBSnapshot {
     snapshot: rocksdb::Snapshot<'static>,
-    db: Arc<rocksdb::DB>,
+    db: Arc<ShardedLock<rocksdb::DB>>,
 }
 
 /// An iterator over the entries of a `RocksDB`.
@@ -90,39 +90,50 @@ impl RocksDB {
             }
         };
         let mut db = Self {
-            db: Arc::new(inner),
+            db: Arc::new(ShardedLock::new(inner)),
             options: *options,
         };
         check_database(&mut db)?;
         Ok(db)
     }
 
-    pub(super) fn rocksdb(&self) -> &rocksdb::DB {
-        self.db.as_ref()
-    }
-
     /// Creates checkpoint of this database in the given directory. See [RocksDB docs] for
     /// details.
     ///
     /// Successfully created checkpoint can be opened using `RocksDB::open`.
     ///
     /// [RocksDB docs]: https://github.com/facebook/rocksdb/wiki/Checkpoints
     pub fn create_checkpoint<T: AsRef<Path>>(&self, path: T) -> crate::Result<()> {
-        let checkpoint = Checkpoint::new(&*self.db)?;
+        let checkpoint = Checkpoint::new(&*self.get_lock_guard())?;
         checkpoint.create_checkpoint(path)?;
         Ok(())
     }
 
+    fn cf_exists(&self, cf_name: &str) -> bool {
+        self.get_lock_guard().cf_handle(cf_name).is_some()
+    }
+
+    fn create_cf(&self, cf_name: &str) -> crate::Result<()> {
+        self.db
+            .write()
+            .expect("Couldn't get write lock to DB")
+            .create_cf(cf_name, &self.options.into())
+            .map_err(Into::into)
+    }
+
+    pub(super) fn get_lock_guard(&self) -> ShardedLockReadGuard<rocksdb::DB> {
+        self.db.read().expect("Couldn't get read lock to DB")
+    }
+
     fn do_merge(&self, patch: Patch, w_opts: &RocksDBWriteOptions) -> crate::Result<()> {
         let mut batch = WriteBatch::default();
         for (resolved, changes) in patch.into_changes() {
-            let cf = match self.db.cf_handle(&resolved.name) {
-                Some(cf) => cf,
-                None => self
-                    .db
-                    .create_cf(&resolved.name, &self.options.into())
-                    .unwrap(),
-            };
+            if !self.cf_exists(&resolved.name) {
+                self.create_cf(&resolved.name)?;
+            }
+
+            let db_reader = self.get_lock_guard();
+            let cf = db_reader.cf_handle(&resolved.name).unwrap();
 
             if changes.is_cleared() {
                 self.clear_prefix(&mut batch, cf, &resolved)?;
@@ -156,14 +167,16 @@ impl RocksDB {
             }
         }
 
-        self.db.write_opt(batch, w_opts).map_err(Into::into)
+        self.get_lock_guard()
+            .write_opt(batch, w_opts)
+            .map_err(Into::into)
     }
 
     /// Removes all keys with a specified prefix from a column family.
     fn clear_prefix(
         &self,
         batch: &mut WriteBatch,
-        cf: ColumnFamily<'_>,
+        cf: &ColumnFamily,
         resolved: &ResolvedAddress,
     ) -> crate::Result<()> {
         let snapshot = self.rocksdb_snapshot();
@@ -180,19 +193,24 @@ impl RocksDB {
         RocksDBSnapshot {
             // SAFETY:
             // The snapshot carries an `Arc` to the database to make sure that database
-            // is not dropped before the snapshot.
-            snapshot: unsafe { mem::transmute(self.db.snapshot()) },
+            // is not dropped before the snapshot. We don't care about changes in the database
+            // after taking a snapshot which could occur from another threads.
+            snapshot: unsafe { mem::transmute(self.get_lock_guard().snapshot()) },
             db: Arc::clone(&self.db),
         }
     }
 }
 
 impl RocksDBSnapshot {
+    fn get_lock_guard(&self) -> ShardedLockReadGuard<rocksdb::DB> {
+        self.db.read().expect("Couldn't get read lock to DB")
+    }
+
     fn rocksdb_iter(&self, name: &ResolvedAddress, from: &[u8]) -> RocksDBIterator<'_> {
         use rocksdb::{Direction, IteratorMode};
 
         let from = name.keyed(from);
-        let iter = match self.db.cf_handle(&name.name) {
+        let iter = match self.get_lock_guard().cf_handle(&name.name) {
             Some(cf) => self
                 .snapshot
                 .iterator_cf(cf, IteratorMode::From(from.as_ref(), Direction::Forward))
@@ -228,7 +246,7 @@ impl Database for RocksDB {
 
 impl Snapshot for RocksDBSnapshot {
     fn get(&self, resolved_addr: &ResolvedAddress, key: &[u8]) -> Option<Vec<u8>> {
-        if let Some(cf) = self.db.cf_handle(&resolved_addr.name) {
+        if let Some(cf) = self.get_lock_guard().cf_handle(&resolved_addr.name) {
             match self.snapshot.get_cf(cf, resolved_addr.keyed(key)) {
                 Ok(value) => value.map(|v| v.to_vec()),
                 Err(e) => panic!(e),

components/merkledb/src/backends/temporarydb.rs

@@ -55,13 +55,14 @@ impl TemporaryDB {
         // For some reason, using a `WriteBatch` is significantly faster than using `DB::drop_cf`,
         // both in debug and release modes.
         let mut batch = WriteBatch::default();
+        let db_reader = self.inner.get_lock_guard();
         for name in &names {
             if name != DEFAULT_CF && name != DB_METADATA {
-                let cf_handle = self.inner.rocksdb().cf_handle(name).ok_or_else(|| {
+                let cf_handle = db_reader.cf_handle(name).ok_or_else(|| {
                     let message = format!("Cannot access column family {}", name);
                     crate::Error::new(message)
                 })?;
-                let mut iter = self.inner.rocksdb().raw_iterator_cf(cf_handle.clone())?;
+                let mut iter = db_reader.raw_iterator_cf(cf_handle)?;
                 iter.seek_to_last();
                 if iter.valid() {
                     if let Some(key) = iter.key() {
@@ -73,7 +74,7 @@ impl TemporaryDB {
                         if key.len() < LARGER_KEY.len() {
                             batch.delete_range_cf::<&[u8]>(cf_handle, &[], LARGER_KEY)?;
                         } else {
-                            batch.delete_range_cf::<&[u8]>(cf_handle.clone(), &[], &key)?;
+                            batch.delete_range_cf::<&[u8]>(cf_handle, &[], &key)?;
                             batch.delete_cf(cf_handle, &key)?;
                         }
                     }
@@ -82,8 +83,7 @@ impl TemporaryDB {
         }
 
         let write_options = WriteOptions::default();
-        self.inner
-            .rocksdb()
+        db_reader
             .write_opt(batch, &write_options)
             .map_err(Into::into)
     }

components/merkledb/src/indexes/group.rs

@@ -1,3 +1,17 @@
+// Copyright 2020 The Exonum Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::marker::PhantomData;
 
 use crate::{

components/merkledb/src/indexes/proof_list/proof_builder.rs

@@ -1,3 +1,17 @@
+// Copyright 2020 The Exonum Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 //! Building `ListProof`s.
 
 use exonum_crypto::Hash;

components/merkledb/src/indexes/proof_map/proof_builder.rs

@@ -1,3 +1,17 @@
+// Copyright 2020 The Exonum Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 //! Building `MapProof`s. See README.md in the module directory for high-level explanation
 //! how the proofs are built.
 

components/merkledb/src/indexes/proof_map/tests.rs

@@ -1648,6 +1648,10 @@ fn test_tree_with_hashed_key() {
     }
 
     impl BinaryKey for Point {
+        fn size(&self) -> usize {
+            4
+        }
+
         fn write(&self, buffer: &mut [u8]) -> usize {
             LittleEndian::write_u16(&mut buffer[0..2], self.x);
             LittleEndian::write_u16(&mut buffer[2..4], self.y);
@@ -1659,10 +1663,6 @@ fn test_tree_with_hashed_key() {
             let y = LittleEndian::read_u16(&buffer[2..4]);
             Self { x, y }
         }
-
-        fn size(&self) -> usize {
-            4
-        }
     }
 
     impl BinaryValue for Point {

components/merkledb/src/lazy.rs

@@ -1,3 +1,17 @@
+// Copyright 2020 The Exonum Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::marker::PhantomData;
 
 use crate::{

components/merkledb/src/views/address.rs

@@ -1,3 +1,17 @@
+// Copyright 2020 The Exonum Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use std::{borrow::Cow, num::NonZeroU64};
 
 use crate::BinaryKey;

components/merkledb/src/views/system_schema.rs

@@ -1,3 +1,17 @@
+// Copyright 2020 The Exonum Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use exonum_crypto::Hash;
 
 use super::{AsReadonly, IndexType, RawAccess, ViewWithMetadata};