From efc4ab0d2b5d9babe0c3aeecd1623b4bf6be48c9 Mon Sep 17 00:00:00 2001 From: "Nathan.fooo" <86001920+appflowy@users.noreply.github.com> Date: Wed, 3 Jan 2024 08:05:17 +0800 Subject: [PATCH 01/13] feat: Support encode v2 (#142) * chore: support encoded v2 * chore: update --- Cargo.lock | 41 +++++++++++---------- collab-document/src/document_data.rs | 4 +- collab-folder/src/folder.rs | 4 +- collab-plugins/src/local_storage/rocksdb.rs | 16 +++----- collab/Cargo.toml | 1 + collab/src/core/collab.rs | 15 ++++++-- collab/src/core/collab_plugin.rs | 28 ++++++++++++-- collab/src/core/transaction.rs | 14 +++++-- 8 files changed, 78 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9083a49a5..a0b9dcb0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -75,7 +75,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.46", ] [[package]] @@ -129,7 +129,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.31", + "syn 2.0.46", ] [[package]] @@ -277,6 +277,7 @@ dependencies = [ "parking_lot", "serde", "serde_json", + "serde_repr", "tempfile", "thiserror", "tokio", @@ -549,7 +550,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.31", + "syn 2.0.46", ] [[package]] @@ -566,7 +567,7 @@ checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.46", ] [[package]] @@ -698,7 +699,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.46", ] [[package]] @@ -1148,7 +1149,7 @@ checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.46", ] [[package]] @@ -1182,7 +1183,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "617feabb81566b593beb4886fb8c1f38064169dae4dccad0e3220160c3b37203" dependencies = [ "proc-macro2", - "syn 2.0.31", + "syn 2.0.46", ] [[package]] @@ -1211,18 +1212,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "2de98502f212cfcea8d0bb305bd0f49d7ebdd75b64ba0a68f937d888f4e0d6db" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -1419,7 +1420,7 @@ checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.46", ] [[package]] @@ -1435,13 +1436,13 @@ dependencies = [ [[package]] name = "serde_repr" -version = "0.1.12" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcec881020c684085e55a25f7fd888954d56609ef363479dc5a1305eb0d40cab" +checksum = "0b2e6b945e9d3df726b65d6ee24060aff8e3533d431f677a9695db04eff9dfdb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.46", ] [[package]] @@ -1552,7 +1553,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.31", + "syn 2.0.46", ] [[package]] @@ -1574,9 +1575,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.31" +version = "2.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" +checksum = "89456b690ff72fddcecf231caedbe615c59480c93358a93dfae7fc29e3ebbf0e" dependencies = [ "proc-macro2", "quote", @@ -1657,7 +1658,7 @@ checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.46", ] [[package]] @@ -1713,7 +1714,7 @@ checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.46", ] [[package]] diff --git a/collab-document/src/document_data.rs b/collab-document/src/document_data.rs index a8cc12243..894ca6880 100644 --- a/collab-document/src/document_data.rs +++ b/collab-document/src/document_data.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use std::sync::Arc; use collab::core::collab::MutexCollab; -use collab::core::collab_plugin::EncodedCollabV1; +use collab::core::collab_plugin::EncodedCollab; use collab::core::origin::CollabOrigin; use nanoid::nanoid; @@ -85,7 +85,7 @@ pub fn default_document_data() -> DocumentData { /// Generates default collab data for a document. This document only contains the initial state /// of the document. -pub fn default_document_collab_data(document_id: &str) -> EncodedCollabV1 { +pub fn default_document_collab_data(document_id: &str) -> EncodedCollab { let document_data = default_document_data(); let collab = Arc::new(MutexCollab::new(CollabOrigin::Empty, document_id, vec![])); let _ = Document::create_with_data(collab.clone(), document_data); diff --git a/collab-folder/src/folder.rs b/collab-folder/src/folder.rs index 3b23bc04f..3f6803e47 100644 --- a/collab-folder/src/folder.rs +++ b/collab-folder/src/folder.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use anyhow::Error; use collab::core::collab::{CollabDocState, MutexCollab}; -use collab::core::collab_plugin::EncodedCollabV1; +use collab::core::collab_plugin::EncodedCollab; use collab::core::collab_state::{SnapshotState, SyncState}; pub use collab::core::origin::CollabOrigin; use collab::preclude::*; @@ -141,7 +141,7 @@ impl Folder { } /// Returns the doc state and the state vector. - pub fn encode_collab_v1(&self) -> EncodedCollabV1 { + pub fn encode_collab_v1(&self) -> EncodedCollab { self.inner.lock().encode_collab_v1() } diff --git a/collab-plugins/src/local_storage/rocksdb.rs b/collab-plugins/src/local_storage/rocksdb.rs index c4cdd9ce5..e1e0e414a 100644 --- a/collab-plugins/src/local_storage/rocksdb.rs +++ b/collab-plugins/src/local_storage/rocksdb.rs @@ -5,7 +5,7 @@ use std::sync::{Arc, Weak}; use collab::core::awareness::Awareness; use collab::core::collab::make_yrs_doc; -use collab::core::collab_plugin::EncodedCollabV1; +use collab::core::collab_plugin::EncodedCollab; use collab::core::origin::CollabOrigin; use collab::preclude::CollabPlugin; use collab_persistence::doc::YrsDocAction; @@ -17,9 +17,8 @@ use yrs::{Doc, ReadTxn, StateVector, Transact, TransactionMut}; use crate::local_storage::CollabPersistenceConfig; pub trait RocksdbBackup: Send + Sync { - fn save_doc(&self, uid: i64, object_id: &str, data: EncodedCollabV1) - -> Result<(), anyhow::Error>; - fn get_doc(&self, uid: i64, object_id: &str) -> Result; + fn save_doc(&self, uid: i64, object_id: &str, data: EncodedCollab) -> Result<(), anyhow::Error>; + fn get_doc(&self, uid: i64, object_id: &str) -> Result; } #[derive(Clone)] @@ -79,7 +78,7 @@ impl RocksdbDiskPlugin { if let Ok(read_txn) = doc.try_transact() { let doc_state = read_txn.encode_state_as_update_v1(&StateVector::default()); let state_vector = read_txn.state_vector().encode_v1(); - let encoded = EncodedCollabV1::new(state_vector, doc_state); + let encoded = EncodedCollab::new_v1(state_vector, doc_state); w_db_txn.flush_doc( self.uid, @@ -185,12 +184,7 @@ impl CollabPlugin for RocksdbDiskPlugin { } } -fn backup_doc( - uid: i64, - backup: &Arc, - object_id: &str, - encoded: EncodedCollabV1, -) { +fn backup_doc(uid: i64, backup: &Arc, object_id: &str, encoded: EncodedCollab) { let weak_backup = Arc::downgrade(backup); let object_id = object_id.to_string(); tokio::spawn(async move { diff --git a/collab/Cargo.toml b/collab/Cargo.toml index c8462cb56..25e4b1a5b 100644 --- a/collab/Cargo.toml +++ b/collab/Cargo.toml @@ -18,6 +18,7 @@ tokio = { version = "1.26", features = ["time", "sync", "rt"] } tokio-stream = { version = "0.1.14", features = ["sync"] } async-trait.workspace = true bincode = "1.3.3" +serde_repr = "0.1" [dev-dependencies] diff --git a/collab/src/core/collab.rs b/collab/src/core/collab.rs index 227117674..f8abd0e54 100644 --- a/collab/src/core/collab.rs +++ b/collab/src/core/collab.rs @@ -22,7 +22,7 @@ use yrs::{ }; use crate::core::awareness::Awareness; -use crate::core::collab_plugin::{CollabPlugin, CollabPluginType, EncodedCollabV1}; +use crate::core::collab_plugin::{CollabPlugin, CollabPluginType, EncodedCollab}; use crate::core::collab_state::{InitState, SnapshotState, State, SyncState}; use crate::core::map_wrapper::{CustomMapRef, MapRefWrapper}; use crate::core::origin::{CollabClient, CollabOrigin}; @@ -140,10 +140,14 @@ impl Collab { } /// Returns the doc state and the state vector. - pub fn encode_collab_v1(&self) -> EncodedCollabV1 { + pub fn encode_collab_v1(&self) -> EncodedCollab { self.doc.get_encoded_collab_v1() } + pub fn encode_collab_v2(&self) -> EncodedCollab { + self.doc.get_encoded_collab_v2() + } + pub fn subscribe_sync_state(&self) -> WatchStream { WatchStream::new(self.state.sync_state_notifier.subscribe()) } @@ -830,11 +834,16 @@ impl MutexCollab { } /// Returns the doc state and the state vector. - pub fn encode_collab_v1(&self) -> EncodedCollabV1 { + pub fn encode_collab_v1(&self) -> EncodedCollab { let collab = self.0.lock(); collab.encode_collab_v1() } + pub fn encode_collab_v2(&self) -> EncodedCollab { + let collab = self.0.lock(); + collab.encode_collab_v2() + } + pub fn to_json_value(&self) -> JsonValue { self.0.lock().to_json_value() } diff --git a/collab/src/core/collab_plugin.rs b/collab/src/core/collab_plugin.rs index 6b4f53901..fca2c5586 100644 --- a/collab/src/core/collab_plugin.rs +++ b/collab/src/core/collab_plugin.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use bytes::Bytes; use serde::{Deserialize, Serialize}; +use serde_repr::*; use yrs::{Doc, TransactionMut}; use crate::core::awareness::Awareness; @@ -106,16 +107,35 @@ where } #[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] -pub struct EncodedCollabV1 { +pub struct EncodedCollab { pub state_vector: Bytes, pub doc_state: Bytes, + #[serde(default)] + pub version: EncoderVersion, } -impl EncodedCollabV1 { - pub fn new>(state_vector: T, doc_state: T) -> Self { +#[derive(Default, Serialize_repr, Deserialize_repr, Eq, PartialEq, Debug, Clone)] +#[repr(u8)] +pub enum EncoderVersion { + #[default] + V1 = 0, + V2 = 1, +} + +impl EncodedCollab { + pub fn new_v1>(state_vector: T, doc_state: T) -> Self { + Self { + state_vector: state_vector.into(), + doc_state: doc_state.into(), + version: EncoderVersion::V1, + } + } + + pub fn new_v2>(state_vector: T, doc_state: T) -> Self { Self { state_vector: state_vector.into(), doc_state: doc_state.into(), + version: EncoderVersion::V2, } } @@ -123,7 +143,7 @@ impl EncodedCollabV1 { bincode::serialize(self) } - pub fn decode_from_bytes(encoded: &[u8]) -> Result { + pub fn decode_from_bytes(encoded: &[u8]) -> Result { bincode::deserialize(encoded) } } diff --git a/collab/src/core/transaction.rs b/collab/src/core/transaction.rs index b0b5e5cdc..c03aab815 100644 --- a/collab/src/core/transaction.rs +++ b/collab/src/core/transaction.rs @@ -1,7 +1,7 @@ use std::thread::sleep; use std::time::{Duration, Instant}; -use crate::core::collab_plugin::EncodedCollabV1; +use crate::core::collab_plugin::EncodedCollab; use yrs::updates::encoder::Encode; use yrs::{Doc, ReadTxn, StateVector, Transact, Transaction, TransactionMut}; @@ -97,13 +97,21 @@ pub trait DocTransactionExtension: Send + Sync { fn doc_transaction(&self) -> Transaction; fn doc_transaction_mut(&self) -> TransactionMut; - fn get_encoded_collab_v1(&self) -> EncodedCollabV1 { + fn get_encoded_collab_v1(&self) -> EncodedCollab { let txn = self.doc_transaction(); - EncodedCollabV1::new( + EncodedCollab::new_v1( txn.state_vector().encode_v1(), txn.encode_state_as_update_v1(&StateVector::default()), ) } + + fn get_encoded_collab_v2(&self) -> EncodedCollab { + let txn = self.doc_transaction(); + EncodedCollab::new_v2( + txn.state_vector().encode_v2(), + txn.encode_state_as_update_v2(&StateVector::default()), + ) + } } impl DocTransactionExtension for Doc { From b036feb396aa6a1dbbab9589bd032e29db8ab024 Mon Sep 17 00:00:00 2001 From: nathan Date: Wed, 3 Jan 2024 09:28:28 +0800 Subject: [PATCH 02/13] fix: decode EncodedCollab --- collab/src/core/collab_plugin.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/collab/src/core/collab_plugin.rs b/collab/src/core/collab_plugin.rs index fca2c5586..15c29970f 100644 --- a/collab/src/core/collab_plugin.rs +++ b/collab/src/core/collab_plugin.rs @@ -144,6 +144,26 @@ impl EncodedCollab { } pub fn decode_from_bytes(encoded: &[u8]) -> Result { - bincode::deserialize(encoded) + // The deserialize_encoded_collab function first tries to deserialize the data as EncodedCollab. + // If it fails (presumably because the data was serialized with EncodedCollabV0), it then tries to deserialize as EncodedCollabV0. + // After successfully deserializing as EncodedCollabV0, it constructs a new EncodedCollab object with the data from + // EncodedCollabV0 and sets the version to a default value. + match bincode::deserialize::(encoded) { + Ok(new_collab) => Ok(new_collab), + Err(_) => { + let old_collab: EncodedCollabV0 = bincode::deserialize(encoded)?; + Ok(EncodedCollab { + state_vector: old_collab.state_vector, + doc_state: old_collab.doc_state, + version: EncoderVersion::V1, + }) + }, + } } } + +#[derive(Serialize, Deserialize)] +pub struct EncodedCollabV0 { + pub state_vector: Bytes, + pub doc_state: Bytes, +} From 665c2652fb41ce5e5ec7365dc942ffb59034fcb1 Mon Sep 17 00:00:00 2001 From: nathan Date: Wed, 3 Jan 2024 09:58:27 +0800 Subject: [PATCH 03/13] chore: add test --- collab/src/core/collab_plugin.rs | 43 ++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/collab/src/core/collab_plugin.rs b/collab/src/core/collab_plugin.rs index 15c29970f..9d7aa0469 100644 --- a/collab/src/core/collab_plugin.rs +++ b/collab/src/core/collab_plugin.rs @@ -167,3 +167,46 @@ pub struct EncodedCollabV0 { pub state_vector: Bytes, pub doc_state: Bytes, } + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn old_encoded_collab_decoded_into_new_encoded_collab() { + let old_encoded_collab = EncodedCollabV0 { + state_vector: Bytes::from(vec![1, 2, 3]), + doc_state: Bytes::from(vec![4, 5, 6]), + }; + + let old_encoded_collab_bytes = bincode::serialize(&old_encoded_collab).unwrap(); + let new_encoded_collab = EncodedCollab::decode_from_bytes(&old_encoded_collab_bytes).unwrap(); + + assert_eq!( + new_encoded_collab, + EncodedCollab { + state_vector: Bytes::from(vec![1, 2, 3]), + doc_state: Bytes::from(vec![4, 5, 6]), + version: EncoderVersion::V1, + } + ); + } + + #[test] + fn new_encoded_collab_decoded_into_old_encoded_collab() { + let new_encoded_collab = EncodedCollab { + state_vector: Bytes::from(vec![1, 2, 3]), + doc_state: Bytes::from(vec![4, 5, 6]), + version: EncoderVersion::V1, + }; + + let new_encoded_collab_bytes = new_encoded_collab.encode_to_bytes().unwrap(); + let old_encoded_collab: EncodedCollabV0 = + bincode::deserialize(&new_encoded_collab_bytes).unwrap(); + + assert_eq!(old_encoded_collab.doc_state, new_encoded_collab.doc_state); + assert_eq!( + old_encoded_collab.state_vector, + new_encoded_collab.state_vector + ); + } +} From 5c425595182ef5a43913484a775fae6f59fe9e52 Mon Sep 17 00:00:00 2001 From: "Nathan.fooo" <86001920+appflowy@users.noreply.github.com> Date: Fri, 5 Jan 2024 00:04:31 +0800 Subject: [PATCH 04/13] chore: Wasm build (#143) * chore: support wasm build * chore: update * chore: remove collab-persistence * chore: remove collab-persistence * chore: remove collab-persistence * refactor: non wasm arch * ci: wasm build * fix: ci * fix: ci * fix: ci * refactor: reanme * refactor: rename * refactor: rename * refactor: rename * refactor: rename --- .github/workflows/ci.yml | 2 +- .github/workflows/wasm_build.yml | 35 ++++ .gitignore | 4 +- Cargo.lock | 191 ++---------------- Cargo.toml | 4 +- Makefile.toml | 15 ++ README.md | 8 +- collab-database/Cargo.toml | 13 +- collab-database/src/blocks/block.rs | 8 +- collab-database/src/blocks/task_controller.rs | 17 +- collab-database/src/database.rs | 6 +- collab-database/src/rows/row.rs | 13 +- collab-database/src/user/user_db.rs | 12 +- collab-database/src/views/view.rs | 2 +- collab-database/src/views/view_map.rs | 2 +- .../tests/database_test/field_setting_test.rs | 3 +- collab-database/tests/database_test/helper.rs | 15 +- .../tests/database_test/restore_test.rs | 6 +- collab-database/tests/helper/util.rs | 6 +- .../tests/user_test/async_test/script.rs | 10 +- collab-database/tests/user_test/helper.rs | 14 +- collab-document/Cargo.toml | 13 +- collab-document/src/lib.rs | 2 - .../tests/blocks/block_test_core.rs | 8 +- .../tests/document/restore_test.rs | 4 +- collab-document/tests/main.rs | 3 + collab-document/tests/util.rs | 14 +- collab-entity/Cargo.toml | 6 + collab-folder/Cargo.toml | 11 +- collab-folder/tests/folder_test/util.rs | 10 +- collab-folder/tests/folder_test/view_test.rs | 14 +- collab-persistence/Cargo.toml | 34 ---- collab-persistence/src/kv/mod.rs | 83 -------- collab-persistence/tests/persistence/main.rs | 4 - .../tests/persistence/rocksdb_cf_test.rs | 22 -- collab-persistence/tests/persistence/util.rs | 32 --- collab-plugins/Cargo.toml | 36 ++-- collab-plugins/src/cloud_storage/mod.rs | 2 - .../src/cloud_storage/postgres/plugin.rs | 14 +- .../network_state.rs => connect_state.rs} | 18 +- collab-plugins/src/lib.rs | 12 +- .../src/local_storage/kv}/db.rs | 87 +++++++- .../src/local_storage/kv}/doc.rs | 32 +-- .../src/local_storage/kv}/error.rs | 14 +- .../src/local_storage/kv}/keys.rs | 0 .../src/local_storage/kv/mod.rs | 1 - .../src/local_storage/kv}/oid.rs | 4 +- .../src/local_storage/kv}/range.rs | 0 .../src/local_storage/kv}/snapshot.rs | 22 +- collab-plugins/src/local_storage/mod.rs | 5 +- .../src/local_storage/rocksdb/kv_impl.rs | 87 ++------ .../src/local_storage/rocksdb/mod.rs | 3 + .../{rocksdb.rs => rocksdb/rocksdb_plugin.rs} | 14 +- .../rocksdb/snapshot_plugin.rs} | 26 +-- collab-plugins/src/snapshot/mod.rs | 4 - collab-plugins/tests/disk/mod.rs | 3 + .../tests/disk}/range_test.rs | 9 +- .../tests/disk}/restore_test.rs | 19 +- collab-plugins/tests/disk/script.rs | 16 +- collab-plugins/tests/disk/util.rs | 11 + collab-plugins/tests/main.rs | 12 +- collab-user/Cargo.toml | 11 +- collab-user/src/user_awareness.rs | 1 + collab-user/tests/util.rs | 6 +- collab/Cargo.toml | 8 +- collab/src/core/any_array.rs | 1 - collab/src/core/collab.rs | 4 + collab/tests/collab_test/helper.rs | 1 + rust-toolchain.toml | 2 +- 69 files changed, 438 insertions(+), 693 deletions(-) create mode 100644 .github/workflows/wasm_build.yml delete mode 100644 collab-persistence/Cargo.toml delete mode 100644 collab-persistence/src/kv/mod.rs delete mode 100644 collab-persistence/tests/persistence/main.rs delete mode 100644 collab-persistence/tests/persistence/rocksdb_cf_test.rs delete mode 100644 collab-persistence/tests/persistence/util.rs rename collab-plugins/src/{cloud_storage/network_state.rs => connect_state.rs} (55%) rename {collab-persistence/src => collab-plugins/src/local_storage/kv}/db.rs (59%) rename {collab-persistence/src => collab-plugins/src/local_storage/kv}/doc.rs (95%) rename {collab-persistence/src => collab-plugins/src/local_storage/kv}/error.rs (83%) rename {collab-persistence/src => collab-plugins/src/local_storage/kv}/keys.rs (100%) rename collab-persistence/src/lib.rs => collab-plugins/src/local_storage/kv/mod.rs (92%) rename {collab-persistence/src => collab-plugins/src/local_storage/kv}/oid.rs (97%) rename {collab-persistence/src => collab-plugins/src/local_storage/kv}/range.rs (100%) rename {collab-persistence/src => collab-plugins/src/local_storage/kv}/snapshot.rs (94%) rename collab-persistence/src/kv/rocks_kv.rs => collab-plugins/src/local_storage/rocksdb/kv_impl.rs (76%) create mode 100644 collab-plugins/src/local_storage/rocksdb/mod.rs rename collab-plugins/src/local_storage/{rocksdb.rs => rocksdb/rocksdb_plugin.rs} (94%) rename collab-plugins/src/{snapshot/plugin.rs => local_storage/rocksdb/snapshot_plugin.rs} (91%) delete mode 100644 collab-plugins/src/snapshot/mod.rs rename {collab-persistence/tests/persistence => collab-plugins/tests/disk}/range_test.rs (96%) rename {collab-persistence/tests/persistence => collab-plugins/tests/disk}/restore_test.rs (85%) create mode 100644 collab-plugins/tests/disk/util.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 540d7ce38..9b0106be6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ on: env: CARGO_TERM_COLOR: always - RUST_TOOLCHAIN: "1.70" + RUST_TOOLCHAIN: "1.75" jobs: fmt: diff --git a/.github/workflows/wasm_build.yml b/.github/workflows/wasm_build.yml new file mode 100644 index 000000000..f264c4cdc --- /dev/null +++ b/.github/workflows/wasm_build.yml @@ -0,0 +1,35 @@ +name: Collab-WASM + +on: + push: + branches: [ main ] + pull_request: + types: [ opened, synchronize, reopened ] + branches: [ main ] + +env: + RUST_TOOLCHAIN: "1.75" + CARGO_MAKE_VERSION: "0.36.6" + +jobs: + wasm_build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + + - name: Install wasm-pack + run: cargo install wasm-pack + + - uses: taiki-e/install-action@v2 + with: + tool: cargo-make@${{ env.CARGO_MAKE_VERSION }} + + - name: Build + run: cargo make wasm_build + diff --git a/.gitignore b/.gitignore index fc74619eb..7b965991e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,6 @@ .idea **/temp/** -**/collab-plugins/.env -**/collab-plugins/.env.test +collab-plugins/.env +collab-plugins/.env.test **/unit_test** \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index a0b9dcb0a..e3b72da2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -90,12 +90,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "base64" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" - [[package]] name = "base64ct" version = "1.6.0" @@ -294,13 +288,12 @@ dependencies = [ "anyhow", "assert-json-diff", "async-trait", - "base64", "chrono", "collab", "collab-entity", - "collab-persistence", "collab-plugins", "futures", + "getrandom 0.2.9", "lazy_static", "lru", "nanoid", @@ -327,9 +320,9 @@ version = "0.1.0" dependencies = [ "anyhow", "collab", - "collab-persistence", "collab-plugins", "futures", + "getrandom 0.2.9", "nanoid", "parking_lot", "serde", @@ -350,6 +343,7 @@ dependencies = [ "anyhow", "bytes", "collab", + "getrandom 0.2.9", "serde", "serde_json", "serde_repr", @@ -364,7 +358,6 @@ dependencies = [ "assert-json-diff", "chrono", "collab", - "collab-persistence", "collab-plugins", "fs_extra", "nanoid", @@ -382,31 +375,6 @@ dependencies = [ "zip", ] -[[package]] -name = "collab-persistence" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "bincode", - "chrono", - "collab", - "collab-persistence", - "futures", - "lazy_static", - "parking_lot", - "rocksdb", - "serde", - "smallvec", - "tempfile", - "test-case", - "thiserror", - "tokio", - "tracing", - "tracing-subscriber", - "yrs", -] - [[package]] name = "collab-plugins" version = "0.1.0" @@ -414,23 +382,24 @@ dependencies = [ "anyhow", "assert-json-diff", "async-trait", + "bincode", "bytes", "chrono", "collab", "collab-entity", - "collab-persistence", - "collab-plugins", "config", - "dashmap", "dotenv", "futures", "futures-util", - "nanoid", + "getrandom 0.2.9", + "lazy_static", "parking_lot", "rand 0.8.5", + "rocksdb", "serde", "serde_json", "similar", + "smallvec", "tempfile", "thiserror", "tokio", @@ -453,6 +422,7 @@ dependencies = [ "collab-entity", "collab-plugins", "fs_extra", + "getrandom 0.2.9", "nanoid", "parking_lot", "serde", @@ -465,9 +435,9 @@ dependencies = [ [[package]] name = "config" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d379af7f68bfc21714c6c7dea883544201741d2ce8274bb12fa54f89507f52a7" +checksum = "23738e11972c7643e4ec947840fc463b6a571afcd3e735bdfce7d03c7a784aca" dependencies = [ "async-trait", "lazy_static", @@ -570,19 +540,6 @@ dependencies = [ "syn 2.0.46", ] -[[package]] -name = "dashmap" -version = "5.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" -dependencies = [ - "cfg-if", - "hashbrown 0.12.3", - "lock_api", - "once_cell", - "parking_lot_core", -] - [[package]] name = "digest" version = "0.10.7" @@ -762,8 +719,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -772,12 +731,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.14.2" @@ -794,15 +747,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -[[package]] -name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - [[package]] name = "hmac" version = "0.12.1" @@ -969,7 +913,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efa59af2ddfad1854ae27d75009d538d0998b4b2fd47083e743ac1a10e46c60" dependencies = [ - "hashbrown 0.14.2", + "hashbrown", ] [[package]] @@ -1002,18 +946,6 @@ dependencies = [ "adler", ] -[[package]] -name = "mio" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" -dependencies = [ - "libc", - "log", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.45.0", -] - [[package]] name = "nanoid" version = "0.4.0" @@ -1052,16 +984,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "num_cpus" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" -dependencies = [ - "hermit-abi", - "libc", -] - [[package]] name = "once_cell" version = "1.18.0" @@ -1186,30 +1108,6 @@ dependencies = [ "syn 2.0.46", ] -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - [[package]] name = "proc-macro2" version = "1.0.74" @@ -1488,15 +1386,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" -[[package]] -name = "signal-hook-registry" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" -dependencies = [ - "libc", -] - [[package]] name = "similar" version = "2.2.1" @@ -1527,16 +1416,6 @@ version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" -[[package]] -name = "socket2" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "strum" version = "0.25.0" @@ -1606,41 +1485,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "test-case" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a1d6e7bde536b0412f20765b76e921028059adfd1b90d8974d33fd3c91b25df" -dependencies = [ - "test-case-macros", -] - -[[package]] -name = "test-case-core" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d10394d5d1e27794f772b6fc854c7e91a2dc26e2cbf807ad523370c2a59c0cee" -dependencies = [ - "cfg-if", - "proc-macro-error", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "test-case-macros" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeb9a44b1c6a54c1ba58b152797739dba2a83ca74e18168a68c980eb142f9404" -dependencies = [ - "proc-macro-error", - "proc-macro2", - "quote", - "syn 1.0.109", - "test-case-core", -] - [[package]] name = "thiserror" version = "1.0.49" @@ -1694,14 +1538,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" dependencies = [ "autocfg", - "bytes", - "libc", - "mio", - "num_cpus", - "parking_lot", "pin-project-lite", - "signal-hook-registry", - "socket2", "tokio-macros", "windows-sys 0.45.0", ] diff --git a/Cargo.toml b/Cargo.toml index 1c086bb07..30295e66a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,18 +1,16 @@ [workspace] members = [ "collab", - "collab-persistence", "collab-database", - "collab-plugins", "collab-user", "collab-entity", "collab-document", "collab-folder", + "collab-plugins", ] [workspace.dependencies] collab = { workspace = true, path = "collab" } -collab-persistence = { workspace = true, path = "collab-persistence" } collab-database = { workspace = true, path = "collab-database" } collab-plugins = { workspace = true, path = "collab-plugins" } collab-user = { workspace = true, path = "collab-user" } diff --git a/Makefile.toml b/Makefile.toml index d244bb611..3ec9e139d 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -75,3 +75,18 @@ run_task = { name = ["clean_profraw_files"] } [env] CARGO_MAKE_EXTEND_WORKSPACE_MAKEFILE = true + + +[tasks.wasm_build] +script_runner = "bash" +script = [ +""" +crates=("collab" "collab-document" "collab-folder" "collab-user") + cd .. + for crate in "${crates[@]}"; do + echo "🔥🔥🔥 Building $crate with wasm-pack..." + cd ./$crate && wasm-pack build --features="wasm_build" || exit 1 + cd .. || exit 1 + done +""" +] diff --git a/README.md b/README.md index b2658a555..22a176fc4 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,6 @@ * `collab-database` * `collab-document` * `collab-folder` -* `collab-persistence` * `collab-plugins` * `collab-sync` @@ -35,12 +34,7 @@ The `collab-folder` crate provides a simple API for creating and managing collab the `collab` crate. ## collab-plugins -The `collab-plugins` crate contains a list of plugins that can be used with the `collab` crate. Currently, it includes -two plugins: the `disk plugin` that uses `collab-persistence` to persist the collaborative documents to disk, and the -`sync plugin` that uses `collab-sync` to sync the collaborative documents to a remote server. - -## collab-persistence -The `collab-persistence` crate uses [rocksdb](https://docs.rs/rocksdb/latest/rocksdb/) or [sled](https://github.com/spacejam/sled) to implement a persistence layer for the `collab` crate. It is easy to extend to support other key/value storage. +The `collab-plugins` crate contains a list of plugins that can be used with the `collab` crate. ## collab-sync The `collab-sync` crate supports syncing the collaborative documents to a remote server. \ No newline at end of file diff --git a/collab-database/Cargo.toml b/collab-database/Cargo.toml index 80086f466..d430ffd16 100644 --- a/collab-database/Cargo.toml +++ b/collab-database/Cargo.toml @@ -4,10 +4,11 @@ version = "0.1.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +crate-type = ["cdylib", "rlib"] [dependencies] collab = { workspace = true } -collab-persistence = { workspace = true } collab-entity = { workspace = true } collab-plugins = { workspace = true } serde = { workspace = true, features = ["derive", "rc"] } @@ -24,18 +25,22 @@ lazy_static = "1.4.0" lru = "0.12.0" async-trait.workspace = true uuid = { version = "1.3.3", features = ["v4", "v5"] } -base64 = "^0.21" tokio-stream = { version = "0.1.14", features = ["sync"] } strum = "0.25" strum_macros = "0.25" +getrandom = { version = "0.2", optional = true } [dev-dependencies] -collab-plugins = { workspace = true, features = ["rocksdb_plugin"] } +collab-plugins = { workspace = true } tempfile = "3.8.0" -tokio = { version = "1.26", features = ["full"] } +tokio = { version = "1.26", features = ["macros"] } assert-json-diff = "2.0.2" lazy_static = "1.4.0" tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } rand = "0.8.4" futures = "0.3.18" zip = "0.6.6" + +[features] +wasm_build = ["getrandom/js"] + diff --git a/collab-database/src/blocks/block.rs b/collab-database/src/blocks/block.rs index f1f8274df..7463bfa12 100644 --- a/collab-database/src/blocks/block.rs +++ b/collab-database/src/blocks/block.rs @@ -5,9 +5,9 @@ use std::sync::{Arc, Weak}; use collab::core::collab::{CollabDocState, MutexCollab}; use collab_entity::CollabType; -use collab_persistence::doc::YrsDocAction; -use collab_persistence::kv::rocks_kv::RocksCollabDB; +use collab_plugins::local_storage::kv::doc::CollabKVAction; use collab_plugins::local_storage::CollabPersistenceConfig; +use collab_plugins::CollabKVDB; use lru::LruCache; use parking_lot::Mutex; use tokio::sync::broadcast; @@ -33,7 +33,7 @@ pub enum BlockEvent { #[derive(Clone)] pub struct Block { uid: i64, - collab_db: Weak, + collab_db: Weak, collab_service: Arc, task_controller: Arc, sequence: Arc, @@ -45,7 +45,7 @@ pub struct Block { impl Block { pub fn new( uid: i64, - collab_db: Weak, + collab_db: Weak, collab_service: Arc, row_change_tx: Option, ) -> Block { diff --git a/collab-database/src/blocks/task_controller.rs b/collab-database/src/blocks/task_controller.rs index 56977f75c..b5cd7cc80 100644 --- a/collab-database/src/blocks/task_controller.rs +++ b/collab-database/src/blocks/task_controller.rs @@ -8,9 +8,9 @@ use async_trait::async_trait; use collab::core::collab::{CollabDocState, MutexCollab}; use collab::core::origin::CollabOrigin; use collab_entity::CollabType; -use collab_persistence::doc::YrsDocAction; -use collab_persistence::kv::rocks_kv::RocksCollabDB; -use collab_persistence::PersistenceError; +use collab_plugins::local_storage::kv::doc::CollabKVAction; +use collab_plugins::local_storage::kv::PersistenceError; +use collab_plugins::CollabKVDB; use tokio::sync::watch; use crate::blocks::queue::{ @@ -28,10 +28,7 @@ pub struct BlockTaskController { } impl BlockTaskController { - pub fn new( - collab_db: Weak, - collab_service: Weak, - ) -> Self { + pub fn new(collab_db: Weak, collab_service: Weak) -> Self { let (runner_notifier_tx, runner_notifier) = watch::channel(false); let task_handler = Arc::new(BlockTaskHandler::new( collab_service, @@ -60,7 +57,7 @@ impl BlockTaskController { } pub struct BlockTaskHandler { - collab_db: Weak, + collab_db: Weak, collab_service: Weak, queue: parking_lot::Mutex>, runner_notifier: Arc>, @@ -69,7 +66,7 @@ pub struct BlockTaskHandler { impl BlockTaskHandler { pub fn new( collab_service: Weak, - collab_db: Weak, + collab_db: Weak, runner_notifier: watch::Sender, ) -> Self { let queue = parking_lot::Mutex::new(TaskQueue::new()); @@ -159,7 +156,7 @@ impl TaskHandler for BlockTaskHandler { } fn save_row>( - collab_db: &Arc, + collab_db: &Arc, collab_doc_state: CollabDocState, uid: i64, row_id: R, diff --git a/collab-database/src/database.rs b/collab-database/src/database.rs index b25a16aa5..73fc81e3f 100644 --- a/collab-database/src/database.rs +++ b/collab-database/src/database.rs @@ -9,8 +9,7 @@ use collab::core::collab_state::{SnapshotState, SyncState}; use collab::preclude::{ Collab, JsonValue, MapRefExtension, MapRefWrapper, ReadTxn, TransactionMut, }; -pub use collab_persistence::doc::YrsDocAction; -use collab_persistence::kv::rocks_kv::RocksCollabDB; +use collab_plugins::CollabKVDB; use nanoid::nanoid; use parking_lot::Mutex; use serde::{Deserialize, Serialize}; @@ -55,7 +54,7 @@ const METAS: &str = "metas"; pub struct DatabaseContext { pub uid: i64, - pub db: Weak, + pub db: Weak, pub collab: Arc, pub collab_service: Arc, pub notifier: Option, @@ -1259,6 +1258,7 @@ impl DatabaseData { pub struct MutexDatabase(Arc>); impl MutexDatabase { + #[allow(clippy::arc_with_non_send_sync)] pub fn new(inner: Database) -> Self { Self(Arc::new(Mutex::new(inner))) } diff --git a/collab-database/src/rows/row.rs b/collab-database/src/rows/row.rs index 55a6429c8..1659ef87e 100644 --- a/collab-database/src/rows/row.rs +++ b/collab-database/src/rows/row.rs @@ -6,11 +6,11 @@ use collab::preclude::{ Any, ArrayRefWrapper, Collab, DeepEventsSubscription, Map, MapPrelim, MapRef, MapRefExtension, MapRefWrapper, ReadTxn, Transaction, TransactionMut, YrsValue, }; -use collab_persistence::doc::YrsDocAction; -use collab_persistence::kv::rocks_kv::RocksCollabDB; use parking_lot::Mutex; use collab::core::value::YrsValueExtension; +use collab_plugins::local_storage::kv::doc::CollabKVAction; +use collab_plugins::CollabKVDB; use serde::{Deserialize, Serialize}; use tracing::error; use uuid::Uuid; @@ -41,7 +41,7 @@ pub struct DatabaseRow { meta: MapRefWrapper, #[allow(dead_code)] comments: ArrayRefWrapper, - collab_db: Weak, + collab_db: Weak, #[allow(dead_code)] subscription: Option, } @@ -51,7 +51,7 @@ impl DatabaseRow { row: T, uid: i64, row_id: RowId, - collab_db: Weak, + collab_db: Weak, collab: Arc, change_tx: Option, ) -> Self { @@ -82,7 +82,7 @@ impl DatabaseRow { pub fn new( uid: i64, row_id: RowId, - collab_db: Weak, + collab_db: Weak, collab: Arc, change_tx: Option, ) -> Self { @@ -94,7 +94,7 @@ impl DatabaseRow { fn inner_new( uid: i64, row_id: RowId, - collab_db: Weak, + collab_db: Weak, collab: Arc, ) -> Self { let collab_guard = collab.lock(); @@ -586,6 +586,7 @@ pub struct MutexDatabaseRow(Arc>); impl MutexDatabaseRow { pub fn new(inner: DatabaseRow) -> Self { + #[allow(clippy::arc_with_non_send_sync)] Self(Arc::new(Mutex::new(inner))) } } diff --git a/collab-database/src/user/user_db.rs b/collab-database/src/user/user_db.rs index 8572b0a79..fb87389e1 100644 --- a/collab-database/src/user/user_db.rs +++ b/collab-database/src/user/user_db.rs @@ -10,10 +10,10 @@ use collab::core::collab::{CollabDocState, MutexCollab}; use collab::preclude::updates::decoder::Decode; use collab::preclude::{Collab, Update}; use collab_entity::CollabType; -use collab_persistence::doc::YrsDocAction; -use collab_persistence::kv::rocks_kv::RocksCollabDB; -use collab_persistence::snapshot::{CollabSnapshot, SnapshotAction}; +use collab_plugins::local_storage::kv::doc::CollabKVAction; +use collab_plugins::local_storage::kv::snapshot::{CollabSnapshot, SnapshotAction}; use collab_plugins::local_storage::CollabPersistenceConfig; +use collab_plugins::CollabKVDB; use parking_lot::Mutex; use crate::database::{Database, DatabaseContext, DatabaseData, MutexDatabase}; @@ -48,7 +48,7 @@ pub trait DatabaseCollabService: Send + Sync + 'static { uid: i64, object_id: &str, object_type: CollabType, - collab_db: Weak, + collab_db: Weak, collab_doc_state: CollabDocState, config: &CollabPersistenceConfig, ) -> Arc; @@ -58,7 +58,7 @@ pub trait DatabaseCollabService: Send + Sync + 'static { pub struct WorkspaceDatabase { uid: i64, collab: Arc, - collab_db: Weak, + collab_db: Weak, config: CollabPersistenceConfig, collab_service: Arc, /// In memory database handlers. @@ -71,7 +71,7 @@ impl WorkspaceDatabase { pub fn open( uid: i64, collab: Arc, - collab_db: Weak, + collab_db: Weak, config: CollabPersistenceConfig, collab_service: T, ) -> Self diff --git a/collab-database/src/views/view.rs b/collab-database/src/views/view.rs index 619d99020..08e67dc06 100644 --- a/collab-database/src/views/view.rs +++ b/collab-database/src/views/view.rs @@ -532,7 +532,7 @@ pub fn view_from_map_ref(map_ref: &MapRef, txn: &T) -> Option, + collab_db: Arc, database: Database, } @@ -80,7 +79,7 @@ pub async fn create_database(uid: i64, database_id: &str) -> DatabaseTest { pub async fn create_database_with_db( uid: i64, database_id: &str, -) -> (Arc, DatabaseTest) { +) -> (Arc, DatabaseTest) { setup_log(); let collab_db = make_rocks_db(); let collab_builder = Arc::new(TestUserDatabaseCollabBuilderImpl()); @@ -117,7 +116,7 @@ pub async fn create_database_with_db( pub fn restore_database_from_db( uid: i64, database_id: &str, - collab_db: Arc, + collab_db: Arc, ) -> DatabaseTest { let collab_builder = Arc::new(TestUserDatabaseCollabBuilderImpl()); let collab = collab_builder.build_collab_with_config( @@ -190,7 +189,7 @@ impl DatabaseTestBuilder { pub async fn build(self) -> DatabaseTest { let tempdir = TempDir::new().unwrap(); let path = tempdir.into_path(); - let collab_db = Arc::new(RocksCollabDB::open_opt(path, false).unwrap()); + let collab_db = Arc::new(CollabKVDB::open_opt(path, false).unwrap()); let collab = CollabBuilder::new(self.uid, &self.database_id) .with_device_id("1") .build() diff --git a/collab-database/tests/database_test/restore_test.rs b/collab-database/tests/database_test/restore_test.rs index 1ce6ffe64..8b796575b 100644 --- a/collab-database/tests/database_test/restore_test.rs +++ b/collab-database/tests/database_test/restore_test.rs @@ -2,10 +2,10 @@ use std::sync::Arc; use collab_database::database::DatabaseData; use collab_database::rows::CreateRowParams; -use collab_persistence::kv::rocks_kv::RocksCollabDB; use serde_json::{json, Value}; use assert_json_diff::assert_json_include; +use collab_plugins::CollabKVDB; use crate::database_test::helper::{ create_database_with_db, restore_database_from_db, DatabaseTest, @@ -100,7 +100,7 @@ async fn restore_from_disk_with_different_uid_test() { ); } -async fn create_database_with_view() -> (Arc, DatabaseTest, Value) { +async fn create_database_with_view() -> (Arc, DatabaseTest, Value) { let (db, database_test) = create_database_with_db(1, "1").await; let expected = json!({ "fields": [], @@ -127,7 +127,7 @@ const HISTORY_DOCUMENT_020: &str = "020_database"; #[tokio::test] async fn open_020_history_database_test() { let (_cleaner, db_path) = unzip_history_database_db(HISTORY_DOCUMENT_020).unwrap(); - let db = std::sync::Arc::new(RocksCollabDB::open_opt(db_path, false).unwrap()); + let db = std::sync::Arc::new(CollabKVDB::open_opt(db_path, false).unwrap()); let database_test = restore_database_from_db( 221439819971039232, "c0e69740-49f0-4790-a488-702e2750ba8d", diff --git a/collab-database/tests/helper/util.rs b/collab-database/tests/helper/util.rs index ad805ddcd..9742a1fc1 100644 --- a/collab-database/tests/helper/util.rs +++ b/collab-database/tests/helper/util.rs @@ -15,7 +15,7 @@ use collab_database::views::{ GroupMapBuilder, GroupSettingBuilder, GroupSettingMap, LayoutSetting, LayoutSettingBuilder, SortMap, SortMapBuilder, }; -use collab_persistence::kv::rocks_kv::RocksCollabDB; +use collab_plugins::CollabKVDB; use nanoid::nanoid; use tempfile::TempDir; use tracing_subscriber::fmt::Subscriber; @@ -584,10 +584,10 @@ impl From for AnyMap { } } -pub fn make_rocks_db() -> Arc { +pub fn make_rocks_db() -> Arc { let tempdir = TempDir::new().unwrap(); let path = tempdir.into_path(); - Arc::new(RocksCollabDB::open_opt(path, false).unwrap()) + Arc::new(CollabKVDB::open_opt(path, false).unwrap()) } pub fn setup_log() { diff --git a/collab-database/tests/user_test/async_test/script.rs b/collab-database/tests/user_test/async_test/script.rs index 6a8b2e06a..8dfc574f4 100644 --- a/collab-database/tests/user_test/async_test/script.rs +++ b/collab-database/tests/user_test/async_test/script.rs @@ -9,9 +9,9 @@ use collab_database::rows::CreateRowParams; use collab_database::rows::{Cells, CellsBuilder, RowId}; use collab_database::user::WorkspaceDatabase; use collab_database::views::{CreateDatabaseParams, OrderObjectPosition}; -use collab_persistence::doc::YrsDocAction; -use collab_persistence::kv::rocks_kv::RocksCollabDB; +use collab_plugins::local_storage::kv::doc::CollabKVAction; use collab_plugins::local_storage::CollabPersistenceConfig; +use collab_plugins::CollabKVDB; use serde_json::Value; use tempfile::TempDir; @@ -58,7 +58,7 @@ pub enum DatabaseScript { #[derive(Clone)] pub struct DatabaseTest { - pub collab_db: Arc, + pub collab_db: Arc, pub db_path: PathBuf, pub workspace_database: Arc, pub config: CollabPersistenceConfig, @@ -72,7 +72,7 @@ impl DatabaseTest { pub async fn new(config: CollabPersistenceConfig) -> Self { let tempdir = TempDir::new().unwrap(); let db_path = tempdir.into_path(); - let collab_db = Arc::new(RocksCollabDB::open_opt(db_path.clone(), false).unwrap()); + let collab_db = Arc::new(CollabKVDB::open_opt(db_path.clone(), false).unwrap()); let workspace_database = workspace_database_with_db(1, Arc::downgrade(&collab_db), Some(config.clone())).await; Self { @@ -113,7 +113,7 @@ impl DatabaseTest { pub async fn run_script( workspace_database: Arc, - db: Arc, + db: Arc, config: CollabPersistenceConfig, script: DatabaseScript, ) { diff --git a/collab-database/tests/user_test/helper.rs b/collab-database/tests/user_test/helper.rs index 6ac47922a..1526c1529 100644 --- a/collab-database/tests/user_test/helper.rs +++ b/collab-database/tests/user_test/helper.rs @@ -17,12 +17,12 @@ use collab_database::user::{ }; use collab_database::views::{CreateDatabaseParams, DatabaseLayout, OrderObjectPosition}; use collab_entity::CollabType; -use collab_persistence::kv::rocks_kv::RocksCollabDB; -use collab_plugins::local_storage::rocksdb::RocksdbDiskPlugin; use collab_plugins::local_storage::CollabPersistenceConfig; use parking_lot::Mutex; use tokio::sync::mpsc::{channel, Receiver}; +use collab_plugins::local_storage::rocksdb::rocksdb_plugin::RocksdbDiskPlugin; +use collab_plugins::CollabKVDB; use rand::Rng; use tempfile::TempDir; @@ -33,7 +33,7 @@ pub struct WorkspaceDatabaseTest { #[allow(dead_code)] uid: i64, inner: WorkspaceDatabase, - pub collab_db: Arc, + pub collab_db: Arc, } impl Deref for WorkspaceDatabaseTest { @@ -74,7 +74,7 @@ impl DatabaseCollabService for TestUserDatabaseCollabBuilderImpl { uid: i64, object_id: &str, _object_type: CollabType, - collab_db: Weak, + collab_db: Weak, doc_state: CollabDocState, config: &CollabPersistenceConfig, ) -> Arc { @@ -126,7 +126,7 @@ pub async fn workspace_database_test_with_config( pub async fn workspace_database_with_db( uid: i64, - collab_db: Weak, + collab_db: Weak, config: Option, ) -> WorkspaceDatabase { let config = config.unwrap_or_else(|| CollabPersistenceConfig::new().snapshot_per_update(5)); @@ -147,7 +147,7 @@ pub async fn workspace_database_with_db( pub async fn user_database_test_with_db( uid: i64, - collab_db: Arc, + collab_db: Arc, ) -> WorkspaceDatabaseTest { let inner = workspace_database_with_db(uid, Arc::downgrade(&collab_db), None).await; WorkspaceDatabaseTest { @@ -160,7 +160,7 @@ pub async fn user_database_test_with_db( pub async fn user_database_test_with_default_data(uid: i64) -> WorkspaceDatabaseTest { let tempdir = TempDir::new().unwrap(); let path = tempdir.into_path(); - let db = Arc::new(RocksCollabDB::open_opt(path, false).unwrap()); + let db = Arc::new(CollabKVDB::open_opt(path, false).unwrap()); let w_database = user_database_test_with_db(uid, db).await; w_database diff --git a/collab-document/Cargo.toml b/collab-document/Cargo.toml index b82172a37..a2ab9211f 100644 --- a/collab-document/Cargo.toml +++ b/collab-document/Cargo.toml @@ -4,13 +4,15 @@ version = "0.1.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +crate-type = ["cdylib", "rlib"] [dependencies] collab = { workspace = true } -collab-persistence = { workspace = true } serde.workspace = true serde_json.workspace = true nanoid = "0.4.0" +getrandom = { version = "0.2", optional = true } thiserror = "1.0.30" anyhow.workspace = true parking_lot.workspace = true @@ -18,10 +20,15 @@ tracing.workspace = true tokio = { version = "1.26", features = ["time", "sync", "rt"] } tokio-stream = { version = "0.1.14", features = ["sync"] } + [dev-dependencies] -tokio = { version = "1.26", features = ["full"] } +tokio = { version = "1.26", features = ["rt"] } tempfile = "3.8.0" tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } -collab-plugins = { workspace = true, features = ["rocksdb_plugin"] } +collab-plugins = { workspace = true } zip = "0.6.6" futures = "0.3.17" + + +[features] +wasm_build = ["getrandom/js"] \ No newline at end of file diff --git a/collab-document/src/lib.rs b/collab-document/src/lib.rs index 3999b6ecf..e07d27309 100644 --- a/collab-document/src/lib.rs +++ b/collab-document/src/lib.rs @@ -1,5 +1,3 @@ -pub use collab_persistence::doc::YrsDocAction; - pub mod blocks; pub mod document; pub mod document_data; diff --git a/collab-document/tests/blocks/block_test_core.rs b/collab-document/tests/blocks/block_test_core.rs index b0eafb932..d9582f7c5 100644 --- a/collab-document/tests/blocks/block_test_core.rs +++ b/collab-document/tests/blocks/block_test_core.rs @@ -7,8 +7,8 @@ use collab_document::blocks::{ Block, BlockAction, BlockActionPayload, BlockActionType, BlockEvent, DocumentData, DocumentMeta, }; use collab_document::document::Document; -use collab_persistence::kv::rocks_kv::RocksCollabDB; -use collab_plugins::local_storage::rocksdb::RocksdbDiskPlugin; +use collab_plugins::local_storage::rocksdb::rocksdb_plugin::RocksdbDiskPlugin; +use collab_plugins::CollabKVDB; use nanoid::nanoid; use serde_json::{json, Value}; @@ -17,7 +17,7 @@ use crate::util::document_storage; pub const TEXT_BLOCK_TYPE: &str = "paragraph"; pub struct BlockTestCore { - pub db: Arc, + pub db: Arc, pub document: Document, pub collab: Arc, } @@ -47,7 +47,7 @@ impl BlockTestCore { } } - pub fn open(collab: Arc, db: Arc) -> Self { + pub fn open(collab: Arc, db: Arc) -> Self { let open_res = Document::open(collab.clone()); open_res .map(|document| BlockTestCore { diff --git a/collab-document/tests/document/restore_test.rs b/collab-document/tests/document/restore_test.rs index 24556ef4b..01cc52085 100644 --- a/collab-document/tests/document/restore_test.rs +++ b/collab-document/tests/document/restore_test.rs @@ -1,6 +1,6 @@ use collab_document::blocks::Block; -use collab_persistence::kv::rocks_kv::RocksCollabDB; +use collab_plugins::CollabKVDB; use futures::executor::block_on; use crate::util::{ @@ -94,7 +94,7 @@ const HISTORY_DOCUMENT_020: &str = "020_document"; #[tokio::test] async fn open_020_history_document_test() { let (_cleaner, db_path) = unzip_history_document_db(HISTORY_DOCUMENT_020).unwrap(); - let db = std::sync::Arc::new(RocksCollabDB::open_opt(db_path, false).unwrap()); + let db = std::sync::Arc::new(CollabKVDB::open_opt(db_path, false).unwrap()); let document = open_document_with_db( 221439819971039232, "631584ec-af71-42c3-94f4-89dcfdafb988", diff --git a/collab-document/tests/main.rs b/collab-document/tests/main.rs index 0b525359c..73ac81e56 100644 --- a/collab-document/tests/main.rs +++ b/collab-document/tests/main.rs @@ -1,3 +1,6 @@ +#[cfg(not(target_arch = "wasm32"))] mod blocks; +#[cfg(not(target_arch = "wasm32"))] mod document; +#[cfg(not(target_arch = "wasm32"))] mod util; diff --git a/collab-document/tests/util.rs b/collab-document/tests/util.rs index 2da6bf21f..94585dc69 100644 --- a/collab-document/tests/util.rs +++ b/collab-document/tests/util.rs @@ -11,8 +11,8 @@ use collab::preclude::CollabBuilder; use collab_document::blocks::{Block, BlockAction, DocumentData, DocumentMeta}; use collab_document::document::Document; use collab_document::error::DocumentError; -use collab_persistence::kv::rocks_kv::RocksCollabDB; -use collab_plugins::local_storage::rocksdb::RocksdbDiskPlugin; +use collab_plugins::local_storage::rocksdb::rocksdb_plugin::RocksdbDiskPlugin; +use collab_plugins::CollabKVDB; use nanoid::nanoid; use serde_json::{json, Value}; use tempfile::TempDir; @@ -21,7 +21,7 @@ use zip::ZipArchive; pub struct DocumentTest { pub document: Document, - pub db: Arc, + pub db: Arc, } impl DocumentTest { @@ -30,7 +30,7 @@ impl DocumentTest { Self::new_with_db(uid, doc_id, db).await } - pub async fn new_with_db(uid: i64, doc_id: &str, db: Arc) -> Self { + pub async fn new_with_db(uid: i64, doc_id: &str, db: Arc) -> Self { let disk_plugin = RocksdbDiskPlugin::new(uid, Arc::downgrade(&db), None); let collab = CollabBuilder::new(1, doc_id) .with_plugin(disk_plugin) @@ -101,7 +101,7 @@ impl Deref for DocumentTest { } } -pub async fn open_document_with_db(uid: i64, doc_id: &str, db: Arc) -> Document { +pub async fn open_document_with_db(uid: i64, doc_id: &str, db: Arc) -> Document { setup_log(); let disk_plugin = RocksdbDiskPlugin::new(uid, Arc::downgrade(&db), None); let collab = CollabBuilder::new(uid, doc_id) @@ -114,10 +114,10 @@ pub async fn open_document_with_db(uid: i64, doc_id: &str, db: Arc Arc { +pub fn document_storage() -> Arc { let tempdir = TempDir::new().unwrap(); let path = tempdir.into_path(); - Arc::new(RocksCollabDB::open_opt(path, false).unwrap()) + Arc::new(CollabKVDB::open_opt(path, false).unwrap()) } fn setup_log() { diff --git a/collab-entity/Cargo.toml b/collab-entity/Cargo.toml index 0d0e1c313..f9e66470c 100644 --- a/collab-entity/Cargo.toml +++ b/collab-entity/Cargo.toml @@ -4,6 +4,8 @@ version = "0.1.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +crate-type = ["cdylib", "rlib"] [dependencies] uuid = { version = "1.3.3", features = ["v4"] } @@ -13,4 +15,8 @@ serde_repr = "0.1" collab = { version = "0.1.0", path = "../collab" } anyhow.workspace = true bytes = { workspace = true, features = ["serde"] } +getrandom = { version = "0.2", optional = true } + +[features] +wasm_build = ["getrandom/js"] diff --git a/collab-folder/Cargo.toml b/collab-folder/Cargo.toml index 9620ae6a3..2415c34a4 100644 --- a/collab-folder/Cargo.toml +++ b/collab-folder/Cargo.toml @@ -4,12 +4,13 @@ name = "collab-folder" version = "0.1.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +crate-type = ["cdylib", "rlib"] [dependencies] anyhow.workspace = true chrono.workspace = true collab = {path = "../collab" } -collab-persistence = { workspace = true } parking_lot.workspace = true serde.workspace = true serde_json.workspace = true @@ -21,11 +22,15 @@ tracing.workspace = true [dev-dependencies] assert-json-diff = "2.0.2" -collab-plugins = { workspace = true, features = ["rocksdb_plugin"]} +collab-plugins = { workspace = true } fs_extra = "1.2.0" nanoid = "0.4.0" tempfile = "3.8.0" -tokio = {version = "1.26", features = ["full"]} +tokio = {version = "1.26", features = ["rt"]} tracing-subscriber = {version = "0.3.3", features = ["env-filter"]} walkdir = "2.3.2" zip = "0.6.6" + +[features] +wasm_build = [] + diff --git a/collab-folder/tests/folder_test/util.rs b/collab-folder/tests/folder_test/util.rs index d94c23e5c..2c2f733bf 100644 --- a/collab-folder/tests/folder_test/util.rs +++ b/collab-folder/tests/folder_test/util.rs @@ -7,8 +7,8 @@ use std::sync::{Arc, Once}; use collab::preclude::CollabBuilder; use collab_folder::*; -use collab_persistence::kv::rocks_kv::RocksCollabDB; -use collab_plugins::local_storage::rocksdb::RocksdbDiskPlugin; +use collab_plugins::local_storage::rocksdb::rocksdb_plugin::RocksdbDiskPlugin; +use collab_plugins::CollabKVDB; use nanoid::nanoid; use tempfile::TempDir; use tracing_subscriber::fmt::Subscriber; @@ -20,7 +20,7 @@ pub struct FolderTest { folder: Folder, #[allow(dead_code)] - db: Arc, + db: Arc, #[allow(dead_code)] cleaner: Cleaner, @@ -51,7 +51,7 @@ pub async fn create_folder_with_data( let tempdir = TempDir::new().unwrap(); let path = tempdir.into_path(); - let db = Arc::new(RocksCollabDB::open_opt(path.clone(), false).unwrap()); + let db = Arc::new(CollabKVDB::open_opt(path.clone(), false).unwrap()); let disk_plugin = RocksdbDiskPlugin::new(uid.as_i64(), Arc::downgrade(&db), None); let cleaner: Cleaner = Cleaner::new(path); @@ -79,7 +79,7 @@ pub async fn create_folder_with_data( } pub async fn open_folder_with_db(uid: UserId, object_id: &str, db_path: PathBuf) -> FolderTest { - let db = Arc::new(RocksCollabDB::open_opt(db_path.clone(), false).unwrap()); + let db = Arc::new(CollabKVDB::open_opt(db_path.clone(), false).unwrap()); let disk_plugin = RocksdbDiskPlugin::new(uid.as_i64(), Arc::downgrade(&db), None); let cleaner: Cleaner = Cleaner::new(db_path); let collab = CollabBuilder::new(1, object_id) diff --git a/collab-folder/tests/folder_test/view_test.rs b/collab-folder/tests/folder_test/view_test.rs index bf30f72a2..8a49d4b0b 100644 --- a/collab-folder/tests/folder_test/view_test.rs +++ b/collab-folder/tests/folder_test/view_test.rs @@ -204,7 +204,7 @@ async fn dissociate_and_associate_view_test() { let r_view = folder_test.views.get_view(view_1_id).unwrap(); assert_eq!(r_view.children.items.iter().len(), 2); - assert_eq!(r_view.children.items.get(0).unwrap().id, view_2_id); + assert_eq!(r_view.children.items.first().unwrap().id, view_2_id); assert_eq!(r_view.children.items.get(1).unwrap().id, view_1_child_id); folder_test @@ -219,7 +219,7 @@ async fn dissociate_and_associate_view_test() { let r_view = folder_test.views.get_view(view_1_id).unwrap(); assert_eq!(r_view.children.items.iter().len(), 2); - assert_eq!(r_view.children.items.get(0).unwrap().id, view_1_child_id); + assert_eq!(r_view.children.items.first().unwrap().id, view_1_child_id); assert_eq!(r_view.children.items.get(1).unwrap().id, view_2_id); } @@ -261,7 +261,7 @@ async fn move_view_across_parent_test() { assert_eq!(view_1_child.parent_view_id, workspace_id); assert_eq!(workspace.child_views.items.len(), 3); assert_eq!( - workspace.child_views.items.get(0).unwrap().id, + workspace.child_views.items.first().unwrap().id, view_1_child_id ); @@ -279,7 +279,7 @@ async fn move_view_across_parent_test() { workspace.child_views.items.get(1).unwrap().id, view_1_child_id ); - assert_eq!(workspace.child_views.items.get(0).unwrap().id, view_1_id); + assert_eq!(workspace.child_views.items.first().unwrap().id, view_1_id); // move view_1_child from current workspace to view_1 folder_test.move_nested_view(view_1_child_id, view_1_id, None); @@ -288,7 +288,7 @@ async fn move_view_across_parent_test() { let view_1_child = folder_test.views.get_view(view_1_child_id).unwrap(); let workspace = folder_test.get_current_workspace().unwrap(); assert_eq!(view_1.children.items.iter().len(), 1); - assert_eq!(view_1.children.items.get(0).unwrap().id, view_1_child_id); + assert_eq!(view_1.children.items.first().unwrap().id, view_1_child_id); assert_eq!(view_1_child.parent_view_id, view_1_id); assert_eq!(view_2.children.items.iter().len(), 0); assert_eq!(workspace.child_views.items.len(), 2); @@ -320,7 +320,7 @@ async fn create_view_test_with_index() { folder_test.insert_view(view_6.clone(), Some(3)); let views = folder_test.get_current_workspace_views(); - assert_eq!(views.get(0).unwrap().id, view_2.id); + assert_eq!(views.first().unwrap().id, view_2.id); assert_eq!(views.get(1).unwrap().id, view_3.id); assert_eq!(views.get(2).unwrap().id, view_1.id); assert_eq!(views.get(3).unwrap().id, view_6.id); @@ -335,7 +335,7 @@ async fn check_created_and_edited_time_test() { let view = make_test_view("v1", "w1", vec![]); folder_test.insert_view(view, Some(0)); let views = folder_test.get_current_workspace_views(); - let v1 = views.get(0).unwrap(); + let v1 = views.first().unwrap(); assert_eq!(v1.created_by.unwrap(), uid.as_i64()); assert_eq!(v1.last_edited_by.unwrap(), uid.as_i64()); assert_eq!(v1.last_edited_time, v1.created_at); diff --git a/collab-persistence/Cargo.toml b/collab-persistence/Cargo.toml deleted file mode 100644 index 519dbe191..000000000 --- a/collab-persistence/Cargo.toml +++ /dev/null @@ -1,34 +0,0 @@ -[package] -name = "collab-persistence" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -collab = { workspace = true } -rocksdb = { version = "0.21.0", optional = true, default-features = false, features = ["zstd"] } -thiserror = "1.0.30" -serde.workspace = true -bincode = "1.3.3" -yrs.workspace = true -smallvec = { version = "1.10", features = ["write", "union", "const_generics", "const_new"] } -chrono.workspace = true -tokio = { version = "1.26", features = ["rt", "sync"] } -tracing.workspace = true -parking_lot.workspace = true -lazy_static = "1.4.0" -async-trait.workspace = true -anyhow = "1.0.75" - -[dev-dependencies] -collab-persistence = { path = "", features = ["rocksdb_persistence"] } -tempfile = "3.8.0" -futures = "0.3.18" -tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } -tokio = { version = "1.26", features = ["full"] } -test-case = "3.1.0" - -[features] -default = [] -rocksdb_persistence = ["rocksdb"] \ No newline at end of file diff --git a/collab-persistence/src/kv/mod.rs b/collab-persistence/src/kv/mod.rs deleted file mode 100644 index c0ad819a1..000000000 --- a/collab-persistence/src/kv/mod.rs +++ /dev/null @@ -1,83 +0,0 @@ -use std::fmt::Debug; -use std::ops::RangeBounds; -use std::sync::Arc; - -use crate::PersistenceError; - -#[cfg(feature = "rocksdb_persistence")] -pub mod rocks_kv; - -pub trait KVStore<'a> { - type Range: Iterator; - type Entry: KVEntry; - type Value: AsRef<[u8]>; - type Error: Into + Debug; - - /// Get a value by key - fn get>(&self, key: K) -> Result, Self::Error>; - - fn insert, V: AsRef<[u8]>>(&self, key: K, value: V) -> Result<(), Self::Error>; - - /// Remove a key, returning the last value if it exists - fn remove(&self, key: &[u8]) -> Result<(), Self::Error>; - - /// Remove all keys in the range [from..to] - /// The upper bound itself is not included on the iteration result. - fn remove_range(&self, from: &[u8], to: &[u8]) -> Result<(), Self::Error>; - - /// Return an iterator over the range of keys - /// The upper bound itself is not included on the iteration result. - fn range, R: RangeBounds>(&self, range: R) -> Result; - - /// Return the entry prior to the given key - fn next_back_entry(&self, key: &[u8]) -> Result, Self::Error>; -} - -/// This trait is used to represents as the generic Range of different implementation. -pub trait KVRange<'a> { - type Range: Iterator; - type Entry: KVEntry; - type Error: Into; - - fn kv_range(self) -> Result; -} - -/// A key-value entry -pub trait KVEntry { - fn key(&self) -> &[u8]; - fn value(&self) -> &[u8]; -} - -impl KVStore<'static> for Arc -where - T: KVStore<'static>, -{ - type Range = >::Range; - type Entry = >::Entry; - type Value = >::Value; - type Error = >::Error; - - fn get>(&self, key: K) -> Result, Self::Error> { - (**self).get(key) - } - - fn insert, V: AsRef<[u8]>>(&self, key: K, value: V) -> Result<(), Self::Error> { - (**self).insert(key, value) - } - - fn remove(&self, key: &[u8]) -> Result<(), Self::Error> { - (**self).remove(key) - } - - fn remove_range(&self, from: &[u8], to: &[u8]) -> Result<(), Self::Error> { - (**self).remove_range(from, to) - } - - fn range, R: RangeBounds>(&self, range: R) -> Result { - self.as_ref().range(range) - } - - fn next_back_entry(&self, key: &[u8]) -> Result, Self::Error> { - (**self).next_back_entry(key) - } -} diff --git a/collab-persistence/tests/persistence/main.rs b/collab-persistence/tests/persistence/main.rs deleted file mode 100644 index cd685eace..000000000 --- a/collab-persistence/tests/persistence/main.rs +++ /dev/null @@ -1,4 +0,0 @@ -mod range_test; -mod restore_test; -mod rocksdb_cf_test; -mod util; diff --git a/collab-persistence/tests/persistence/rocksdb_cf_test.rs b/collab-persistence/tests/persistence/rocksdb_cf_test.rs deleted file mode 100644 index 419d91aaf..000000000 --- a/collab-persistence/tests/persistence/rocksdb_cf_test.rs +++ /dev/null @@ -1,22 +0,0 @@ -use collab_persistence::kv::rocks_kv::RocksCollabDB; -use collab_persistence::kv::KVStore; - -use crate::util::rocks_db; - -#[tokio::test] -async fn open_same_cf_test() { - let uid = 1; - let (path, db_a) = rocks_db(); - db_a - .with_write_txn(|txn| { - txn.insert("1", "a")?; - Ok(()) - }) - .unwrap(); - drop(db_a); - - let db_b = RocksCollabDB::open_with_cfs(vec![uid.to_string()], path).unwrap(); - let txn = db_b.read_txn(); - let value = txn.get("1").unwrap().unwrap(); - assert_eq!(value, "a".as_bytes()); -} diff --git a/collab-persistence/tests/persistence/util.rs b/collab-persistence/tests/persistence/util.rs deleted file mode 100644 index b87f29afa..000000000 --- a/collab-persistence/tests/persistence/util.rs +++ /dev/null @@ -1,32 +0,0 @@ -use std::path::PathBuf; -use std::sync::Once; - -use collab_persistence::kv::rocks_kv::RocksCollabDB; - -use tempfile::TempDir; -use tracing_subscriber::{fmt::Subscriber, util::SubscriberInitExt, EnvFilter}; - -pub fn rocks_db() -> (PathBuf, RocksCollabDB) { - setup_log(); - - let tempdir = TempDir::new().unwrap(); - let path = tempdir.into_path(); - let cloned_path = path.clone(); - (path, RocksCollabDB::open_opt(cloned_path, false).unwrap()) -} - -fn setup_log() { - static START: Once = Once::new(); - START.call_once(|| { - let level = "info"; - let mut filters = vec![]; - filters.push(format!("collab_persistence={}", level)); - std::env::set_var("RUST_LOG", filters.join(",")); - - let subscriber = Subscriber::builder() - .with_env_filter(EnvFilter::from_default_env()) - .with_ansi(true) - .finish(); - subscriber.try_init().unwrap(); - }); -} diff --git a/collab-plugins/Cargo.toml b/collab-plugins/Cargo.toml index 613766624..35324706f 100644 --- a/collab-plugins/Cargo.toml +++ b/collab-plugins/Cargo.toml @@ -4,49 +4,53 @@ version = "0.1.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +crate-type = ["cdylib", "rlib"] [dependencies] yrs.workspace = true - -#collab collab = { workspace = true } collab-entity = { workspace = true } -collab-persistence = { workspace = true, optional = true } futures-util = { version = "0.3", features = ["sink"] } -tokio = { version = "1.26.0", features = ["sync"] } +tokio = { version = "1.26.0", features = ["sync", "rt"] } tracing.workspace = true parking_lot.workspace = true - anyhow.workspace = true + tokio-retry = "0.3" async-trait.workspace = true thiserror.workspace = true serde.workspace = true serde_json.workspace = true -rand = { version = "0.8" } similar = { version = "2.2.1" } tokio-stream = { version = "0.1.14", features = ["sync"] } uuid = { version = "1.3.3", features = ["v4"] } bytes.workspace = true +rand = { version = "0.8", optional = true } +lazy_static = "1.4.0" +smallvec = { version = "1.10", features = ["write", "union", "const_generics", "const_new"] } +chrono = { version = "0.4.22", default-features = false, features = ["clock"] } +bincode = "1.3.3" +getrandom = { version = "0.2", optional = true } + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +rocksdb = { version = "0.21.0", default-features = false, features = ["zstd"] } + [dev-dependencies] -collab-plugins = { workspace = true, features = ["rocksdb_plugin", "snapshot_plugin"] } +tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } +tokio = { version = "1.26.0", features = ["macros"] } +rand = { version = "0.8" } tempfile = "3.8.0" assert-json-diff = "2.0.2" -tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } -tokio = { version = "1.26.0", features = ["macros", "net"] } tokio-util = { version = "0.7", features = ["codec"] } -dashmap = "5.4.0" -nanoid = "0.4.0" config = { version = "0.13.3", default-features = false, features = ["yaml"] } dotenv = "0.15.0" futures = "0.3.17" -chrono.workspace = true [features] -default = ["postgres_storage_plugin"] -rocksdb_plugin = ["collab-persistence/rocksdb_persistence"] -postgres_storage_plugin = ["collab-persistence/rocksdb_persistence"] -snapshot_plugin = ["collab-persistence/rocksdb_persistence"] +default = [] +postgres_plugin = ["rand"] +wasm_build = ["getrandom/js"] \ No newline at end of file diff --git a/collab-plugins/src/cloud_storage/mod.rs b/collab-plugins/src/cloud_storage/mod.rs index 35cec5b9b..9bf83a418 100644 --- a/collab-plugins/src/cloud_storage/mod.rs +++ b/collab-plugins/src/cloud_storage/mod.rs @@ -8,8 +8,6 @@ pub use yrs::Update as YrsUpdate; pub mod postgres; -pub mod network_state; - mod channel; mod error; mod msg; diff --git a/collab-plugins/src/cloud_storage/postgres/plugin.rs b/collab-plugins/src/cloud_storage/postgres/plugin.rs index 9611e0fce..d98907c91 100644 --- a/collab-plugins/src/cloud_storage/postgres/plugin.rs +++ b/collab-plugins/src/cloud_storage/postgres/plugin.rs @@ -11,9 +11,6 @@ use collab::core::collab_state::SnapshotState; use collab::core::origin::CollabOrigin; use collab::preclude::{Collab, CollabPlugin}; use collab_entity::CollabObject; -use collab_persistence::doc::YrsDocAction; -use collab_persistence::kv::rocks_kv::RocksCollabDB; -use collab_persistence::TransactionMutExt; use parking_lot::RwLock; use tokio_retry::strategy::FibonacciBackoff; use tokio_retry::{Action, Retry}; @@ -26,12 +23,15 @@ use crate::cloud_storage::remote_collab::{ should_create_snapshot, RemoteCollab, RemoteCollabStorage, }; use crate::cloud_storage::sink::{SinkConfig, SinkStrategy}; +use crate::local_storage::kv::doc::CollabKVAction; +use crate::local_storage::kv::TransactionMutExt; +use crate::CollabKVDB; pub struct SupabaseDBPlugin { uid: i64, object: CollabObject, local_collab: Weak, - local_collab_storage: Weak, + local_collab_storage: Weak, remote_collab: Arc, remote_collab_storage: Arc, pending_updates: Arc>>>, @@ -45,7 +45,7 @@ impl SupabaseDBPlugin { local_collab: Weak, sync_per_secs: u64, remote_collab_storage: Arc, - local_collab_storage: Weak, + local_collab_storage: Weak, ) -> Self { let pending_updates = Arc::new(RwLock::new(Vec::new())); let is_first_sync_done = Arc::new(AtomicBool::new(false)); @@ -134,7 +134,7 @@ fn create_snapshot_if_need( object: CollabObject, remote_update: Vec, weak_local_collab: Weak, - weak_local_collab_storage: Weak, + weak_local_collab_storage: Weak, weak_remote_collab_storage: Weak, ) { tokio::spawn(async move { @@ -217,7 +217,7 @@ struct InitSyncAction { object: CollabObject, remote_collab: Weak, local_collab: Weak, - local_collab_storage: Weak, + local_collab_storage: Weak, remote_collab_storage: Weak, pending_updates: Weak>>>, is_first_sync_done: Weak, diff --git a/collab-plugins/src/cloud_storage/network_state.rs b/collab-plugins/src/connect_state.rs similarity index 55% rename from collab-plugins/src/cloud_storage/network_state.rs rename to collab-plugins/src/connect_state.rs index a12f84415..44adbf54e 100644 --- a/collab-plugins/src/cloud_storage/network_state.rs +++ b/collab-plugins/src/connect_state.rs @@ -2,20 +2,20 @@ use parking_lot::Mutex; use tokio::sync::broadcast; #[derive(Clone, Eq, PartialEq)] -pub enum CollabNetworkState { +pub enum CollabConnectState { Connected, Disconnected, } -pub struct CollabNetworkReachability { - state: Mutex, - state_sender: broadcast::Sender, +pub struct CollabConnectReachability { + state: Mutex, + state_sender: broadcast::Sender, } -impl Default for CollabNetworkReachability { +impl Default for CollabConnectReachability { fn default() -> Self { let (state_sender, _) = broadcast::channel(1000); - let state = Mutex::new(CollabNetworkState::Connected); + let state = Mutex::new(CollabConnectState::Connected); Self { state, state_sender, @@ -23,12 +23,12 @@ impl Default for CollabNetworkReachability { } } -impl CollabNetworkReachability { +impl CollabConnectReachability { pub fn new() -> Self { Self::default() } - pub fn set_state(&self, new_state: CollabNetworkState) { + pub fn set_state(&self, new_state: CollabConnectState) { let mut lock_guard = self.state.lock(); if *lock_guard != new_state { *lock_guard = new_state.clone(); @@ -36,7 +36,7 @@ impl CollabNetworkReachability { } } - pub fn subscribe(&self) -> broadcast::Receiver { + pub fn subscribe(&self) -> broadcast::Receiver { self.state_sender.subscribe() } } diff --git a/collab-plugins/src/lib.rs b/collab-plugins/src/lib.rs index 9409619de..8c1db34ae 100644 --- a/collab-plugins/src/lib.rs +++ b/collab-plugins/src/lib.rs @@ -1,10 +1,10 @@ -#[cfg(any(feature = "rocksdb_plugin"))] -pub use collab_persistence::*; - pub mod local_storage; -#[cfg(feature = "postgres_storage_plugin")] +#[cfg(feature = "postgres_plugin")] pub mod cloud_storage; +pub mod connect_state; -#[cfg(feature = "snapshot_plugin")] -pub mod snapshot; +#[cfg(not(target_arch = "wasm32"))] +use crate::local_storage::rocksdb::kv_impl::RocksStore; +#[cfg(not(target_arch = "wasm32"))] +pub type CollabKVDB = RocksStore; diff --git a/collab-persistence/src/db.rs b/collab-plugins/src/local_storage/kv/db.rs similarity index 59% rename from collab-persistence/src/db.rs rename to collab-plugins/src/local_storage/kv/db.rs index 3d1a74cc2..0722e7b33 100644 --- a/collab-persistence/src/db.rs +++ b/collab-plugins/src/local_storage/kv/db.rs @@ -1,18 +1,76 @@ use std::fmt::Debug; use std::io::Write; +use std::ops::RangeBounds; use std::panic; use std::panic::AssertUnwindSafe; +use std::sync::Arc; +use crate::local_storage::kv::keys::*; +use crate::local_storage::kv::oid::{LOCAL_DOC_ID_GEN, OID}; +use crate::local_storage::kv::snapshot::CollabSnapshot; +use crate::local_storage::kv::PersistenceError; use smallvec::SmallVec; use yrs::{TransactionMut, Update}; -use crate::error::PersistenceError; -use crate::keys::{ - clock_from_key, make_doc_update_key, make_snapshot_update_key, Clock, DocID, Key, SnapshotID, -}; -use crate::kv::{KVEntry, KVStore}; -use crate::oid::{DOC_ID_LEN, LOCAL_DOC_ID_GEN, OID}; -use crate::snapshot::CollabSnapshot; +pub trait KVStore<'a> { + type Range: Iterator; + type Entry: KVEntry; + type Value: AsRef<[u8]>; + type Error: Into + Debug; + + /// Get a value by key + fn get>(&self, key: K) -> Result, Self::Error>; + + fn insert, V: AsRef<[u8]>>(&self, key: K, value: V) -> Result<(), Self::Error>; + + /// Remove a key, returning the last value if it exists + fn remove(&self, key: &[u8]) -> Result<(), Self::Error>; + + /// Remove all keys in the range [from..to] + /// The upper bound itself is not included on the iteration result. + fn remove_range(&self, from: &[u8], to: &[u8]) -> Result<(), Self::Error>; + + /// Return an iterator over the range of keys + /// The upper bound itself is not included on the iteration result. + fn range, R: RangeBounds>(&self, range: R) -> Result; + + /// Return the entry prior to the given key + fn next_back_entry(&self, key: &[u8]) -> Result, Self::Error>; +} + +impl KVStore<'static> for Arc +where + T: KVStore<'static>, +{ + type Range = >::Range; + type Entry = >::Entry; + type Value = >::Value; + type Error = >::Error; + + fn get>(&self, key: K) -> Result, Self::Error> { + (**self).get(key) + } + + fn insert, V: AsRef<[u8]>>(&self, key: K, value: V) -> Result<(), Self::Error> { + (**self).insert(key, value) + } + + fn remove(&self, key: &[u8]) -> Result<(), Self::Error> { + (**self).remove(key) + } + + fn remove_range(&self, from: &[u8], to: &[u8]) -> Result<(), Self::Error> { + (**self).remove_range(from, to) + } + + fn range, R: RangeBounds>(&self, range: R) -> Result { + self.as_ref().range(range) + } + + fn next_back_entry(&self, key: &[u8]) -> Result, Self::Error> { + (**self).next_back_entry(key) + } +} pub fn insert_snapshot_update<'a, K, S>( store: &S, @@ -147,3 +205,18 @@ impl<'doc> TransactionMutExt<'doc> for TransactionMut<'doc> { } } } + +/// This trait is used to represents as the generic Range of different implementation. +pub trait KVRange<'a> { + type Range: Iterator; + type Entry: KVEntry; + type Error: Into; + + fn kv_range(self) -> Result; +} + +/// A key-value entry +pub trait KVEntry { + fn key(&self) -> &[u8]; + fn value(&self) -> &[u8]; +} diff --git a/collab-persistence/src/doc.rs b/collab-plugins/src/local_storage/kv/doc.rs similarity index 95% rename from collab-persistence/src/doc.rs rename to collab-plugins/src/local_storage/kv/doc.rs index 179626b69..07207d087 100644 --- a/collab-persistence/src/doc.rs +++ b/collab-plugins/src/local_storage/kv/doc.rs @@ -1,30 +1,13 @@ use std::fmt::Debug; +use crate::local_storage::kv::keys::*; +use crate::local_storage::kv::snapshot::SnapshotAction; +use crate::local_storage::kv::*; use yrs::updates::decoder::Decode; use yrs::updates::encoder::Encode; use yrs::{Doc, ReadTxn, StateVector, Transact, TransactionMut, Update}; -use crate::keys::{ - make_doc_end_key, make_doc_id_key, make_doc_start_key, make_doc_state_key, make_doc_update_key, - make_state_vector_key, oid_from_key, Clock, DocID, Key, DOC_SPACE, DOC_SPACE_OBJECT, - DOC_SPACE_OBJECT_KEY, -}; -use crate::kv::KVEntry; -use crate::kv::KVStore; -use crate::snapshot::SnapshotAction; -use crate::{ - get_id_for_key, get_last_update_key, insert_doc_update, make_doc_id_for_key, PersistenceError, - TransactionMutExt, -}; - -impl<'a, T> YrsDocAction<'a> for T -where - T: KVStore<'a>, - PersistenceError: From<>::Error>, -{ -} - -pub trait YrsDocAction<'a>: KVStore<'a> + Sized +pub trait CollabKVAction<'a>: KVStore<'a> + Sized + 'a where PersistenceError: From<>::Error>, { @@ -372,6 +355,13 @@ where } } +impl<'a, T> CollabKVAction<'a> for T +where + T: KVStore<'a> + 'a, + PersistenceError: From<>::Error>, +{ +} + /// Get or create a document id for the given object id. fn get_or_create_did<'a, K, S>( uid: i64, diff --git a/collab-persistence/src/error.rs b/collab-plugins/src/local_storage/kv/error.rs similarity index 83% rename from collab-persistence/src/error.rs rename to collab-plugins/src/local_storage/kv/error.rs index 07f27d470..873f87521 100644 --- a/collab-persistence/src/error.rs +++ b/collab-plugins/src/local_storage/kv/error.rs @@ -1,24 +1,20 @@ #[derive(Debug, thiserror::Error)] pub enum PersistenceError { - #[cfg(feature = "sled_db_persistence")] - #[error(transparent)] - SledDb(#[from] sled::Error), - - #[cfg(feature = "rocksdb_persistence")] + #[cfg(not(target_arch = "wasm32"))] #[error("Rocksdb corruption:{0}")] RocksdbCorruption(String), - #[cfg(feature = "rocksdb_persistence")] + #[cfg(not(target_arch = "wasm32"))] #[error("Rocksdb repair:{0}")] RocksdbRepairFail(String), - #[cfg(feature = "rocksdb_persistence")] + #[cfg(not(target_arch = "wasm32"))] #[error("{0}")] RocksdbBusy(String), // If the database is already locked by another process, it will return an IO error. It // happens when the database is already opened by another process. - #[cfg(feature = "rocksdb_persistence")] + #[cfg(not(target_arch = "wasm32"))] #[error("{0}")] RocksdbIOError(String), @@ -50,7 +46,7 @@ pub enum PersistenceError { Internal(#[from] anyhow::Error), } -#[cfg(feature = "rocksdb_persistence")] +#[cfg(not(target_arch = "wasm32"))] impl From for PersistenceError { fn from(value: rocksdb::Error) -> Self { match value.kind() { diff --git a/collab-persistence/src/keys.rs b/collab-plugins/src/local_storage/kv/keys.rs similarity index 100% rename from collab-persistence/src/keys.rs rename to collab-plugins/src/local_storage/kv/keys.rs diff --git a/collab-persistence/src/lib.rs b/collab-plugins/src/local_storage/kv/mod.rs similarity index 92% rename from collab-persistence/src/lib.rs rename to collab-plugins/src/local_storage/kv/mod.rs index 9200d2a9e..59e3a1f5d 100644 --- a/collab-persistence/src/lib.rs +++ b/collab-plugins/src/local_storage/kv/mod.rs @@ -6,7 +6,6 @@ mod db; pub mod doc; pub mod error; pub mod keys; -pub mod kv; mod oid; mod range; pub mod snapshot; diff --git a/collab-persistence/src/oid.rs b/collab-plugins/src/local_storage/kv/oid.rs similarity index 97% rename from collab-persistence/src/oid.rs rename to collab-plugins/src/local_storage/kv/oid.rs index 079246a79..b39a2d5d4 100644 --- a/collab-persistence/src/oid.rs +++ b/collab-plugins/src/local_storage/kv/oid.rs @@ -15,8 +15,6 @@ const SEQUENCE_MASK: u64 = (1 << SEQUENCE_BITS) - 1; pub type OID = u64; -pub const DOC_ID_LEN: usize = 8; - lazy_static! { pub static ref LOCAL_DOC_ID_GEN: Mutex = Mutex::new(DocIDGen::new()); } @@ -80,7 +78,7 @@ mod tests { use std::sync::Arc; use std::thread; - use crate::oid::LOCAL_DOC_ID_GEN; + use crate::local_storage::kv::oid::LOCAL_DOC_ID_GEN; use parking_lot::RwLock; #[test] diff --git a/collab-persistence/src/range.rs b/collab-plugins/src/local_storage/kv/range.rs similarity index 100% rename from collab-persistence/src/range.rs rename to collab-plugins/src/local_storage/kv/range.rs diff --git a/collab-persistence/src/snapshot.rs b/collab-plugins/src/local_storage/kv/snapshot.rs similarity index 94% rename from collab-persistence/src/snapshot.rs rename to collab-plugins/src/local_storage/kv/snapshot.rs index 06041662f..392bdff10 100644 --- a/collab-persistence/src/snapshot.rs +++ b/collab-plugins/src/local_storage/kv/snapshot.rs @@ -2,18 +2,12 @@ use std::fmt::Debug; use std::panic; use std::panic::AssertUnwindSafe; +use crate::local_storage::kv::keys::*; +use crate::local_storage::kv::*; use serde::{Deserialize, Serialize}; use yrs::updates::encoder::{Encoder, EncoderV1}; use yrs::{ReadTxn, Snapshot}; -use crate::keys::{make_snapshot_id_key, make_snapshot_update_key, Clock, Key, SnapshotID}; -use crate::kv::KVEntry; -use crate::kv::KVStore; -use crate::{ - get_id_for_key, get_last_update_key, insert_snapshot_update, make_doc_id_for_key, - PersistenceError, -}; - impl<'a, T> SnapshotAction<'a> for T where T: KVStore<'a>, @@ -188,6 +182,18 @@ pub fn try_encode_snapshot( } } +pub trait SnapshotPersistence: Send + Sync { + fn get_snapshots(&self, uid: i64, object_id: &str) -> Vec; + + fn create_snapshot( + &self, + uid: i64, + object_id: &str, + title: String, + snapshot_data: Vec, + ) -> Result<(), PersistenceError>; +} + #[derive(Serialize, Deserialize)] pub struct CollabSnapshot { pub data: Vec, diff --git a/collab-plugins/src/local_storage/mod.rs b/collab-plugins/src/local_storage/mod.rs index 4a35d93a0..26ed0faaf 100644 --- a/collab-plugins/src/local_storage/mod.rs +++ b/collab-plugins/src/local_storage/mod.rs @@ -1,4 +1,6 @@ -#[cfg(feature = "rocksdb_plugin")] +pub mod kv; + +#[cfg(not(target_arch = "wasm32"))] pub mod rocksdb; #[derive(Clone)] @@ -12,6 +14,7 @@ pub struct CollabPersistenceConfig { /// Flush the document. Default is [true]. /// After flush the document, all updates will be removed and the document state vector that /// contains all the updates will be reset. + #[allow(dead_code)] pub(crate) flush_doc: bool, } diff --git a/collab-persistence/src/kv/rocks_kv.rs b/collab-plugins/src/local_storage/rocksdb/kv_impl.rs similarity index 76% rename from collab-persistence/src/kv/rocks_kv.rs rename to collab-plugins/src/local_storage/rocksdb/kv_impl.rs index 6976f8f73..5ba71b311 100644 --- a/collab-persistence/src/kv/rocks_kv.rs +++ b/collab-plugins/src/local_storage/rocksdb/kv_impl.rs @@ -3,19 +3,15 @@ use std::ops::RangeBounds; use std::path::Path; use std::sync::Arc; -use rocksdb::backup::{BackupEngine, BackupEngineOptions}; +use crate::local_storage::kv::doc::CollabKVAction; +use crate::local_storage::kv::{KVEntry, KVStore, PersistenceError}; use rocksdb::Direction::Forward; use rocksdb::{ - ColumnFamilyDescriptor, DBIteratorWithThreadMode, Direction, Env, ErrorKind, IteratorMode, - Options, ReadOptions, SingleThreaded, Transaction, TransactionDB, TransactionDBOptions, - TransactionOptions, WriteOptions, + DBIteratorWithThreadMode, Direction, ErrorKind, IteratorMode, Options, ReadOptions, + SingleThreaded, Transaction, TransactionDB, TransactionDBOptions, TransactionOptions, + WriteOptions, }; -use crate::kv::{KVEntry, KVStore}; -use crate::PersistenceError; - -pub type RocksCollabDB = RocksStore; - #[derive(Clone)] pub struct RocksStore { db: Arc, @@ -88,55 +84,14 @@ impl RocksStore { Ok(()) } - #[allow(dead_code)] - fn backup_engine(backup_dir: impl AsRef) -> Result { - let backup_opts = BackupEngineOptions::new(backup_dir)?; - let env = Env::new()?; - let backup_engine = BackupEngine::open(&backup_opts, &env)?; - Ok(backup_engine) - } - - pub fn open_with_cfs( - names: Vec, - path: impl AsRef, - ) -> Result { - let txn_db_opts = TransactionDBOptions::default(); - let mut db_opts = Options::default(); - db_opts.create_if_missing(true); - db_opts.create_missing_column_families(true); - - // CFs - let cf_opts = Options::default(); - let cfs = names - .into_iter() - .map(|name| ColumnFamilyDescriptor::new(name, cf_opts.clone())) - .collect::>(); - let db = Arc::new(TransactionDB::open_cf_descriptors( - &db_opts, - &txn_db_opts, - path, - cfs, - )?); - Ok(Self { db }) - } - /// Return a read transaction that accesses the database exclusively. - pub fn read_txn(&self) -> RocksKVStoreImpl<'_, TransactionDB> { + pub fn read_txn(&self) -> impl CollabKVAction<'_, Error = PersistenceError> { let mut txn_options = TransactionOptions::default(); txn_options.set_snapshot(true); let txn = self .db .transaction_opt(&WriteOptions::default(), &txn_options); - MutexRocksKVStoreImpl::new(txn) - } - - pub fn write_txn(&self) -> RocksKVStoreImpl<'_, TransactionDB> { - let mut txn_options = TransactionOptions::default(); - txn_options.set_snapshot(true); - let txn = self - .db - .transaction_opt(&WriteOptions::default(), &txn_options); - MutexRocksKVStoreImpl::new(txn) + RocksKVStoreImpl::new(txn) } /// Create a write transaction that accesses the database exclusively. @@ -154,7 +109,7 @@ impl RocksStore { let txn = self .db .transaction_opt(&WriteOptions::default(), &txn_options); - let store = MutexRocksKVStoreImpl::new(txn); + let store = RocksKVStoreImpl::new(txn); let result = f(&store)?; store.0.commit()?; Ok(result) @@ -163,12 +118,11 @@ impl RocksStore { /// Implementation of [KVStore] for [RocksStore]. This is a wrapper around [Transaction]. // pub struct RocksKVStoreImpl<'a, DB: Send + Sync>(Transaction<'a, DB>); -pub type RocksKVStoreImpl<'a, DB> = MutexRocksKVStoreImpl<'a, DB>; -pub struct MutexRocksKVStoreImpl<'a, DB: Send>(Transaction<'a, DB>); +pub struct RocksKVStoreImpl<'a, DB: Send>(Transaction<'a, DB>); -unsafe impl<'db, DB: Send> Send for MutexRocksKVStoreImpl<'db, DB> {} +unsafe impl<'a, DB: Send> Send for RocksKVStoreImpl<'a, DB> {} -impl<'a, DB: Send + Sync> MutexRocksKVStoreImpl<'a, DB> { +impl<'a, DB: Send + Sync> RocksKVStoreImpl<'a, DB> { pub fn new(txn: Transaction<'a, DB>) -> Self { Self(txn) } @@ -179,7 +133,7 @@ impl<'a, DB: Send + Sync> MutexRocksKVStoreImpl<'a, DB> { } } -impl<'a, DB: Send + Sync> KVStore<'a> for MutexRocksKVStoreImpl<'a, DB> { +impl<'a, DB: Send + Sync> KVStore<'a> for RocksKVStoreImpl<'a, DB> { type Range = RocksDBRange<'a, DB>; type Entry = RocksDBEntry; type Value = RocksDBVec; @@ -269,25 +223,10 @@ impl<'a, DB: Send + Sync> KVStore<'a> for MutexRocksKVStoreImpl<'a, DB> { impl<'a, DB: Send + Sync> From> for RocksKVStoreImpl<'a, DB> { #[inline(always)] fn from(txn: Transaction<'a, DB>) -> Self { - MutexRocksKVStoreImpl::new(txn) + RocksKVStoreImpl::new(txn) } } -// impl<'a, DB: Send + Sync> From> for Transaction<'a, DB> { -// fn from(store: RocksKVStoreImpl<'a, DB>) -> Self { -// store.0.lock() -// } -// } - -// impl<'a, DB: Send + Sync> Deref for RocksKVStoreImpl<'a, DB> { -// type Target = Transaction<'a, DB>; -// -// #[inline(always)] -// fn deref(&self) -> &Self::Target { -// &self.0 -// } -// } - pub type RocksDBVec = Vec; pub struct RocksDBRange<'a, DB> { diff --git a/collab-plugins/src/local_storage/rocksdb/mod.rs b/collab-plugins/src/local_storage/rocksdb/mod.rs new file mode 100644 index 000000000..530db1351 --- /dev/null +++ b/collab-plugins/src/local_storage/rocksdb/mod.rs @@ -0,0 +1,3 @@ +pub mod kv_impl; +pub mod rocksdb_plugin; +pub mod snapshot_plugin; diff --git a/collab-plugins/src/local_storage/rocksdb.rs b/collab-plugins/src/local_storage/rocksdb/rocksdb_plugin.rs similarity index 94% rename from collab-plugins/src/local_storage/rocksdb.rs rename to collab-plugins/src/local_storage/rocksdb/rocksdb_plugin.rs index e1e0e414a..f76fb3bf8 100644 --- a/collab-plugins/src/local_storage/rocksdb.rs +++ b/collab-plugins/src/local_storage/rocksdb/rocksdb_plugin.rs @@ -3,17 +3,17 @@ use std::sync::atomic::Ordering::SeqCst; use std::sync::atomic::{AtomicBool, AtomicU32}; use std::sync::{Arc, Weak}; +use crate::CollabKVDB; use collab::core::awareness::Awareness; use collab::core::collab::make_yrs_doc; use collab::core::collab_plugin::EncodedCollab; use collab::core::origin::CollabOrigin; use collab::preclude::CollabPlugin; -use collab_persistence::doc::YrsDocAction; -use collab_persistence::kv::rocks_kv::RocksCollabDB; use tracing::{error, instrument}; use yrs::updates::encoder::Encode; use yrs::{Doc, ReadTxn, StateVector, Transact, TransactionMut}; +use crate::local_storage::kv::doc::CollabKVAction; use crate::local_storage::CollabPersistenceConfig; pub trait RocksdbBackup: Send + Sync { @@ -24,7 +24,7 @@ pub trait RocksdbBackup: Send + Sync { #[derive(Clone)] pub struct RocksdbDiskPlugin { uid: i64, - db: Weak, + db: Weak, did_load: Arc, /// the number of updates on disk when opening the document initial_update_count: Arc, @@ -34,7 +34,7 @@ pub struct RocksdbDiskPlugin { } impl Deref for RocksdbDiskPlugin { - type Target = Weak; + type Target = Weak; fn deref(&self) -> &Self::Target { &self.db @@ -44,7 +44,7 @@ impl Deref for RocksdbDiskPlugin { impl RocksdbDiskPlugin { pub fn new_with_config( uid: i64, - db: Weak, + db: Weak, config: CollabPersistenceConfig, backup: Option>, ) -> Self { @@ -62,7 +62,7 @@ impl RocksdbDiskPlugin { } } - pub fn new(uid: i64, db: Weak, backup: Option>) -> Self { + pub fn new(uid: i64, db: Weak, backup: Option>) -> Self { Self::new_with_config(uid, db, CollabPersistenceConfig::default(), backup) } @@ -71,7 +71,7 @@ impl RocksdbDiskPlugin { } #[instrument(skip_all)] - fn flush_doc(&self, db: &Arc, object_id: &str) { + fn flush_doc(&self, db: &Arc, object_id: &str) { let _ = db.with_write_txn(|w_db_txn| { let doc = make_yrs_doc(); w_db_txn.load_doc_with_txn(self.uid, object_id, &mut doc.transact_mut())?; diff --git a/collab-plugins/src/snapshot/plugin.rs b/collab-plugins/src/local_storage/rocksdb/snapshot_plugin.rs similarity index 91% rename from collab-plugins/src/snapshot/plugin.rs rename to collab-plugins/src/local_storage/rocksdb/snapshot_plugin.rs index 5d5444b74..02d7b6a36 100644 --- a/collab-plugins/src/snapshot/plugin.rs +++ b/collab-plugins/src/local_storage/rocksdb/snapshot_plugin.rs @@ -3,12 +3,12 @@ use std::panic::AssertUnwindSafe; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, Weak}; +use crate::local_storage::kv::doc::CollabKVAction; +use crate::local_storage::kv::snapshot::{CollabSnapshot, SnapshotAction, SnapshotPersistence}; +use crate::local_storage::kv::PersistenceError; +use crate::CollabKVDB; use collab::preclude::{Collab, CollabPlugin}; use collab_entity::CollabObject; -use collab_persistence::doc::YrsDocAction; -use collab_persistence::kv::rocks_kv::RocksCollabDB; -use collab_persistence::snapshot::{CollabSnapshot, SnapshotAction}; -use collab_persistence::PersistenceError; use parking_lot::RwLock; use similar::{ChangeTag, TextDiff}; use yrs::updates::decoder::Decode; @@ -31,22 +31,10 @@ impl GenSnapshotState { } } -pub trait SnapshotPersistence: Send + Sync { - fn get_snapshots(&self, uid: i64, object_id: &str) -> Vec; - - fn create_snapshot( - &self, - uid: i64, - object_id: &str, - title: String, - snapshot_data: Vec, - ) -> Result<(), PersistenceError>; -} - pub struct CollabSnapshotPlugin { uid: i64, object: CollabObject, - collab_db: Weak, + collab_db: Weak, /// the number of updates on disk when opening the document update_count: Arc, snapshot_per_update: u32, @@ -59,7 +47,7 @@ impl CollabSnapshotPlugin { uid: i64, object: CollabObject, snapshot_persistence: Arc, - collab_db: Weak, + collab_db: Weak, snapshot_per_update: u32, ) -> Self { let state = Arc::new(RwLock::new(GenSnapshotState::Idle)); @@ -147,7 +135,7 @@ impl CollabPlugin for CollabSnapshotPlugin { } } -impl SnapshotPersistence for Arc { +impl SnapshotPersistence for Arc { fn get_snapshots(&self, uid: i64, object_id: &str) -> Vec { self.read_txn().get_snapshots(uid, object_id) } diff --git a/collab-plugins/src/snapshot/mod.rs b/collab-plugins/src/snapshot/mod.rs deleted file mode 100644 index 9c5ff2ea5..000000000 --- a/collab-plugins/src/snapshot/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub use collab_persistence::snapshot::try_encode_snapshot; -pub use plugin::*; - -mod plugin; diff --git a/collab-plugins/tests/disk/mod.rs b/collab-plugins/tests/disk/mod.rs index e0e2fda3d..6f2648ab1 100644 --- a/collab-plugins/tests/disk/mod.rs +++ b/collab-plugins/tests/disk/mod.rs @@ -1,5 +1,8 @@ mod delete_test; mod insert_test; +mod range_test; +mod restore_test; mod script; mod snapshot_test; mod undo_test; +mod util; diff --git a/collab-persistence/tests/persistence/range_test.rs b/collab-plugins/tests/disk/range_test.rs similarity index 96% rename from collab-persistence/tests/persistence/range_test.rs rename to collab-plugins/tests/disk/range_test.rs index 4ebad6969..16b6d5440 100644 --- a/collab-persistence/tests/persistence/range_test.rs +++ b/collab-plugins/tests/disk/range_test.rs @@ -2,12 +2,11 @@ use std::ops::{Deref, Range, RangeTo}; use std::sync::Arc; use std::thread; -use collab_persistence::keys::{clock_from_key, make_doc_update_key, Clock}; -use collab_persistence::kv::{KVEntry, KVStore}; +use crate::disk::util::rocks_db; +use collab_plugins::local_storage::kv::keys::{clock_from_key, make_doc_update_key, Clock}; +use collab_plugins::local_storage::kv::{KVEntry, KVStore}; use smallvec::SmallVec; -use crate::util::rocks_db; - #[tokio::test] async fn rocks_id_test() { let rocks_db = rocks_db().1; @@ -60,7 +59,7 @@ async fn rocks_id_test() { let txn = rocks_db.read_txn(); let value = txn.get([0, 0, 0, 0, 0, 0, 0, 2]).unwrap().unwrap(); - assert_eq!(value, &[0, 1, 3]); + assert_eq!(value.as_ref(), &[0, 1, 3]); } #[tokio::test] diff --git a/collab-persistence/tests/persistence/restore_test.rs b/collab-plugins/tests/disk/restore_test.rs similarity index 85% rename from collab-persistence/tests/persistence/restore_test.rs rename to collab-plugins/tests/disk/restore_test.rs index 7c97fd08f..e5d8670b6 100644 --- a/collab-persistence/tests/persistence/restore_test.rs +++ b/collab-plugins/tests/disk/restore_test.rs @@ -1,11 +1,10 @@ use std::thread; -use collab_persistence::doc::YrsDocAction; -use collab_persistence::kv::rocks_kv::RocksCollabDB; +use crate::disk::util::rocks_db; +use collab_plugins::local_storage::kv::doc::CollabKVAction; +use collab_plugins::CollabKVDB; use yrs::{Doc, GetString, Text, Transact}; -use crate::util::rocks_db; - #[tokio::test] async fn single_thread_test() { let (path, db) = rocks_db(); @@ -14,9 +13,11 @@ async fn single_thread_test() { let doc = Doc::new(); { let txn = doc.transact(); - let store = db.write_txn(); - store.create_new_doc(1, &oid, &txn).unwrap(); - store.commit_transaction().unwrap(); + db.with_write_txn(|db_w_txn| { + db_w_txn.create_new_doc(1, &oid, &txn).unwrap(); + Ok(()) + }) + .unwrap(); } { let text = doc.get_or_insert_text("text"); @@ -32,7 +33,7 @@ async fn single_thread_test() { } drop(db); - let db = RocksCollabDB::open_opt(path, false).unwrap(); + let db = CollabKVDB::open_opt(path, false).unwrap(); for i in 0..100 { let oid = format!("doc_{}", i); let doc = Doc::new(); @@ -79,7 +80,7 @@ async fn rocks_multiple_thread_test() { } drop(db); - let db = RocksCollabDB::open_opt(path, false).unwrap(); + let db = CollabKVDB::open_opt(path, false).unwrap(); for i in 0..100 { let oid = format!("doc_{}", i); let doc = Doc::new(); diff --git a/collab-plugins/tests/disk/script.rs b/collab-plugins/tests/disk/script.rs index 39fcee159..c50eed819 100644 --- a/collab-plugins/tests/disk/script.rs +++ b/collab-plugins/tests/disk/script.rs @@ -6,13 +6,13 @@ use std::time::Duration; use collab::core::collab::MutexCollab; use collab::preclude::*; use collab_entity::{CollabObject, CollabType}; -use collab_persistence::doc::YrsDocAction; -use collab_persistence::kv::rocks_kv::RocksCollabDB; -use collab_plugins::local_storage::rocksdb::RocksdbDiskPlugin; use collab_plugins::local_storage::CollabPersistenceConfig; -use collab_plugins::snapshot::CollabSnapshotPlugin; use yrs::updates::decoder::Decode; +use collab_plugins::local_storage::kv::doc::CollabKVAction; +use collab_plugins::local_storage::rocksdb::rocksdb_plugin::RocksdbDiskPlugin; +use collab_plugins::local_storage::rocksdb::snapshot_plugin::CollabSnapshotPlugin; +use collab_plugins::CollabKVDB; use tempfile::TempDir; use crate::setup_log; @@ -72,7 +72,7 @@ pub struct CollabPersistenceTest { collab_by_id: HashMap>, #[allow(dead_code)] cleaner: Cleaner, - db: Arc, + db: Arc, disk_plugin: Arc, config: CollabPersistenceConfig, } @@ -83,7 +83,7 @@ impl CollabPersistenceTest { let tempdir = TempDir::new().unwrap(); let db_path = tempdir.into_path(); let uid = 1; - let db = Arc::new(RocksCollabDB::open_opt(db_path.clone(), false).unwrap()); + let db = Arc::new(CollabKVDB::open_opt(db_path.clone(), false).unwrap()); let disk_plugin = Arc::new(RocksdbDiskPlugin::new_with_config( uid, Arc::downgrade(&db), @@ -341,10 +341,10 @@ impl CollabPersistenceTest { } } -pub fn disk_plugin(uid: i64) -> (Arc, RocksdbDiskPlugin) { +pub fn disk_plugin(uid: i64) -> (Arc, RocksdbDiskPlugin) { let tempdir = TempDir::new().unwrap(); let path = tempdir.into_path(); - let db = Arc::new(RocksCollabDB::open_opt(path, false).unwrap()); + let db = Arc::new(CollabKVDB::open_opt(path, false).unwrap()); let plugin = RocksdbDiskPlugin::new_with_config( uid, Arc::downgrade(&db), diff --git a/collab-plugins/tests/disk/util.rs b/collab-plugins/tests/disk/util.rs new file mode 100644 index 000000000..a98c407db --- /dev/null +++ b/collab-plugins/tests/disk/util.rs @@ -0,0 +1,11 @@ +use std::path::PathBuf; + +use collab_plugins::CollabKVDB; +use tempfile::TempDir; + +pub fn rocks_db() -> (PathBuf, CollabKVDB) { + let tempdir = TempDir::new().unwrap(); + let path = tempdir.into_path(); + let cloned_path = path.clone(); + (path, CollabKVDB::open_opt(cloned_path, false).unwrap()) +} diff --git a/collab-plugins/tests/main.rs b/collab-plugins/tests/main.rs index 1d4ebeb96..15fe9d24f 100644 --- a/collab-plugins/tests/main.rs +++ b/collab-plugins/tests/main.rs @@ -1,13 +1,11 @@ -use std::sync::Once; - -use tracing_subscriber::fmt::Subscriber; use tracing_subscriber::util::SubscriberInitExt; -use tracing_subscriber::EnvFilter; +#[cfg(not(target_arch = "wasm32"))] mod disk; +#[cfg(not(target_arch = "wasm32"))] pub fn setup_log() { - static START: Once = Once::new(); + static START: std::sync::Once = std::sync::Once::new(); START.call_once(|| { let level = "trace"; let mut filters = vec![]; @@ -17,8 +15,8 @@ pub fn setup_log() { filters.push(format!("collab_plugins={}", level)); std::env::set_var("RUST_LOG", filters.join(",")); - let subscriber = Subscriber::builder() - .with_env_filter(EnvFilter::from_default_env()) + let subscriber = tracing_subscriber::fmt::Subscriber::builder() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) .with_ansi(true) .finish(); subscriber.try_init().unwrap(); diff --git a/collab-user/Cargo.toml b/collab-user/Cargo.toml index 7fdd613b0..b244cb7ec 100644 --- a/collab-user/Cargo.toml +++ b/collab-user/Cargo.toml @@ -4,6 +4,8 @@ version = "0.1.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +crate-type = ["cdylib", "rlib"] [dependencies] serde.workspace = true @@ -15,11 +17,16 @@ tokio = {version = "1.26", features = ["rt", "sync"]} tokio-stream = {version = "0.1.14", features = ["sync"]} tracing.workspace = true parking_lot.workspace = true +getrandom = { version = "0.2", optional = true } [dev-dependencies] assert-json-diff = "2.0.2" -collab-plugins = { workspace = true, features = ["rocksdb_plugin"]} +collab-plugins = { workspace = true } fs_extra = "1.2.0" nanoid = "0.4.0" tempfile = "3.8.0" -tokio = {version = "1.26", features = ["full"]} +tokio = {version = "1.26", features = ["rt", "macros"]} + + +[features] +wasm_build = ["getrandom/js"] \ No newline at end of file diff --git a/collab-user/src/user_awareness.rs b/collab-user/src/user_awareness.rs index 10ef86180..f48408c12 100644 --- a/collab-user/src/user_awareness.rs +++ b/collab-user/src/user_awareness.rs @@ -26,6 +26,7 @@ const APPEARANCE_SETTINGS: &str = "appearance_settings"; pub struct MutexUserAwareness(Arc>); impl MutexUserAwareness { pub fn new(inner: UserAwareness) -> Self { + #[allow(clippy::arc_with_non_send_sync)] Self(Arc::new(Mutex::new(inner))) } } diff --git a/collab-user/tests/util.rs b/collab-user/tests/util.rs index ee695ae54..1cc39a173 100644 --- a/collab-user/tests/util.rs +++ b/collab-user/tests/util.rs @@ -5,8 +5,8 @@ use std::time::Duration; use anyhow::Result; use collab::preclude::CollabBuilder; -use collab_plugins::kv::rocks_kv::RocksCollabDB; -use collab_plugins::local_storage::rocksdb::RocksdbDiskPlugin; +use collab_plugins::local_storage::rocksdb::rocksdb_plugin::RocksdbDiskPlugin; +use collab_plugins::CollabKVDB; use collab_user::core::{ MutexUserAwareness, RemindersChangeSender, UserAwareness, UserAwarenessNotifier, }; @@ -36,7 +36,7 @@ impl UserAwarenessTest { let tempdir = TempDir::new().unwrap(); let path = tempdir.into_path(); - let db = Arc::new(RocksCollabDB::open(path.clone()).unwrap()); + let db = Arc::new(CollabKVDB::open(path.clone()).unwrap()); let disk_plugin = RocksdbDiskPlugin::new(uid, Arc::downgrade(&db), None); let cleaner: Cleaner = Cleaner::new(path); diff --git a/collab/Cargo.toml b/collab/Cargo.toml index 25e4b1a5b..89a9b8e97 100644 --- a/collab/Cargo.toml +++ b/collab/Cargo.toml @@ -4,6 +4,8 @@ version = "0.1.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +crate-type = ["cdylib", "rlib"] [dependencies] yrs.workspace = true @@ -20,9 +22,8 @@ async-trait.workspace = true bincode = "1.3.3" serde_repr = "0.1" - [dev-dependencies] -tokio = { version = "1.26", features = ["full"] } +tokio = { version = "1.26", features = ["rt"] } tempfile = "3.8.0" collab = { path = "", features = ["default"] } nanoid = "0.4.0" @@ -32,4 +33,5 @@ tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } [features] default = [] -async-plugin = [] \ No newline at end of file +async-plugin = [] +wasm_build = [] \ No newline at end of file diff --git a/collab/src/core/any_array.rs b/collab/src/core/any_array.rs index ca6f93bae..0b861bcbc 100644 --- a/collab/src/core/any_array.rs +++ b/collab/src/core/any_array.rs @@ -141,7 +141,6 @@ impl<'a, 'b> ArrayMapUpdate<'a, 'b> { F: FnOnce(AnyMap) -> AnyMap, { if let Some(pos) = self.index_of(id) { - let pos = pos; if let YrsValue::YMap(map_ref) = self.array_ref.get(self.txn, pos).unwrap() { let any_map = AnyMap::from_map_ref(self.txn, &map_ref); f(any_map).fill_map_ref(self.txn, &map_ref); diff --git a/collab/src/core/collab.rs b/collab/src/core/collab.rs index f8abd0e54..950542b93 100644 --- a/collab/src/core/collab.rs +++ b/collab/src/core/collab.rs @@ -810,12 +810,14 @@ impl Deref for Plugins { } } +#[allow(clippy::arc_with_non_send_sync)] #[derive(Clone)] pub struct MutexCollab(Arc>); impl MutexCollab { pub fn new(origin: CollabOrigin, object_id: &str, plugins: Vec>) -> Self { let collab = Collab::new_with_origin(origin, object_id, plugins); + #[allow(clippy::arc_with_non_send_sync)] MutexCollab(Arc::new(Mutex::new(collab))) } @@ -826,10 +828,12 @@ impl MutexCollab { plugins: Vec>, ) -> Result { let collab = Collab::new_with_doc_state(origin, object_id, collab_doc_state, plugins)?; + #[allow(clippy::arc_with_non_send_sync)] Ok(MutexCollab(Arc::new(Mutex::new(collab)))) } pub fn from_collab(collab: Collab) -> Self { + #[allow(clippy::arc_with_non_send_sync)] MutexCollab(Arc::new(Mutex::new(collab))) } diff --git a/collab/tests/collab_test/helper.rs b/collab/tests/collab_test/helper.rs index 59d958aa0..6a18cb258 100644 --- a/collab/tests/collab_test/helper.rs +++ b/collab/tests/collab_test/helper.rs @@ -112,6 +112,7 @@ mod tests { } } +#[allow(clippy::items_after_test_module)] pub fn setup_log() { static START: Once = Once::new(); START.call_once(|| { diff --git a/rust-toolchain.toml b/rust-toolchain.toml index f400973ca..6d833ff50 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "1.70" +channel = "1.75" From 1883988532108765f1f0baf99da9532c15a524bb Mon Sep 17 00:00:00 2001 From: "Nathan.fooo" <86001920+appflowy@users.noreply.github.com> Date: Sat, 6 Jan 2024 11:54:04 +0800 Subject: [PATCH 05/13] feat: subscribe indexing content (#132) * feat: subscribe indexing view * chore: fmt * chore: remove sending view index content in views map * feat: expose type of index content * feat: document text for indexing * chore: clippy --------- Co-authored-by: Jiraffe7 --- collab-document/src/blocks/text.rs | 22 ++++++++ collab-document/src/document.rs | 54 ++++++++++++++++++- .../tests/document/document_test.rs | 43 ++++++++++++++- collab-folder/src/folder.rs | 10 +++- collab-folder/src/folder_observe.rs | 19 ++++++- collab-folder/src/view.rs | 25 +++++++++ collab-folder/tests/folder_test/view_test.rs | 36 ++++++++++++- collab/src/core/collab.rs | 22 +++++++- collab/src/core/collab_search.rs | 1 + collab/src/core/mod.rs | 1 + 10 files changed, 226 insertions(+), 7 deletions(-) create mode 100644 collab/src/core/collab_search.rs diff --git a/collab-document/src/blocks/text.rs b/collab-document/src/blocks/text.rs index 432af7692..229f94b88 100644 --- a/collab-document/src/blocks/text.rs +++ b/collab-document/src/blocks/text.rs @@ -69,4 +69,26 @@ impl TextOperation { }) .collect() } + + /// get all text delta and join as string + pub fn stringify_all_text_delta(&self) -> HashMap { + let txn = self.root.transact(); + self + .root + .iter(&txn) + .filter_map(|(k, _)| { + self.get_delta_with_txn(&txn, k).map(|delta| { + let text: Vec = delta + .iter() + .filter_map(|d| match d { + TextDelta::Inserted(s, _) => Some(s.clone()), + _ => None, + }) + .collect(); + let text = text.join(""); + (k.to_string(), text) + }) + }) + .collect() + } } diff --git a/collab-document/src/document.rs b/collab-document/src/document.rs index 664e6db66..fb3f80c63 100644 --- a/collab-document/src/document.rs +++ b/collab-document/src/document.rs @@ -6,13 +6,14 @@ use collab::core::collab::{CollabDocState, MutexCollab}; use collab::core::collab_state::SyncState; use collab::core::origin::CollabOrigin; use collab::preclude::*; +use serde::{Deserialize, Serialize}; use serde_json::Value; use tokio_stream::wrappers::WatchStream; use crate::blocks::{ deserialize_text_delta, Block, BlockAction, BlockActionPayload, BlockActionType, BlockEvent, BlockOperation, ChildrenOperation, DocumentData, DocumentMeta, RootDeepSubscription, - TextOperation, + TextOperation, EXTERNAL_TYPE_TEXT, }; use crate::error::DocumentError; @@ -642,3 +643,54 @@ impl Document { } } } + +/// Represents a the index content of a document. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct DocumentIndexContent { + pub page_id: String, + pub text: String, +} + +impl From<&Document> for DocumentIndexContent { + fn from(value: &Document) -> Self { + let collab_guard = value.inner.lock(); + let txn = collab_guard.transact(); + let page_id = value + .root + .get_str_with_txn(&txn, PAGE_ID) + .expect("document should have page_id"); + + drop(txn); + drop(collab_guard); + + let blocks = value.block_operation.get_all_blocks(); + let children_map = value.children_operation.get_all_children(); + let text_map = value.text_operation.stringify_all_text_delta(); + + let page_block = blocks + .get(&page_id) + .expect("document data should contain page block"); + let children_key = &page_block.children; + let children_ids = children_map + .get(children_key) + .expect("children map should contain page's children key"); + + let text: Vec<_> = children_ids + .iter() + .filter_map(|id| blocks.get(id)) // get block of child + .filter_map(|block| { // get external id of blocks with external type text + let Some(ty) = block.external_type.as_ref() else { return None;}; + if ty == EXTERNAL_TYPE_TEXT { + return block.external_id.as_ref(); + } + None + }) + .filter_map(|ext_id| text_map.get(ext_id).filter(|t| !t.is_empty())) // get text of block + .cloned() + .collect(); + + let text = text.join(" "); // all text of document + + Self { page_id, text } + } +} diff --git a/collab-document/tests/document/document_test.rs b/collab-document/tests/document/document_test.rs index 6cd0cef39..5f429869b 100644 --- a/collab-document/tests/document/document_test.rs +++ b/collab-document/tests/document/document_test.rs @@ -1,7 +1,12 @@ -use collab_document::blocks::{Block, BlockAction, BlockActionPayload, BlockActionType}; +use collab_document::{ + blocks::{Block, BlockAction, BlockActionPayload, BlockActionType}, + document::DocumentIndexContent, +}; use nanoid::nanoid; -use crate::util::{apply_actions, get_document_data, open_document_with_db, DocumentTest}; +use crate::util::{ + apply_actions, get_document_data, insert_block, open_document_with_db, DocumentTest, +}; #[tokio::test] async fn insert_block_with_empty_parent_id_and_empty_prev_id() { @@ -68,3 +73,37 @@ async fn reopen_document() { let (page_id2, _, _) = get_document_data(&document); assert_eq!(page_id, page_id2); } + +#[tokio::test] +async fn document_index_data_from_document() { + let doc_id = "1"; + let test = DocumentTest::new(1, doc_id).await; + let document = test.document; + + let (page_id, _blocks, _children_map) = get_document_data(&document); + let index_content = DocumentIndexContent::from(&document); + assert_eq!(index_content.page_id, page_id); + assert_eq!(index_content.text, ""); + + let block_id = nanoid!(10); + let text_id = nanoid!(10); + let block = Block { + id: block_id, + ty: "paragraph".to_owned(), + parent: page_id.clone(), + children: "".to_string(), + external_id: Some(text_id.clone()), + external_type: Some("text".to_owned()), + data: Default::default(), + }; + + insert_block(&document, block, "".to_string()).unwrap(); + document.create_text( + &text_id, + r#"[{"insert": "Hello "}, {"insert": "world!"}]"#.to_owned(), + ); + + let index_content = DocumentIndexContent::from(&document); + assert_eq!(index_content.page_id, page_id); + assert_eq!(index_content.text, "Hello world!"); +} diff --git a/collab-folder/src/folder.rs b/collab-folder/src/folder.rs index 3f6803e47..808ec584b 100644 --- a/collab-folder/src/folder.rs +++ b/collab-folder/src/folder.rs @@ -2,7 +2,7 @@ use std::rc::Rc; use std::sync::Arc; use anyhow::Error; -use collab::core::collab::{CollabDocState, MutexCollab}; +use collab::core::collab::{CollabDocState, IndexContentReceiver, MutexCollab}; use collab::core::collab_plugin::EncodedCollab; use collab::core::collab_state::{SnapshotState, SyncState}; pub use collab::core::origin::CollabOrigin; @@ -140,6 +140,10 @@ impl Folder { self.inner.lock().subscribe_snapshot_state() } + pub fn subscribe_index_content(&self) -> IndexContentReceiver { + self.inner.lock().subscribe_index_content() + } + /// Returns the doc state and the state vector. pub fn encode_collab_v1(&self) -> EncodedCollab { self.inner.lock().encode_collab_v1() @@ -564,6 +568,7 @@ fn create_folder>( ) -> Folder { let uid = uid.into(); let collab_guard = collab.lock(); + let index_json_sender = collab_guard.index_json_sender.clone(); let (folder, views, section, meta, subscription) = collab_guard.with_origin_transact_mut(|txn| { // create the folder let mut folder = collab_guard.insert_map_with_txn_if_not_exist(txn, FOLDER); @@ -593,6 +598,7 @@ fn create_folder>( .map(|notifier| notifier.view_change_tx.clone()), view_relations, section.clone(), + index_json_sender, )); if let Some(folder_data) = folder_data { @@ -642,6 +648,7 @@ fn open_folder>( ) -> Option { let uid = uid.into(); let collab_guard = collab.lock(); + let index_json_sender = collab_guard.index_json_sender.clone(); let txn = collab_guard.transact(); // create the folder @@ -672,6 +679,7 @@ fn open_folder>( .map(|notifier| notifier.view_change_tx.clone()), view_relations, section_map.clone(), + index_json_sender, )); drop(txn); drop(collab_guard); diff --git a/collab-folder/src/folder_observe.rs b/collab-folder/src/folder_observe.rs index cce0be4be..7b41675e7 100644 --- a/collab-folder/src/folder_observe.rs +++ b/collab-folder/src/folder_observe.rs @@ -2,14 +2,16 @@ use std::collections::HashMap; use std::rc::Rc; use std::sync::Arc; +use collab::core::collab::{IndexContent, IndexContentSender}; use collab::preclude::{ DeepEventsSubscription, DeepObservable, EntryChange, Event, MapRefWrapper, ToJson, YrsValue, }; use parking_lot::RwLock; +use serde_json::json; use tokio::sync::broadcast; use crate::section::SectionMap; -use crate::{view_from_map_ref, UserId, View, ViewRelations}; +use crate::{view_from_map_ref, UserId, View, ViewIndexContent, ViewRelations}; #[derive(Debug, Clone)] pub enum ViewChange { @@ -58,6 +60,7 @@ pub(crate) fn subscribe_view_change( change_tx: ViewChangeSender, view_relations: Rc, section_map: Rc, + index_sender: IndexContentSender, ) -> DeepEventsSubscription { let uid = uid.clone(); root.observe_deep(move |txn, events| { @@ -77,6 +80,11 @@ pub(crate) fn subscribe_view_change( view_cache .write() .insert(view.id.clone(), Arc::new(view.clone())); + + // Send indexing view + let index_content = ViewIndexContent::from(&view); + let _ = index_sender.send(IndexContent::Create(json!(index_content))); + let _ = change_tx.send(ViewChange::DidCreateView { view }); } } @@ -88,6 +96,11 @@ pub(crate) fn subscribe_view_change( view_cache .write() .insert(view.id.clone(), Arc::new(view.clone())); + + // Update indexing view + let index_content = ViewIndexContent::from(&view); + let _ = index_sender.send(IndexContent::Update(json!(index_content))); + let _ = change_tx.send(ViewChange::DidUpdate { view }); } }, @@ -99,6 +112,10 @@ pub(crate) fn subscribe_view_change( .collect::>>(); if !views.is_empty() { + // Delete indexing views + let delete_ids: Vec = views.iter().map(|v| v.id.to_owned()).collect(); + let _ = index_sender.send(IndexContent::Delete(delete_ids)); + let _ = change_tx.send(ViewChange::DidDeleteView { views }); } }, diff --git a/collab-folder/src/view.rs b/collab-folder/src/view.rs index f60271a34..5a475d180 100644 --- a/collab-folder/src/view.rs +++ b/collab-folder/src/view.rs @@ -3,6 +3,7 @@ use std::rc::Rc; use std::sync::Arc; use anyhow::bail; +use collab::core::collab::IndexContentSender; use collab::preclude::{ Any, DeepEventsSubscription, MapRef, MapRefExtension, MapRefWrapper, ReadTxn, TransactionMut, }; @@ -55,6 +56,7 @@ impl ViewsMap { change_tx: Option, view_relations: Rc, section_map: Rc, + index_json_sender: IndexContentSender, ) -> ViewsMap { let view_cache = Arc::new(RwLock::new(HashMap::new())); let subscription = change_tx.as_ref().map(|change_tx| { @@ -65,6 +67,7 @@ impl ViewsMap { change_tx.clone(), view_relations.clone(), section_map.clone(), + index_json_sender.clone(), ) }); Self { @@ -650,6 +653,28 @@ pub struct View { pub last_edited_by: Option, // user id } +/// Represents a the index of a view. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct ViewIndexContent { + pub id: String, + pub parent_view_id: String, + pub name: String, + pub is_favorite: bool, + pub layout: ViewLayout, +} + +impl From<&View> for ViewIndexContent { + fn from(value: &View) -> Self { + Self { + id: value.id.clone(), + parent_view_id: value.parent_view_id.clone(), + name: value.name.clone(), + is_favorite: value.is_favorite, + layout: value.layout.clone(), + } + } +} + impl Default for View { fn default() -> Self { Self { diff --git a/collab-folder/tests/folder_test/view_test.rs b/collab-folder/tests/folder_test/view_test.rs index 8a49d4b0b..67e578c7c 100644 --- a/collab-folder/tests/folder_test/view_test.rs +++ b/collab-folder/tests/folder_test/view_test.rs @@ -1,11 +1,13 @@ use crate::util::{create_folder_with_workspace, make_test_view}; -use collab_folder::{timestamp, IconType, UserId, ViewIcon}; +use collab::core::collab::IndexContent; +use collab_folder::{timestamp, IconType, UserId, ViewIcon, ViewIndexContent}; #[tokio::test] async fn create_view_test() { let uid = UserId::from(1); let folder_test = create_folder_with_workspace(uid.clone(), "w1").await; let o_view = make_test_view("v1", "w1", vec![]); + // Insert a new view folder_test.insert_view(o_view.clone(), None); let r_view = folder_test.views.get_view("v1").unwrap(); @@ -340,3 +342,35 @@ async fn check_created_and_edited_time_test() { assert_eq!(v1.last_edited_by.unwrap(), uid.as_i64()); assert_eq!(v1.last_edited_time, v1.created_at); } +#[tokio::test] +async fn create_view_and_then_sub_index_content_test() { + let uid = UserId::from(1); + let folder_test = create_folder_with_workspace(uid.clone(), "w1").await; + let mut index_content_rx = folder_test.subscribe_index_content(); + let o_view = make_test_view("v1", "w1", vec![]); + + // subscribe the index content + let (tx, rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + if let IndexContent::Create(json) = index_content_rx.recv().await.unwrap() { + tx.send(serde_json::from_value::(json).unwrap()) + .unwrap(); + } else { + panic!("expected IndexContent::Create"); + } + }); + + // Insert a new view + folder_test.insert_view(o_view.clone(), None); + + let r_view = folder_test.views.get_view("v1").unwrap(); + assert_eq!(o_view.name, r_view.name); + assert_eq!(o_view.parent_view_id, r_view.parent_view_id); + assert_eq!(o_view.children, r_view.children); + + // check the index content + let index_content = rx.await.unwrap(); + assert_eq!(index_content.id, o_view.id); + assert_eq!(index_content.parent_view_id, o_view.parent_view_id); + assert_eq!(index_content.name, o_view.name); +} diff --git a/collab/src/core/collab.rs b/collab/src/core/collab.rs index 950542b93..bd7a43e37 100644 --- a/collab/src/core/collab.rs +++ b/collab/src/core/collab.rs @@ -8,6 +8,7 @@ use std::vec::IntoIter; use parking_lot::{Mutex, RwLock}; use serde::de::DeserializeOwned; use serde::Serialize; + use tokio_stream::wrappers::WatchStream; use tracing::error; use yrs::block::Prelim; @@ -42,6 +43,14 @@ type AfterTransactionSubscription = Subscription; pub type MapSubscription = Subscription; +#[derive(Debug, Clone)] +pub enum IndexContent { + Create(serde_json::Value), + Update(serde_json::Value), + Delete(Vec), +} +pub type IndexContentSender = tokio::sync::broadcast::Sender; +pub type IndexContentReceiver = tokio::sync::broadcast::Receiver; /// A [Collab] is a wrapper around a [Doc] and [Awareness] that provides a set /// of helper methods for interacting with the [Doc] and [Awareness]. The [MutexCollab] /// is a thread-safe wrapper around the [Collab]. @@ -74,6 +83,7 @@ pub struct Collab { undo_manager: Mutex>, update_subscription: RwLock>, after_txn_subscription: RwLock>, + pub index_json_sender: IndexContentSender, } pub fn make_yrs_doc() -> Doc { @@ -123,7 +133,6 @@ impl Collab { let plugins = Plugins::new(plugins); let state = Arc::new(State::new(&object_id)); let awareness = Awareness::new(doc.clone()); - Self { origin, object_id, @@ -136,6 +145,7 @@ impl Collab { state, update_subscription: Default::default(), after_txn_subscription: Default::default(), + index_json_sender: tokio::sync::broadcast::channel(100).0, } } @@ -156,6 +166,16 @@ impl Collab { WatchStream::new(self.state.snapshot_state_notifier.subscribe()) } + /// Subscribes to the `IndexJson` associated with a `Collab` object. + /// + /// `IndexJson` is a JSON object containing data used for indexing purposes. The structure and + /// content of this data may vary between different collaborative objects derived from `Collab`. + /// The interpretation of `IndexJson` is specific to the subscriber, as only they know how to + /// process and utilize the contained indexing information. + pub fn subscribe_index_content(&self) -> IndexContentReceiver { + self.index_json_sender.subscribe() + } + /// Returns the [Doc] associated with the [Collab]. pub fn get_doc(&self) -> &Doc { &self.doc diff --git a/collab/src/core/collab_search.rs b/collab/src/core/collab_search.rs new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/collab/src/core/collab_search.rs @@ -0,0 +1 @@ + diff --git a/collab/src/core/mod.rs b/collab/src/core/mod.rs index 5759388fc..885ae2b55 100644 --- a/collab/src/core/mod.rs +++ b/collab/src/core/mod.rs @@ -4,6 +4,7 @@ pub mod array_wrapper; pub mod awareness; pub mod collab; pub mod collab_plugin; +mod collab_search; mod collab_serde; pub mod collab_state; pub mod map_wrapper; From b566a65b3eb3d0abff7c09371cad309341865cde Mon Sep 17 00:00:00 2001 From: "Nathan.fooo" <86001920+appflowy@users.noreply.github.com> Date: Sat, 6 Jan 2024 12:10:25 +0800 Subject: [PATCH 06/13] refactor: remove snapshot (#144) * chore: remove snapshot * chore: remove snapshot * chore: using Zstd as compression type * chore: set rocksdb log file * chore: disable compress type * chore: compress type * chore: compress type * chore: compress type --- Cargo.lock | 1 - collab-database/src/blocks/block.rs | 2 +- collab-database/src/user/user_db.rs | 4 +- collab-database/tests/database_test/helper.rs | 4 +- collab-database/tests/user_test/helper.rs | 7 +- .../tests/blocks/block_test_core.rs | 2 +- collab-document/tests/util.rs | 4 +- collab-folder/tests/folder_test/util.rs | 4 +- collab-plugins/src/lib.rs | 6 +- .../src/local_storage/kv/snapshot.rs | 5 +- .../src/local_storage/rocksdb/kv_impl.rs | 5 + .../local_storage/rocksdb/rocksdb_plugin.rs | 35 +- .../local_storage/rocksdb/snapshot_plugin.rs | 90 +----- collab-plugins/tests/disk/mod.rs | 1 - collab-plugins/tests/disk/script.rs | 125 +------- collab-plugins/tests/disk/snapshot_test.rs | 302 ------------------ collab-user/tests/util.rs | 2 +- 17 files changed, 38 insertions(+), 561 deletions(-) delete mode 100644 collab-plugins/tests/disk/snapshot_test.rs diff --git a/Cargo.lock b/Cargo.lock index e3b72da2f..2ddbe9b0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -853,7 +853,6 @@ dependencies = [ "glob", "libc", "libz-sys", - "zstd-sys", ] [[package]] diff --git a/collab-database/src/blocks/block.rs b/collab-database/src/blocks/block.rs index 7463bfa12..5fbe4cc5e 100644 --- a/collab-database/src/blocks/block.rs +++ b/collab-database/src/blocks/block.rs @@ -247,7 +247,7 @@ impl Block { CollabType::DatabaseRow, self.collab_db.clone(), doc_state, - &config, + config, ) } } diff --git a/collab-database/src/user/user_db.rs b/collab-database/src/user/user_db.rs index fb87389e1..78dcf0d57 100644 --- a/collab-database/src/user/user_db.rs +++ b/collab-database/src/user/user_db.rs @@ -50,7 +50,7 @@ pub trait DatabaseCollabService: Send + Sync + 'static { object_type: CollabType, collab_db: Weak, collab_doc_state: CollabDocState, - config: &CollabPersistenceConfig, + config: CollabPersistenceConfig, ) -> Arc; } @@ -345,7 +345,7 @@ impl WorkspaceDatabase { CollabType::Database, self.collab_db.clone(), doc_state, - &self.config, + self.config.clone(), ) } diff --git a/collab-database/tests/database_test/helper.rs b/collab-database/tests/database_test/helper.rs index 026a30a40..23ea8e9dd 100644 --- a/collab-database/tests/database_test/helper.rs +++ b/collab-database/tests/database_test/helper.rs @@ -89,7 +89,7 @@ pub async fn create_database_with_db( CollabType::Database, Arc::downgrade(&collab_db), CollabDocState::default(), - &CollabPersistenceConfig::default(), + CollabPersistenceConfig::default(), ); let context = DatabaseContext { uid, @@ -125,7 +125,7 @@ pub fn restore_database_from_db( CollabType::Database, Arc::downgrade(&collab_db), CollabDocState::default(), - &CollabPersistenceConfig::default(), + CollabPersistenceConfig::default(), ); let context = DatabaseContext { uid, diff --git a/collab-database/tests/user_test/helper.rs b/collab-database/tests/user_test/helper.rs index 1526c1529..346d9c548 100644 --- a/collab-database/tests/user_test/helper.rs +++ b/collab-database/tests/user_test/helper.rs @@ -76,7 +76,7 @@ impl DatabaseCollabService for TestUserDatabaseCollabBuilderImpl { _object_type: CollabType, collab_db: Weak, doc_state: CollabDocState, - config: &CollabPersistenceConfig, + config: CollabPersistenceConfig, ) -> Arc { let collab = CollabBuilder::new(uid, object_id) .with_device_id("1") @@ -85,7 +85,6 @@ impl DatabaseCollabService for TestUserDatabaseCollabBuilderImpl { uid, collab_db, config.clone(), - None, )) .build() .unwrap(); @@ -114,7 +113,7 @@ pub async fn workspace_database_test_with_config( CollabType::WorkspaceDatabase, Arc::downgrade(&collab_db), CollabDocState::default(), - &config, + config.clone(), ); let inner = WorkspaceDatabase::open(uid, collab, Arc::downgrade(&collab_db), config, builder); WorkspaceDatabaseTest { @@ -140,7 +139,7 @@ pub async fn workspace_database_with_db( CollabType::WorkspaceDatabase, collab_db.clone(), CollabDocState::default(), - &config, + config.clone(), ); WorkspaceDatabase::open(uid, collab, collab_db, config, builder) } diff --git a/collab-document/tests/blocks/block_test_core.rs b/collab-document/tests/blocks/block_test_core.rs index d9582f7c5..bde8b3855 100644 --- a/collab-document/tests/blocks/block_test_core.rs +++ b/collab-document/tests/blocks/block_test_core.rs @@ -26,7 +26,7 @@ impl BlockTestCore { pub async fn new() -> Self { let db = document_storage(); let doc_id = "1"; - let disk_plugin = RocksdbDiskPlugin::new(1, Arc::downgrade(&db), None); + let disk_plugin = RocksdbDiskPlugin::new(1, Arc::downgrade(&db)); let collab = CollabBuilder::new(1, doc_id) .with_plugin(disk_plugin) .with_device_id("1") diff --git a/collab-document/tests/util.rs b/collab-document/tests/util.rs index 94585dc69..09e9e24f9 100644 --- a/collab-document/tests/util.rs +++ b/collab-document/tests/util.rs @@ -31,7 +31,7 @@ impl DocumentTest { } pub async fn new_with_db(uid: i64, doc_id: &str, db: Arc) -> Self { - let disk_plugin = RocksdbDiskPlugin::new(uid, Arc::downgrade(&db), None); + let disk_plugin = RocksdbDiskPlugin::new(uid, Arc::downgrade(&db)); let collab = CollabBuilder::new(1, doc_id) .with_plugin(disk_plugin) .with_device_id("1") @@ -103,7 +103,7 @@ impl Deref for DocumentTest { pub async fn open_document_with_db(uid: i64, doc_id: &str, db: Arc) -> Document { setup_log(); - let disk_plugin = RocksdbDiskPlugin::new(uid, Arc::downgrade(&db), None); + let disk_plugin = RocksdbDiskPlugin::new(uid, Arc::downgrade(&db)); let collab = CollabBuilder::new(uid, doc_id) .with_plugin(disk_plugin) .with_device_id("1") diff --git a/collab-folder/tests/folder_test/util.rs b/collab-folder/tests/folder_test/util.rs index 2c2f733bf..a4803963f 100644 --- a/collab-folder/tests/folder_test/util.rs +++ b/collab-folder/tests/folder_test/util.rs @@ -52,7 +52,7 @@ pub async fn create_folder_with_data( let path = tempdir.into_path(); let db = Arc::new(CollabKVDB::open_opt(path.clone(), false).unwrap()); - let disk_plugin = RocksdbDiskPlugin::new(uid.as_i64(), Arc::downgrade(&db), None); + let disk_plugin = RocksdbDiskPlugin::new(uid.as_i64(), Arc::downgrade(&db)); let cleaner: Cleaner = Cleaner::new(path); let collab = CollabBuilder::new(uid.as_i64(), workspace_id) @@ -80,7 +80,7 @@ pub async fn create_folder_with_data( pub async fn open_folder_with_db(uid: UserId, object_id: &str, db_path: PathBuf) -> FolderTest { let db = Arc::new(CollabKVDB::open_opt(db_path.clone(), false).unwrap()); - let disk_plugin = RocksdbDiskPlugin::new(uid.as_i64(), Arc::downgrade(&db), None); + let disk_plugin = RocksdbDiskPlugin::new(uid.as_i64(), Arc::downgrade(&db)); let cleaner: Cleaner = Cleaner::new(db_path); let collab = CollabBuilder::new(1, object_id) .with_plugin(disk_plugin) diff --git a/collab-plugins/src/lib.rs b/collab-plugins/src/lib.rs index 8c1db34ae..b90190e98 100644 --- a/collab-plugins/src/lib.rs +++ b/collab-plugins/src/lib.rs @@ -1,10 +1,8 @@ pub mod local_storage; -#[cfg(feature = "postgres_plugin")] +#[cfg(all(feature = "postgres_plugin", not(target_arch = "wasm32")))] pub mod cloud_storage; pub mod connect_state; #[cfg(not(target_arch = "wasm32"))] -use crate::local_storage::rocksdb::kv_impl::RocksStore; -#[cfg(not(target_arch = "wasm32"))] -pub type CollabKVDB = RocksStore; +pub type CollabKVDB = local_storage::rocksdb::kv_impl::RocksStore; diff --git a/collab-plugins/src/local_storage/kv/snapshot.rs b/collab-plugins/src/local_storage/kv/snapshot.rs index 392bdff10..4cffa3ac0 100644 --- a/collab-plugins/src/local_storage/kv/snapshot.rs +++ b/collab-plugins/src/local_storage/kv/snapshot.rs @@ -4,6 +4,7 @@ use std::panic::AssertUnwindSafe; use crate::local_storage::kv::keys::*; use crate::local_storage::kv::*; +use collab_entity::CollabType; use serde::{Deserialize, Serialize}; use yrs::updates::encoder::{Encoder, EncoderV1}; use yrs::{ReadTxn, Snapshot}; @@ -189,8 +190,8 @@ pub trait SnapshotPersistence: Send + Sync { &self, uid: i64, object_id: &str, - title: String, - snapshot_data: Vec, + collab_type: &CollabType, + encoded_v1: Vec, ) -> Result<(), PersistenceError>; } diff --git a/collab-plugins/src/local_storage/rocksdb/kv_impl.rs b/collab-plugins/src/local_storage/rocksdb/kv_impl.rs index 5ba71b311..49ca7178e 100644 --- a/collab-plugins/src/local_storage/rocksdb/kv_impl.rs +++ b/collab-plugins/src/local_storage/rocksdb/kv_impl.rs @@ -29,6 +29,11 @@ impl RocksStore { // On the other hand, setting it too high could lead to excessive CPU and I/O usage, impacting the overall // performance of the system. db_opts.set_max_background_jobs(4); + db_opts.set_compression_type(rocksdb::DBCompressionType::Zstd); + db_opts.set_blob_compression_type(rocksdb::DBCompressionType::Zstd); + db_opts.set_recycle_log_file_num(5); + db_opts.set_keep_log_file_num(5); + db_opts.set_db_log_dir(path.as_ref().join("logs")); db_opts.create_if_missing(true); let open_result = TransactionDB::::open(&db_opts, &txn_db_opts, &path); diff --git a/collab-plugins/src/local_storage/rocksdb/rocksdb_plugin.rs b/collab-plugins/src/local_storage/rocksdb/rocksdb_plugin.rs index f76fb3bf8..91fcb0c75 100644 --- a/collab-plugins/src/local_storage/rocksdb/rocksdb_plugin.rs +++ b/collab-plugins/src/local_storage/rocksdb/rocksdb_plugin.rs @@ -30,7 +30,6 @@ pub struct RocksdbDiskPlugin { initial_update_count: Arc, update_count: Arc, config: CollabPersistenceConfig, - backup: Option>, } impl Deref for RocksdbDiskPlugin { @@ -42,12 +41,7 @@ impl Deref for RocksdbDiskPlugin { } impl RocksdbDiskPlugin { - pub fn new_with_config( - uid: i64, - db: Weak, - config: CollabPersistenceConfig, - backup: Option>, - ) -> Self { + pub fn new_with_config(uid: i64, db: Weak, config: CollabPersistenceConfig) -> Self { let initial_update_count = Arc::new(AtomicU32::new(0)); let update_count = Arc::new(AtomicU32::new(0)); let did_load = Arc::new(AtomicBool::new(false)); @@ -58,12 +52,11 @@ impl RocksdbDiskPlugin { initial_update_count, update_count, config, - backup, } } - pub fn new(uid: i64, db: Weak, backup: Option>) -> Self { - Self::new_with_config(uid, db, CollabPersistenceConfig::default(), backup) + pub fn new(uid: i64, db: Weak) -> Self { + Self::new_with_config(uid, db, CollabPersistenceConfig::default()) } fn increase_count(&self) -> u32 { @@ -86,10 +79,6 @@ impl RocksdbDiskPlugin { encoded.state_vector.to_vec(), encoded.doc_state.to_vec(), )?; - - if let Some(backup) = &self.backup { - backup_doc(self.uid, backup, object_id, encoded); - } } Ok(()) @@ -183,21 +172,3 @@ impl CollabPlugin for RocksdbDiskPlugin { } } } - -fn backup_doc(uid: i64, backup: &Arc, object_id: &str, encoded: EncodedCollab) { - let weak_backup = Arc::downgrade(backup); - let object_id = object_id.to_string(); - tokio::spawn(async move { - let _ = tokio::task::spawn_blocking(move || { - if let Some(backup) = weak_backup.upgrade() { - match backup.save_doc(uid, &object_id, encoded) { - Ok(_) => {}, - Err(err) => { - error!("rocksdb backup save doc failed: {}", err); - }, - } - } - }) - .await; - }); -} diff --git a/collab-plugins/src/local_storage/rocksdb/snapshot_plugin.rs b/collab-plugins/src/local_storage/rocksdb/snapshot_plugin.rs index 02d7b6a36..82cb62f65 100644 --- a/collab-plugins/src/local_storage/rocksdb/snapshot_plugin.rs +++ b/collab-plugins/src/local_storage/rocksdb/snapshot_plugin.rs @@ -1,18 +1,15 @@ -use std::panic; -use std::panic::AssertUnwindSafe; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, Weak}; use crate::local_storage::kv::doc::CollabKVAction; -use crate::local_storage::kv::snapshot::{CollabSnapshot, SnapshotAction, SnapshotPersistence}; +use crate::local_storage::kv::snapshot::{CollabSnapshot, SnapshotPersistence}; use crate::local_storage::kv::PersistenceError; use crate::CollabKVDB; use collab::preclude::{Collab, CollabPlugin}; use collab_entity::CollabObject; use parking_lot::RwLock; -use similar::{ChangeTag, TextDiff}; -use yrs::updates::decoder::Decode; -use yrs::{ReadTxn, StateVector, TransactionMut, Update}; + +use yrs::{ReadTxn, StateVector, TransactionMut}; #[derive(Clone, Debug)] enum GenSnapshotState { @@ -112,12 +109,12 @@ impl CollabPlugin for CollabSnapshotPlugin { // Generate the snapshot let txn = snapshot_collab.transact(); - let snapshot_data = txn.encode_state_as_update_v1(&StateVector::default()); + let encoded_v1 = txn.encode_state_as_update_v1(&StateVector::default()); match snapshot_persistence.create_snapshot( uid, &object.object_id, - object.collab_type.to_string(), - snapshot_data, + &object.collab_type, + encoded_v1, ) { Ok(_) => *state.write() = GenSnapshotState::Idle, Err(e) => { @@ -134,78 +131,3 @@ impl CollabPlugin for CollabSnapshotPlugin { } } } - -impl SnapshotPersistence for Arc { - fn get_snapshots(&self, uid: i64, object_id: &str) -> Vec { - self.read_txn().get_snapshots(uid, object_id) - } - - fn create_snapshot( - &self, - uid: i64, - object_id: &str, - _title: String, - snapshot_data: Vec, - ) -> Result<(), PersistenceError> { - self.with_write_txn(|txn| { - txn.create_snapshot_with_data(uid, object_id, snapshot_data)?; - Ok(()) - }) - } -} - -pub fn calculate_snapshot_diff( - uid: i64, - object_id: &str, - old_snapshot: &[u8], - new_snapshot: &[u8], -) -> Result { - if old_snapshot.is_empty() { - return Ok("".to_string()); - } - - if new_snapshot.is_empty() { - return Err(anyhow::anyhow!( - "The new {} snapshot data is empty", - object_id - )); - } - - let old = try_decode_snapshot(uid, object_id, old_snapshot)?; - let new = try_decode_snapshot(uid, object_id, new_snapshot)?; - - let mut display_str = String::new(); - let diff = TextDiff::from_lines(&old, &new); - for change in diff.iter_all_changes() { - let sign = match change.tag() { - ChangeTag::Delete => "-", - ChangeTag::Insert => "+", - ChangeTag::Equal => " ", - }; - display_str.push_str(&format!("{}{}", sign, change)); - } - Ok(display_str) -} - -pub fn try_decode_snapshot( - uid: i64, - object_id: &str, - data: &[u8], -) -> Result { - let mut decoded_str = String::new(); - match { - let mut wrapper = AssertUnwindSafe(&mut decoded_str); - panic::catch_unwind(move || { - let collab = Collab::new(uid, object_id, "1", vec![]); - if let Ok(update) = Update::decode_v1(data) { - let mut txn = collab.origin_transact_mut(); - txn.apply_update(update); - drop(txn); - } - **wrapper = collab.to_plain_text(); - }) - } { - Ok(_) => Ok(decoded_str), - Err(e) => Err(PersistenceError::InvalidData(format!("{:?}", e))), - } -} diff --git a/collab-plugins/tests/disk/mod.rs b/collab-plugins/tests/disk/mod.rs index 6f2648ab1..22b1143bb 100644 --- a/collab-plugins/tests/disk/mod.rs +++ b/collab-plugins/tests/disk/mod.rs @@ -3,6 +3,5 @@ mod insert_test; mod range_test; mod restore_test; mod script; -mod snapshot_test; mod undo_test; mod util; diff --git a/collab-plugins/tests/disk/script.rs b/collab-plugins/tests/disk/script.rs index c50eed819..be58219cf 100644 --- a/collab-plugins/tests/disk/script.rs +++ b/collab-plugins/tests/disk/script.rs @@ -1,17 +1,15 @@ use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; -use std::time::Duration; use collab::core::collab::MutexCollab; use collab::preclude::*; -use collab_entity::{CollabObject, CollabType}; + use collab_plugins::local_storage::CollabPersistenceConfig; -use yrs::updates::decoder::Decode; use collab_plugins::local_storage::kv::doc::CollabKVAction; use collab_plugins::local_storage::rocksdb::rocksdb_plugin::RocksdbDiskPlugin; -use collab_plugins::local_storage::rocksdb::snapshot_plugin::CollabSnapshotPlugin; + use collab_plugins::CollabKVDB; use tempfile::TempDir; @@ -25,6 +23,7 @@ pub enum Script { OpenDocumentWithDiskPlugin { id: String, }, + #[allow(dead_code)] OpenDocument { id: String, }, @@ -44,27 +43,13 @@ pub enum Script { key: String, expected: Option, }, - AssertSnapshot { - id: String, - index: u32, - expected: JsonValue, - }, AssertNumOfUpdates { id: String, expected: usize, }, - AssertNumOfSnapshots { - id: String, - expected: usize, - }, AssertNumOfDocuments { expected: usize, }, - AssertDocument { - id: String, - expected: JsonValue, - }, - Wait(u64), } pub struct CollabPersistenceTest { @@ -72,8 +57,10 @@ pub struct CollabPersistenceTest { collab_by_id: HashMap>, #[allow(dead_code)] cleaner: Cleaner, + #[allow(dead_code)] db: Arc, disk_plugin: Arc, + #[allow(dead_code)] config: CollabPersistenceConfig, } @@ -88,7 +75,6 @@ impl CollabPersistenceTest { uid, Arc::downgrade(&db), config.clone(), - None, )); let cleaner = Cleaner::new(db_path); Self { @@ -107,29 +93,6 @@ impl CollabPersistenceTest { } } - fn make_snapshot_plugin( - &self, - uid: i64, - object_id: String, - object_ty: CollabType, - _collab: Arc, - ) -> Arc { - let object = CollabObject::new( - uid, - object_id, - object_ty, - "".to_string(), - "fake_device_id".to_string(), - ); - Arc::new(CollabSnapshotPlugin::new( - self.uid, - object, - Arc::new(self.db.clone()), - Arc::downgrade(&self.db), - self.config.snapshot_per_update, - )) - } - pub async fn create_collab(&mut self, doc_id: String) { let collab = Arc::new( CollabBuilder::new(1, &doc_id) @@ -138,13 +101,6 @@ impl CollabPersistenceTest { .unwrap(), ); collab.lock().add_plugin(self.disk_plugin.clone()); - let object_id = collab.lock().object_id.clone(); - collab.lock().add_plugin(self.make_snapshot_plugin( - self.uid, - object_id, - CollabType::Document, - collab.clone(), - )); collab.lock().initialize(); self.collab_by_id.insert(doc_id, collab); @@ -178,12 +134,6 @@ impl CollabPersistenceTest { .unwrap(), ); collab.lock().add_plugin(self.disk_plugin.clone()); - collab.lock().add_plugin(self.make_snapshot_plugin( - self.uid, - id.to_string(), - CollabType::Document, - collab.clone(), - )); collab.lock().initialize(); let json = collab.to_json_value(); @@ -223,16 +173,7 @@ impl CollabPersistenceTest { .unwrap(), ); self.disk_plugin = Arc::new(plugin); - let object_id = collab.lock().object_id.clone(); - collab.lock().add_plugin(self.make_snapshot_plugin( - self.uid, - object_id, - CollabType::Document, - collab.clone(), - )); - collab.lock().initialize(); - self.collab_by_id.insert(id, collab); }, Script::OpenDocument { id } => { @@ -276,67 +217,12 @@ impl CollabPersistenceTest { .unwrap(); assert_eq!(updates.len(), expected) }, - Script::AssertNumOfSnapshots { id, expected } => { - let snapshot_plugin = self.make_snapshot_plugin( - self.uid, - id.clone(), - CollabType::Document, - self.collab_by_id.get(&id).unwrap().clone(), - ); - let snapshot = snapshot_plugin.get_snapshots(&id); - assert_eq!(snapshot.len(), expected); - }, Script::AssertNumOfDocuments { expected } => { let collab_db = self.disk_plugin.upgrade().unwrap(); let docs = collab_db.read_txn().get_all_docs().unwrap(); assert_eq!(docs.count(), expected); }, - Script::AssertSnapshot { - id, - index, - expected, - } => { - let snapshot_plugin = self.make_snapshot_plugin( - self.uid, - id.clone(), - CollabType::Document, - self.collab_by_id.get(&id).unwrap().clone(), - ); - let snapshots = snapshot_plugin.get_snapshots(&id); - let collab = CollabBuilder::new(1, &id) - .with_device_id("1") - .build() - .unwrap(); - collab.lock().with_origin_transact_mut(|txn| { - txn.apply_update(Update::decode_v1(&snapshots[index as usize].data).unwrap()); - }); - - let json = collab.lock().to_json_value(); - assert_json_diff::assert_json_eq!(json, expected); - }, - Script::AssertDocument { id, expected } => { - let collab = Arc::new( - CollabBuilder::new(1, &id) - .with_device_id("1") - .build() - .unwrap(), - ); - collab.lock().add_plugin(self.disk_plugin.clone()); - collab.lock().add_plugin(self.make_snapshot_plugin( - self.uid, - id, - CollabType::Document, - collab.clone(), - )); - collab.lock().initialize(); - - let json = collab.to_json_value(); - assert_json_diff::assert_json_eq!(json, expected); - }, - Script::Wait(secs) => { - tokio::time::sleep(Duration::from_secs(secs)).await; - }, } } } @@ -349,7 +235,6 @@ pub fn disk_plugin(uid: i64) -> (Arc, RocksdbDiskPlugin) { uid, Arc::downgrade(&db), CollabPersistenceConfig::default(), - None, ); (db, plugin) } diff --git a/collab-plugins/tests/disk/snapshot_test.rs b/collab-plugins/tests/disk/snapshot_test.rs deleted file mode 100644 index b27bc9a47..000000000 --- a/collab-plugins/tests/disk/snapshot_test.rs +++ /dev/null @@ -1,302 +0,0 @@ -use collab_plugins::local_storage::CollabPersistenceConfig; -use rand::Rng; -use serde_json::json; - -use crate::disk::script::CollabPersistenceTest; -use crate::disk::script::Script::*; - -#[tokio::test] -async fn disable_snapshot_test() { - let mut test = CollabPersistenceTest::new(CollabPersistenceConfig::new().enable_snapshot(false)); - let doc_id = "1".to_string(); - test - .run_scripts(vec![OpenDocument { id: doc_id.clone() }]) - .await; - - for i in 1..=20 { - test - .run_script(InsertKeyValue { - id: doc_id.clone(), - key: i.to_string(), - value: i.into(), - }) - .await; - } - - test - .run_scripts(vec![ - AssertNumOfUpdates { - id: doc_id.clone(), - expected: 20, - }, - AssertNumOfSnapshots { - id: doc_id, - expected: 0, - }, - ]) - .await; -} - -#[tokio::test] -async fn reopen_doc_snapshot_test() { - let mut test = CollabPersistenceTest::new( - CollabPersistenceConfig::new() - .enable_snapshot(true) - .snapshot_per_update(9), - ); - let doc_id = "1".to_string(); - test - .run_scripts(vec![OpenDocument { id: doc_id.clone() }]) - .await; - for i in 1..=9 { - test - .run_script(InsertKeyValue { - id: doc_id.clone(), - key: i.to_string(), - value: i.into(), - }) - .await; - } - test - .run_scripts(vec![ - AssertNumOfUpdates { - id: doc_id.clone(), - expected: 9, - }, - // wait for snapshot to write to disk for 1 second - Wait(1), - AssertNumOfSnapshots { - id: doc_id.clone(), - expected: 1, - }, - CloseDocument { id: doc_id.clone() }, - ]) - .await; - - // reopen - test - .run_scripts(vec![OpenDocument { id: doc_id.clone() }]) - .await; - test - .run_scripts(vec![ - AssertNumOfUpdates { - id: doc_id.clone(), - expected: 9, - }, - AssertNumOfSnapshots { - id: doc_id.clone(), - expected: 1, - }, - AssertDocument { - id: doc_id.clone(), - expected: json!({ - "1": 1.0, - "2": 2.0, - "3": 3.0, - "4": 4.0, - "5": 5.0, - "6": 6.0, - "7": 7.0, - "8": 8.0, - "9": 9.0 - }), - }, - AssertSnapshot { - id: doc_id.clone(), - index: 0, - expected: json!({ - "1": 1.0, - "2": 2.0, - "3": 3.0, - "4": 4.0, - "5": 5.0, - "6": 6.0, - "7": 7.0, - "8": 8.0, - "9": 9.0, - }), - }, - ]) - .await; -} - -#[tokio::test] -async fn periodically_gen_snapshot_test() { - let snapshot_per_update = 5; - let mut test = CollabPersistenceTest::new( - CollabPersistenceConfig::new() - .enable_snapshot(true) - .snapshot_per_update(snapshot_per_update), - ); - let doc_id = "1".to_string(); - test - .run_scripts(vec![OpenDocument { id: doc_id.clone() }]) - .await; - - for i in 0..20 { - test - .run_script(InsertKeyValue { - id: doc_id.clone(), - key: i.to_string(), - value: i.into(), - }) - .await; - - if i != 0 && i % snapshot_per_update == 0 { - test - .run_scripts(vec![ - // wait for snapshot to write to disk for 1 second for each snapshot trigger - Wait(1), - AssertNumOfUpdates { - id: doc_id.clone(), - expected: i as usize + 1, - }, - ]) - .await; - } - } - // test.run_script(Wait(1)).await; - test - .run_script(AssertSnapshot { - id: doc_id.clone(), - index: 0, - expected: json!( { - "0": 0.0, - "1": 1.0, - "2": 2.0, - "3": 3.0, - "4": 4.0, - "5": 5.0 - }), - }) - .await; - - test - .run_script(AssertSnapshot { - id: doc_id.clone(), - index: 1, - expected: json!({ - "0": 0.0, - "1": 1.0, - "2": 2.0, - "3": 3.0, - "4": 4.0, - "5": 5.0, - "6": 6.0, - "7": 7.0, - "8": 8.0, - "9": 9.0, - "10": 10.0 - }), - }) - .await; - - test - .run_script(AssertSnapshot { - id: doc_id.clone(), - index: 2, - expected: json!({ - "0": 0.0, - "1": 1.0, - "2": 2.0, - "3": 3.0, - "4": 4.0, - "5": 5.0, - "6": 6.0, - "7": 7.0, - "8": 8.0, - "9": 9.0, - "10": 10.0, - "11": 11.0, - "12": 12.0, - "13": 13.0, - "14": 14.0, - "15": 15.0 - }), - }) - .await; - test - .run_scripts(vec![ - AssertNumOfSnapshots { - id: doc_id.clone(), - expected: 3, - }, - AssertNumOfUpdates { - id: doc_id, - expected: 20, - }, - ]) - .await; -} - -#[tokio::test] -async fn gen_big_snapshot_test() { - let snapshot_per_update = 100; - let mut test = CollabPersistenceTest::new( - CollabPersistenceConfig::new() - .enable_snapshot(true) - .snapshot_per_update(snapshot_per_update), - ); - let doc_id = "1".to_string(); - test - .run_scripts(vec![OpenDocument { id: doc_id.clone() }]) - .await; - - let mut map_1 = serde_json::map::Map::new(); - let mut map_2 = serde_json::map::Map::new(); - for i in 0..300 { - let s = generate_random_string(100); - if i < 100 { - map_1.insert(i.to_string(), json!(&s)); - } - - if i < 200 { - map_2.insert(i.to_string(), json!(&s)); - } - if i != 0 && i % snapshot_per_update == 0 { - test - .run_scripts(vec![ - // wait for snapshot to write to disk for 1 second for each snapshot trigger - Wait(1), - ]) - .await; - } - - test - .run_script(InsertKeyValue { - id: doc_id.clone(), - key: i.to_string(), - value: s.into(), - }) - .await; - } - - test - .run_scripts(vec![ - Wait(2), - AssertSnapshot { - id: doc_id.clone(), - index: 0, - expected: serde_json::Value::Object(map_1), - }, - AssertSnapshot { - id: doc_id.clone(), - index: 1, - expected: serde_json::Value::Object(map_2), - }, - ]) - .await; -} - -fn generate_random_string(length: usize) -> String { - const CHARSET: &[u8] = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - let mut rng = rand::thread_rng(); - let random_string: String = (0..length) - .map(|_| { - let index = rng.gen_range(0..CHARSET.len()); - CHARSET[index] as char - }) - .collect(); - - random_string -} diff --git a/collab-user/tests/util.rs b/collab-user/tests/util.rs index 1cc39a173..f119b75ee 100644 --- a/collab-user/tests/util.rs +++ b/collab-user/tests/util.rs @@ -37,7 +37,7 @@ impl UserAwarenessTest { let path = tempdir.into_path(); let db = Arc::new(CollabKVDB::open(path.clone()).unwrap()); - let disk_plugin = RocksdbDiskPlugin::new(uid, Arc::downgrade(&db), None); + let disk_plugin = RocksdbDiskPlugin::new(uid, Arc::downgrade(&db)); let cleaner: Cleaner = Cleaner::new(path); let collab = CollabBuilder::new(1, uid.to_string()) From 5eea65abae724f9260a42f96c143308104e8c63c Mon Sep 17 00:00:00 2001 From: "Nathan.fooo" <86001920+appflowy@users.noreply.github.com> Date: Sun, 7 Jan 2024 11:11:03 +0800 Subject: [PATCH 07/13] refactor: snapshot trait (#145) * chore: refactor snapshot trait * chore: update * chore: update * refactor: snapshot * refactor: snapshot * refactor: remove old workspace and favrorite * refactor: remove old workspace and favrorite * fix: wasm build --- Cargo.lock | 5 + Makefile.toml | 29 +-- collab-database/tests/user_test/helper.rs | 5 +- collab-document/Cargo.toml | 1 + .../tests/blocks/block_test_core.rs | 9 +- collab-document/tests/util.rs | 17 +- collab-folder/Cargo.toml | 4 +- collab-folder/src/folder_migration.rs | 13 +- collab-folder/tests/folder_test/util.rs | 17 +- .../src/local_storage/kv/snapshot.rs | 2 - collab-plugins/src/local_storage/mod.rs | 7 - .../local_storage/rocksdb/rocksdb_plugin.rs | 88 ++++++--- .../local_storage/rocksdb/snapshot_plugin.rs | 169 ++++++++---------- collab-plugins/tests/disk/delete_test.rs | 21 ++- collab-plugins/tests/disk/insert_test.rs | 40 ++--- collab-plugins/tests/disk/script.rs | 64 +++---- collab-user/Cargo.toml | 1 + collab-user/tests/util.rs | 10 +- 18 files changed, 290 insertions(+), 212 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2ddbe9b0b..eb678d22c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -320,6 +320,7 @@ version = "0.1.0" dependencies = [ "anyhow", "collab", + "collab-entity", "collab-plugins", "futures", "getrandom 0.2.9", @@ -358,8 +359,10 @@ dependencies = [ "assert-json-diff", "chrono", "collab", + "collab-entity", "collab-plugins", "fs_extra", + "getrandom 0.2.9", "nanoid", "parking_lot", "serde", @@ -431,6 +434,7 @@ dependencies = [ "tokio", "tokio-stream", "tracing", + "uuid", ] [[package]] @@ -853,6 +857,7 @@ dependencies = [ "glob", "libc", "libz-sys", + "zstd-sys", ] [[package]] diff --git a/Makefile.toml b/Makefile.toml index 3ec9e139d..c54f8f622 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -68,25 +68,34 @@ script = [""" """] [config] -on_error_task = "catch" +default_to_workspace = false +#on_error_task = "catch" -[tasks.catch] -run_task = { name = ["clean_profraw_files"] } +#[tasks.catch] +#run_task = { name = ["clean_profraw_files"] } [env] CARGO_MAKE_EXTEND_WORKSPACE_MAKEFILE = true - [tasks.wasm_build] script_runner = "bash" script = [ -""" -crates=("collab" "collab-document" "collab-folder" "collab-user") - cd .. + """ + #!/bin/bash + BASE_DIR=$(pwd) + crates=("collab" "collab-document" "collab-folder" "collab-user") + + # Iterate over each crate and build it for crate in "${crates[@]}"; do echo "🔥🔥🔥 Building $crate with wasm-pack..." - cd ./$crate && wasm-pack build --features="wasm_build" || exit 1 - cd .. || exit 1 + + # Navigate to the crate directory + cd "$BASE_DIR/$crate" || { echo "Failed to enter directory $crate"; exit 1; } + + # Build the crate + wasm-pack build --features="wasm_build" || { echo "Build failed for $crate"; exit 1; } + + # No need to cd back since we use absolute paths done -""" + """ ] diff --git a/collab-database/tests/user_test/helper.rs b/collab-database/tests/user_test/helper.rs index 346d9c548..49df3101e 100644 --- a/collab-database/tests/user_test/helper.rs +++ b/collab-database/tests/user_test/helper.rs @@ -73,7 +73,7 @@ impl DatabaseCollabService for TestUserDatabaseCollabBuilderImpl { &self, uid: i64, object_id: &str, - _object_type: CollabType, + object_type: CollabType, collab_db: Weak, doc_state: CollabDocState, config: CollabPersistenceConfig, @@ -83,8 +83,11 @@ impl DatabaseCollabService for TestUserDatabaseCollabBuilderImpl { .with_doc_state(doc_state) .with_plugin(RocksdbDiskPlugin::new_with_config( uid, + object_id.to_string(), + object_type, collab_db, config.clone(), + None, )) .build() .unwrap(); diff --git a/collab-document/Cargo.toml b/collab-document/Cargo.toml index a2ab9211f..302c4d9bc 100644 --- a/collab-document/Cargo.toml +++ b/collab-document/Cargo.toml @@ -9,6 +9,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] collab = { workspace = true } +collab-entity = { workspace = true } serde.workspace = true serde_json.workspace = true nanoid = "0.4.0" diff --git a/collab-document/tests/blocks/block_test_core.rs b/collab-document/tests/blocks/block_test_core.rs index bde8b3855..85379d397 100644 --- a/collab-document/tests/blocks/block_test_core.rs +++ b/collab-document/tests/blocks/block_test_core.rs @@ -7,6 +7,7 @@ use collab_document::blocks::{ Block, BlockAction, BlockActionPayload, BlockActionType, BlockEvent, DocumentData, DocumentMeta, }; use collab_document::document::Document; +use collab_entity::CollabType; use collab_plugins::local_storage::rocksdb::rocksdb_plugin::RocksdbDiskPlugin; use collab_plugins::CollabKVDB; use nanoid::nanoid; @@ -26,7 +27,13 @@ impl BlockTestCore { pub async fn new() -> Self { let db = document_storage(); let doc_id = "1"; - let disk_plugin = RocksdbDiskPlugin::new(1, Arc::downgrade(&db)); + let disk_plugin = RocksdbDiskPlugin::new( + 1, + doc_id.to_string(), + CollabType::Document, + Arc::downgrade(&db), + None, + ); let collab = CollabBuilder::new(1, doc_id) .with_plugin(disk_plugin) .with_device_id("1") diff --git a/collab-document/tests/util.rs b/collab-document/tests/util.rs index 09e9e24f9..2a62c66e8 100644 --- a/collab-document/tests/util.rs +++ b/collab-document/tests/util.rs @@ -11,6 +11,7 @@ use collab::preclude::CollabBuilder; use collab_document::blocks::{Block, BlockAction, DocumentData, DocumentMeta}; use collab_document::document::Document; use collab_document::error::DocumentError; +use collab_entity::CollabType; use collab_plugins::local_storage::rocksdb::rocksdb_plugin::RocksdbDiskPlugin; use collab_plugins::CollabKVDB; use nanoid::nanoid; @@ -31,7 +32,13 @@ impl DocumentTest { } pub async fn new_with_db(uid: i64, doc_id: &str, db: Arc) -> Self { - let disk_plugin = RocksdbDiskPlugin::new(uid, Arc::downgrade(&db)); + let disk_plugin = RocksdbDiskPlugin::new( + uid, + doc_id.to_string(), + CollabType::Document, + Arc::downgrade(&db), + None, + ); let collab = CollabBuilder::new(1, doc_id) .with_plugin(disk_plugin) .with_device_id("1") @@ -103,7 +110,13 @@ impl Deref for DocumentTest { pub async fn open_document_with_db(uid: i64, doc_id: &str, db: Arc) -> Document { setup_log(); - let disk_plugin = RocksdbDiskPlugin::new(uid, Arc::downgrade(&db)); + let disk_plugin = RocksdbDiskPlugin::new( + uid, + doc_id.to_string(), + CollabType::Document, + Arc::downgrade(&db), + None, + ); let collab = CollabBuilder::new(uid, doc_id) .with_plugin(disk_plugin) .with_device_id("1") diff --git a/collab-folder/Cargo.toml b/collab-folder/Cargo.toml index 2415c34a4..a858ace98 100644 --- a/collab-folder/Cargo.toml +++ b/collab-folder/Cargo.toml @@ -11,6 +11,7 @@ crate-type = ["cdylib", "rlib"] anyhow.workspace = true chrono.workspace = true collab = {path = "../collab" } +collab-entity.workspace = true parking_lot.workspace = true serde.workspace = true serde_json.workspace = true @@ -19,6 +20,7 @@ thiserror = "1.0.30" tokio = {version = "1.26", features = ["rt", "sync"]} tokio-stream = {version = "0.1.14", features = ["sync"]} tracing.workspace = true +getrandom = { version = "0.2", optional = true } [dev-dependencies] assert-json-diff = "2.0.2" @@ -32,5 +34,5 @@ walkdir = "2.3.2" zip = "0.6.6" [features] -wasm_build = [] +wasm_build = ["getrandom/js"] diff --git a/collab-folder/src/folder_migration.rs b/collab-folder/src/folder_migration.rs index 3e0a555ab..e38129983 100644 --- a/collab-folder/src/folder_migration.rs +++ b/collab-folder/src/folder_migration.rs @@ -1,7 +1,6 @@ use anyhow::bail; use collab::preclude::{Any, Array, MapRefExtension, MapRefWrapper, ReadTxn, YrsValue}; use serde::{Deserialize, Serialize}; -use tracing::error; use crate::folder::FAVORITES_V1; use crate::{Folder, SectionItem, View, ViewRelations, Workspace}; @@ -27,6 +26,13 @@ impl Folder { } } } + drop(txn); + + if !favorites.is_empty() { + self.root.with_transact_mut(|txn| { + self.root.delete_with_txn(txn, FAVORITES_V1); + }); + } favorites } @@ -40,11 +46,10 @@ impl Folder { .flat_map(|map_ref| to_workspace_with_txn(&txn, &map_ref, &self.views.view_relations)) .collect::>() }; - if workspace.is_empty() { - error!("No workspace found. When migrating from v1 to v2, the workspace must be present."); - } else { + if !workspace.is_empty() { let workspace = workspace.pop().unwrap(); self.root.with_transact_mut(|txn| { + self.root.delete_with_txn(txn, WORKSPACES); self .views .insert_view_with_txn(txn, View::from(workspace), None); diff --git a/collab-folder/tests/folder_test/util.rs b/collab-folder/tests/folder_test/util.rs index a4803963f..f2bada3b8 100644 --- a/collab-folder/tests/folder_test/util.rs +++ b/collab-folder/tests/folder_test/util.rs @@ -6,6 +6,7 @@ use std::path::PathBuf; use std::sync::{Arc, Once}; use collab::preclude::CollabBuilder; +use collab_entity::CollabType; use collab_folder::*; use collab_plugins::local_storage::rocksdb::rocksdb_plugin::RocksdbDiskPlugin; use collab_plugins::CollabKVDB; @@ -52,7 +53,13 @@ pub async fn create_folder_with_data( let path = tempdir.into_path(); let db = Arc::new(CollabKVDB::open_opt(path.clone(), false).unwrap()); - let disk_plugin = RocksdbDiskPlugin::new(uid.as_i64(), Arc::downgrade(&db)); + let disk_plugin = RocksdbDiskPlugin::new( + uid.as_i64(), + workspace_id.to_string(), + CollabType::Folder, + Arc::downgrade(&db), + None, + ); let cleaner: Cleaner = Cleaner::new(path); let collab = CollabBuilder::new(uid.as_i64(), workspace_id) @@ -80,7 +87,13 @@ pub async fn create_folder_with_data( pub async fn open_folder_with_db(uid: UserId, object_id: &str, db_path: PathBuf) -> FolderTest { let db = Arc::new(CollabKVDB::open_opt(db_path.clone(), false).unwrap()); - let disk_plugin = RocksdbDiskPlugin::new(uid.as_i64(), Arc::downgrade(&db)); + let disk_plugin = RocksdbDiskPlugin::new( + uid.as_i64(), + object_id.to_string(), + CollabType::Folder, + Arc::downgrade(&db), + None, + ); let cleaner: Cleaner = Cleaner::new(db_path); let collab = CollabBuilder::new(1, object_id) .with_plugin(disk_plugin) diff --git a/collab-plugins/src/local_storage/kv/snapshot.rs b/collab-plugins/src/local_storage/kv/snapshot.rs index 4cffa3ac0..28f88f59c 100644 --- a/collab-plugins/src/local_storage/kv/snapshot.rs +++ b/collab-plugins/src/local_storage/kv/snapshot.rs @@ -184,8 +184,6 @@ pub fn try_encode_snapshot( } pub trait SnapshotPersistence: Send + Sync { - fn get_snapshots(&self, uid: i64, object_id: &str) -> Vec; - fn create_snapshot( &self, uid: i64, diff --git a/collab-plugins/src/local_storage/mod.rs b/collab-plugins/src/local_storage/mod.rs index 26ed0faaf..019ea87b2 100644 --- a/collab-plugins/src/local_storage/mod.rs +++ b/collab-plugins/src/local_storage/mod.rs @@ -10,12 +10,6 @@ pub struct CollabPersistenceConfig { /// Generate a snapshot every N updates /// Default is 20. The value must be greater than 0. pub snapshot_per_update: u32, - - /// Flush the document. Default is [true]. - /// After flush the document, all updates will be removed and the document state vector that - /// contains all the updates will be reset. - #[allow(dead_code)] - pub(crate) flush_doc: bool, } impl CollabPersistenceConfig { @@ -40,7 +34,6 @@ impl Default for CollabPersistenceConfig { Self { enable_snapshot: true, snapshot_per_update: 100, - flush_doc: true, } } } diff --git a/collab-plugins/src/local_storage/rocksdb/rocksdb_plugin.rs b/collab-plugins/src/local_storage/rocksdb/rocksdb_plugin.rs index 91fcb0c75..765557ca8 100644 --- a/collab-plugins/src/local_storage/rocksdb/rocksdb_plugin.rs +++ b/collab-plugins/src/local_storage/rocksdb/rocksdb_plugin.rs @@ -9,11 +9,14 @@ use collab::core::collab::make_yrs_doc; use collab::core::collab_plugin::EncodedCollab; use collab::core::origin::CollabOrigin; use collab::preclude::CollabPlugin; +use collab_entity::CollabType; use tracing::{error, instrument}; use yrs::updates::encoder::Encode; use yrs::{Doc, ReadTxn, StateVector, Transact, TransactionMut}; use crate::local_storage::kv::doc::CollabKVAction; +use crate::local_storage::kv::snapshot::SnapshotPersistence; +use crate::local_storage::rocksdb::snapshot_plugin::CollabSnapshot; use crate::local_storage::CollabPersistenceConfig; pub trait RocksdbBackup: Send + Sync { @@ -24,43 +27,86 @@ pub trait RocksdbBackup: Send + Sync { #[derive(Clone)] pub struct RocksdbDiskPlugin { uid: i64, - db: Weak, + object_id: String, + collab_type: CollabType, + collab_db: Weak, did_load: Arc, - /// the number of updates on disk when opening the document - initial_update_count: Arc, update_count: Arc, config: CollabPersistenceConfig, + snapshot: Option, } impl Deref for RocksdbDiskPlugin { type Target = Weak; fn deref(&self) -> &Self::Target { - &self.db + &self.collab_db } } impl RocksdbDiskPlugin { - pub fn new_with_config(uid: i64, db: Weak, config: CollabPersistenceConfig) -> Self { - let initial_update_count = Arc::new(AtomicU32::new(0)); + pub fn new_with_config( + uid: i64, + object_id: String, + collab_type: CollabType, + collab_db: Weak, + config: CollabPersistenceConfig, + snapshot_persistence: Option>, + ) -> Self { let update_count = Arc::new(AtomicU32::new(0)); let did_load = Arc::new(AtomicBool::new(false)); + + let mut snapshot = None; + if config.enable_snapshot { + snapshot = snapshot_persistence.map(CollabSnapshot::new); + } + Self { - db, + object_id, + collab_type, + collab_db, uid, did_load, - initial_update_count, update_count, config, + snapshot, } } - pub fn new(uid: i64, db: Weak) -> Self { - Self::new_with_config(uid, db, CollabPersistenceConfig::default()) + pub fn new( + uid: i64, + object_id: String, + collab_type: CollabType, + collab_db: Weak, + snapshot_persistence: Option>, + ) -> Self { + Self::new_with_config( + uid, + object_id, + collab_type, + collab_db, + CollabPersistenceConfig::default(), + snapshot_persistence, + ) } - fn increase_count(&self) -> u32 { - self.update_count.fetch_add(1, SeqCst) + fn increase_count(&self) { + let update_count = self.update_count.fetch_add(1, SeqCst); + self.create_snapshot_if_need(update_count); + } + + fn create_snapshot_if_need(&self, update_count: u32) { + if update_count != 0 && update_count % self.config.snapshot_per_update == 0 { + if let Some(snapshot) = &self.snapshot { + snapshot.create_snapshot( + self.collab_db.clone(), + self.uid, + &self.object_id, + &self.collab_type, + ); + } + self.update_count.store(0, SeqCst); + } } #[instrument(skip_all)] @@ -88,7 +134,7 @@ impl RocksdbDiskPlugin { impl CollabPlugin for RocksdbDiskPlugin { fn init(&self, object_id: &str, origin: &CollabOrigin, doc: &Doc) { - if let Some(db) = self.db.upgrade() { + if let Some(db) = self.collab_db.upgrade() { let rocksdb_read = db.read_txn(); // Check the document is exist or not if rocksdb_read.is_exist(self.uid, object_id) { @@ -96,7 +142,7 @@ impl CollabPlugin for RocksdbDiskPlugin { // Safety: The document is exist, so it must be loaded successfully. let update_count = match rocksdb_read.load_doc_with_txn(self.uid, object_id, &mut txn) { Ok(update_count) => { - self.initial_update_count.store(update_count, SeqCst); + self.update_count.store(update_count, SeqCst); update_count }, Err(e) => { @@ -108,9 +154,9 @@ impl CollabPlugin for RocksdbDiskPlugin { txn.commit(); drop(txn); - if self.config.flush_doc && update_count != 0 && update_count % 20 == 0 { + if update_count != 0 && update_count % self.config.snapshot_per_update == 0 { self.flush_doc(&db, object_id); - self.initial_update_count.store(0, SeqCst); + self.create_snapshot_if_need(update_count); } } else { let txn = doc.transact(); @@ -137,9 +183,9 @@ impl CollabPlugin for RocksdbDiskPlugin { if !self.did_load.load(SeqCst) { return; } - if let Some(db) = self.db.upgrade() { - let _ = self.increase_count(); - // /Acquire a write transaction to ensure consistency + if let Some(db) = self.collab_db.upgrade() { + self.increase_count(); + //Acquire a write transaction to ensure consistency let result = db.with_write_txn(|w_db_txn| { let _ = w_db_txn.push_update(self.uid, object_id, update)?; Ok(()) @@ -156,7 +202,7 @@ impl CollabPlugin for RocksdbDiskPlugin { fn after_transaction(&self, _object_id: &str, _txn: &mut TransactionMut) {} fn reset(&self, object_id: &str) { - if let Some(db) = self.db.upgrade() { + if let Some(db) = self.collab_db.upgrade() { if let Err(e) = db.with_write_txn(|w_db_txn| { w_db_txn.delete_all_updates(self.uid, object_id)?; Ok(()) @@ -167,7 +213,7 @@ impl CollabPlugin for RocksdbDiskPlugin { } fn flush(&self, object_id: &str, _doc: &Doc) { - if let Some(db) = self.db.upgrade() { + if let Some(db) = self.collab_db.upgrade() { self.flush_doc(&db, object_id); } } diff --git a/collab-plugins/src/local_storage/rocksdb/snapshot_plugin.rs b/collab-plugins/src/local_storage/rocksdb/snapshot_plugin.rs index 82cb62f65..4d8047ce8 100644 --- a/collab-plugins/src/local_storage/rocksdb/snapshot_plugin.rs +++ b/collab-plugins/src/local_storage/rocksdb/snapshot_plugin.rs @@ -1,133 +1,104 @@ -use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, Weak}; use crate::local_storage::kv::doc::CollabKVAction; -use crate::local_storage::kv::snapshot::{CollabSnapshot, SnapshotPersistence}; +use crate::local_storage::kv::snapshot::SnapshotPersistence; use crate::local_storage::kv::PersistenceError; use crate::CollabKVDB; -use collab::preclude::{Collab, CollabPlugin}; -use collab_entity::CollabObject; +use collab::preclude::Collab; +use collab_entity::CollabType; use parking_lot::RwLock; +use tracing::debug; -use yrs::{ReadTxn, StateVector, TransactionMut}; +use yrs::{ReadTxn, StateVector}; #[derive(Clone, Debug)] -enum GenSnapshotState { +pub(crate) enum SnapshotState { Idle, Processing, Fail, } -impl GenSnapshotState { - fn is_idle(&self) -> bool { - matches!(self, Self::Idle) - } - - fn is_fail(&self) -> bool { - matches!(self, Self::Fail) +impl SnapshotState { + fn is_processing(&self) -> bool { + matches!(self, Self::Processing) } } -pub struct CollabSnapshotPlugin { - uid: i64, - object: CollabObject, - collab_db: Weak, - /// the number of updates on disk when opening the document - update_count: Arc, - snapshot_per_update: u32, - state: Arc>, +#[derive(Clone)] +pub struct CollabSnapshot { + state: Arc>, snapshot_persistence: Arc, } -impl CollabSnapshotPlugin { - pub fn new( - uid: i64, - object: CollabObject, - snapshot_persistence: Arc, - collab_db: Weak, - snapshot_per_update: u32, - ) -> Self { - let state = Arc::new(RwLock::new(GenSnapshotState::Idle)); - let initial_update_count = Arc::new(AtomicU32::new(0)); +impl CollabSnapshot { + pub fn new(snapshot_persistence: Arc) -> Self { + let state = Arc::new(RwLock::new(SnapshotState::Idle)); Self { - uid, snapshot_persistence, - object, - collab_db, - update_count: initial_update_count, - snapshot_per_update, state, } } - /// Return the snapshots for the given object id - pub fn get_snapshots(&self, object_id: &str) -> Vec { - self.snapshot_persistence.get_snapshots(self.uid, object_id) + fn should_create_snapshot(&self) -> bool { + if let Some(mut state) = self.state.try_write() { + if !state.is_processing() { + *state = SnapshotState::Processing; + return true; + } + } + false } -} - -impl CollabPlugin for CollabSnapshotPlugin { - fn after_transaction(&self, _object_id: &str, _txn: &mut TransactionMut) { - // After each transaction, we increment the update count - let old_value = self.update_count.fetch_add(1, Ordering::SeqCst); - // If the number of updates is greater than the threshold, we generate a snapshot - // and push it to the database. If the state is fail, which means the previous snapshot - // generation failed, we try to generate a new snapshot again on the next transaction. - let should_create_snapshot = old_value != 0 && (old_value + 1) % self.snapshot_per_update == 0; - let state = self.state.read().clone(); - if should_create_snapshot || state.is_fail() { - let is_ready = state.is_fail() || state.is_idle(); - if is_ready { - *self.state.write() = GenSnapshotState::Processing; - let weak_collab_db = self.collab_db.clone(); - let weak_state = Arc::downgrade(&self.state); - let weak_snapshot_persistence = Arc::downgrade(&self.snapshot_persistence); - let uid = self.uid; - let object = self.object.clone(); + pub(crate) fn create_snapshot( + &self, + weak_collab_db: Weak, + uid: i64, + object_id: &str, + collab_type: &CollabType, + ) { + let should_create_snapshot = self.should_create_snapshot(); + if should_create_snapshot { + debug!("{}: create snapshot", object_id); + let weak_state = Arc::downgrade(&self.state); + let weak_snapshot_persistence = Arc::downgrade(&self.snapshot_persistence); + let object_id = object_id.to_string(); + let collab_type = collab_type.clone(); - // We use a blocking task to generate the snapshot - tokio::spawn(async move { - let _ = tokio::task::spawn_blocking(move || { - if let (Some(state), Some(collab_db), Some(snapshot_persistence)) = ( - weak_state.upgrade(), - weak_collab_db.upgrade(), - weak_snapshot_persistence.upgrade(), - ) { - let snapshot_collab = Collab::new(uid, object.object_id.clone(), "1", vec![]); - let mut txn = snapshot_collab.origin_transact_mut(); - if let Err(e) = - collab_db - .read_txn() - .load_doc_with_txn(uid, &object.object_id, &mut txn) - { - tracing::error!("{} snapshot generation failed: {}", object.object_id, e); - *state.write() = GenSnapshotState::Fail; - return Ok::<(), PersistenceError>(()); - } - drop(txn); + // We use a blocking task to generate the snapshot + tokio::spawn(async move { + let _ = tokio::task::spawn_blocking(move || { + if let (Some(state), Some(collab_db), Some(snapshot_persistence)) = ( + weak_state.upgrade(), + weak_collab_db.upgrade(), + weak_snapshot_persistence.upgrade(), + ) { + let snapshot_collab = Collab::new(uid, object_id.clone(), "1", vec![]); + let mut txn = snapshot_collab.origin_transact_mut(); + if let Err(e) = collab_db + .read_txn() + .load_doc_with_txn(uid, &object_id, &mut txn) + { + tracing::error!("{} snapshot generation failed: {}", object_id, e); + *state.write() = SnapshotState::Fail; + return Ok::<(), PersistenceError>(()); + } + drop(txn); - // Generate the snapshot - let txn = snapshot_collab.transact(); - let encoded_v1 = txn.encode_state_as_update_v1(&StateVector::default()); - match snapshot_persistence.create_snapshot( - uid, - &object.object_id, - &object.collab_type, - encoded_v1, - ) { - Ok(_) => *state.write() = GenSnapshotState::Idle, - Err(e) => { - tracing::error!("{} snapshot generation failed: {}", object.object_id, e); - *state.write() = GenSnapshotState::Fail; - }, - } + // Generate the snapshot + let txn = snapshot_collab.transact(); + let encoded_v1 = txn.encode_state_as_update_v1(&StateVector::default()); + match snapshot_persistence.create_snapshot(uid, &object_id, &collab_type, encoded_v1) { + Ok(_) => *state.write() = SnapshotState::Idle, + Err(e) => { + tracing::error!("{} snapshot generation failed: {}", object_id, e); + *state.write() = SnapshotState::Fail; + }, } - Ok::<(), PersistenceError>(()) - }) - .await; - }); - } + } + Ok::<(), PersistenceError>(()) + }) + .await; + }); } } } diff --git a/collab-plugins/tests/disk/delete_test.rs b/collab-plugins/tests/disk/delete_test.rs index ec3ef86d8..e980aac33 100644 --- a/collab-plugins/tests/disk/delete_test.rs +++ b/collab-plugins/tests/disk/delete_test.rs @@ -1,17 +1,16 @@ +use crate::disk::script::CollabPersistenceTest; use crate::disk::script::Script::*; -use crate::disk::script::{disk_plugin, CollabPersistenceTest}; use collab_plugins::local_storage::CollabPersistenceConfig; #[tokio::test] async fn delete_single_doc_test() { let mut test = CollabPersistenceTest::new(CollabPersistenceConfig::default()); let doc_id = "1".to_string(); - let (_db, disk_plugin) = disk_plugin(test.uid); test .run_scripts(vec![ - CreateDocumentWithDiskPlugin { + CreateDocumentWithCollabDB { id: doc_id.clone(), - plugin: disk_plugin, + db: test.db.clone(), }, AssertNumOfDocuments { expected: 1 }, DeleteDocument { id: doc_id }, @@ -22,20 +21,20 @@ async fn delete_single_doc_test() { #[tokio::test] async fn delete_multiple_docs_test() { let mut test = CollabPersistenceTest::new(CollabPersistenceConfig::default()); - let (_db, disk_plugin) = disk_plugin(test.uid); + let db = test.db.clone(); test .run_scripts(vec![ - CreateDocumentWithDiskPlugin { + CreateDocumentWithCollabDB { id: "1".to_string(), - plugin: disk_plugin.clone(), + db: db.clone(), }, - CreateDocumentWithDiskPlugin { + CreateDocumentWithCollabDB { id: "2".to_string(), - plugin: disk_plugin.clone(), + db: db.clone(), }, - CreateDocumentWithDiskPlugin { + CreateDocumentWithCollabDB { id: "3".to_string(), - plugin: disk_plugin, + db: db.clone(), }, DeleteDocument { id: "1".to_string(), diff --git a/collab-plugins/tests/disk/insert_test.rs b/collab-plugins/tests/disk/insert_test.rs index 6088e8a18..cf94cae28 100644 --- a/collab-plugins/tests/disk/insert_test.rs +++ b/collab-plugins/tests/disk/insert_test.rs @@ -1,17 +1,17 @@ +use crate::disk::script::CollabPersistenceTest; use crate::disk::script::Script::*; -use crate::disk::script::{disk_plugin, CollabPersistenceTest}; use collab_plugins::local_storage::CollabPersistenceConfig; #[tokio::test] async fn insert_single_change_and_restore_from_disk() { let doc_id = "1".to_string(); let mut test = CollabPersistenceTest::new(CollabPersistenceConfig::new()); - let (_db, disk_plugin) = disk_plugin(test.uid); + let db = test.db.clone(); test .run_scripts(vec![ - CreateDocumentWithDiskPlugin { + CreateDocumentWithCollabDB { id: doc_id.clone(), - plugin: disk_plugin, + db: db.clone(), }, InsertKeyValue { id: doc_id.clone(), @@ -37,12 +37,12 @@ async fn insert_single_change_and_restore_from_disk() { async fn insert_multiple_changes_and_restore_from_disk() { let mut test = CollabPersistenceTest::new(CollabPersistenceConfig::new()); let doc_id = "1".to_string(); - let (_db, disk_plugin) = disk_plugin(test.uid); + let db = test.db.clone(); test .run_scripts(vec![ - CreateDocumentWithDiskPlugin { + CreateDocumentWithCollabDB { id: doc_id.clone(), - plugin: disk_plugin, + db: db.clone(), }, InsertKeyValue { id: doc_id.clone(), @@ -101,32 +101,32 @@ async fn insert_multiple_changes_and_restore_from_disk() { #[tokio::test] async fn insert_multiple_docs() { let mut test = CollabPersistenceTest::new(CollabPersistenceConfig::new()); - let (_db, disk_plugin) = disk_plugin(test.uid); + let db = test.db.clone(); test .run_scripts(vec![ - CreateDocumentWithDiskPlugin { + CreateDocumentWithCollabDB { id: "1".to_string(), - plugin: disk_plugin.clone(), + db: db.clone(), }, - CreateDocumentWithDiskPlugin { + CreateDocumentWithCollabDB { id: "2".to_string(), - plugin: disk_plugin.clone(), + db: db.clone(), }, - CreateDocumentWithDiskPlugin { + CreateDocumentWithCollabDB { id: "3".to_string(), - plugin: disk_plugin.clone(), + db: db.clone(), }, - CreateDocumentWithDiskPlugin { + CreateDocumentWithCollabDB { id: "4".to_string(), - plugin: disk_plugin.clone(), + db: db.clone(), }, - CreateDocumentWithDiskPlugin { + CreateDocumentWithCollabDB { id: "5".to_string(), - plugin: disk_plugin.clone(), + db: db.clone(), }, - CreateDocumentWithDiskPlugin { + CreateDocumentWithCollabDB { id: "6".to_string(), - plugin: disk_plugin, + db: db.clone(), }, AssertNumOfDocuments { expected: 6 }, ]) diff --git a/collab-plugins/tests/disk/script.rs b/collab-plugins/tests/disk/script.rs index be58219cf..d115bec11 100644 --- a/collab-plugins/tests/disk/script.rs +++ b/collab-plugins/tests/disk/script.rs @@ -10,15 +10,16 @@ use collab_plugins::local_storage::CollabPersistenceConfig; use collab_plugins::local_storage::kv::doc::CollabKVAction; use collab_plugins::local_storage::rocksdb::rocksdb_plugin::RocksdbDiskPlugin; +use collab_entity::CollabType; use collab_plugins::CollabKVDB; use tempfile::TempDir; use crate::setup_log; pub enum Script { - CreateDocumentWithDiskPlugin { + CreateDocumentWithCollabDB { id: String, - plugin: RocksdbDiskPlugin, + db: Arc, }, OpenDocumentWithDiskPlugin { id: String, @@ -58,8 +59,7 @@ pub struct CollabPersistenceTest { #[allow(dead_code)] cleaner: Cleaner, #[allow(dead_code)] - db: Arc, - disk_plugin: Arc, + pub db: Arc, #[allow(dead_code)] config: CollabPersistenceConfig, } @@ -71,16 +71,10 @@ impl CollabPersistenceTest { let db_path = tempdir.into_path(); let uid = 1; let db = Arc::new(CollabKVDB::open_opt(db_path.clone(), false).unwrap()); - let disk_plugin = Arc::new(RocksdbDiskPlugin::new_with_config( - uid, - Arc::downgrade(&db), - config.clone(), - )); let cleaner = Cleaner::new(db_path); Self { uid, collab_by_id: HashMap::default(), - disk_plugin, cleaner, db, config, @@ -100,7 +94,9 @@ impl CollabPersistenceTest { .build() .unwrap(), ); - collab.lock().add_plugin(self.disk_plugin.clone()); + let disk_plugin = disk_plugin_with_db(self.uid, self.db.clone(), &doc_id, CollabType::Document) + as Arc; + collab.lock().add_plugin(disk_plugin); collab.lock().initialize(); self.collab_by_id.insert(doc_id, collab); @@ -133,7 +129,9 @@ impl CollabPersistenceTest { .build() .unwrap(), ); - collab.lock().add_plugin(self.disk_plugin.clone()); + let disk_plugin = disk_plugin_with_db(self.uid, self.db.clone(), id, CollabType::Document) + as Arc; + collab.lock().add_plugin(disk_plugin); collab.lock().initialize(); let json = collab.to_json_value(); @@ -164,15 +162,15 @@ impl CollabPersistenceTest { pub async fn run_script(&mut self, script: Script) { match script { - Script::CreateDocumentWithDiskPlugin { id, plugin } => { + Script::CreateDocumentWithCollabDB { id, db } => { + let disk_plugin = disk_plugin_with_db(self.uid, db, &id, CollabType::Document); let collab = Arc::new( CollabBuilder::new(1, &id) .with_device_id("1") - .with_plugin(plugin.clone()) + .with_plugin(disk_plugin) .build() .unwrap(), ); - self.disk_plugin = Arc::new(plugin); collab.lock().initialize(); self.collab_by_id.insert(id, collab); }, @@ -183,17 +181,19 @@ impl CollabPersistenceTest { self.collab_by_id.remove(&id); }, Script::OpenDocumentWithDiskPlugin { id } => { + let disk_plugin = disk_plugin_with_db(self.uid, self.db.clone(), &id, CollabType::Document); + let collab = CollabBuilder::new(1, &id) .with_device_id("1") - .with_plugin(self.disk_plugin.clone()) + .with_plugin(disk_plugin) .build() .unwrap(); collab.lock().initialize(); self.collab_by_id.insert(id, Arc::new(collab)); }, Script::DeleteDocument { id } => { - let collab_db = self.disk_plugin.upgrade().unwrap(); - collab_db + self + .db .with_write_txn(|store| store.delete_doc(self.uid, &id)) .unwrap(); }, @@ -210,33 +210,37 @@ impl CollabPersistenceTest { assert_eq!(text, expected) }, Script::AssertNumOfUpdates { id, expected } => { - let collab_db = self.disk_plugin.upgrade().unwrap(); - let updates = collab_db + let updates = self + .db .read_txn() .get_decoded_v1_updates(self.uid, &id) .unwrap(); assert_eq!(updates.len(), expected) }, Script::AssertNumOfDocuments { expected } => { - let collab_db = self.disk_plugin.upgrade().unwrap(); - - let docs = collab_db.read_txn().get_all_docs().unwrap(); + let docs = self.db.read_txn().get_all_docs().unwrap(); assert_eq!(docs.count(), expected); }, } } } -pub fn disk_plugin(uid: i64) -> (Arc, RocksdbDiskPlugin) { - let tempdir = TempDir::new().unwrap(); - let path = tempdir.into_path(); - let db = Arc::new(CollabKVDB::open_opt(path, false).unwrap()); - let plugin = RocksdbDiskPlugin::new_with_config( +pub fn disk_plugin_with_db( + uid: i64, + db: Arc, + object_id: &str, + collab_type: CollabType, +) -> Arc { + let object_id = object_id.to_string(); + let collab_type = collab_type.clone(); + Arc::new(RocksdbDiskPlugin::new_with_config( uid, + object_id, + collab_type, Arc::downgrade(&db), CollabPersistenceConfig::default(), - ); - (db, plugin) + None, + )) } struct Cleaner(PathBuf); diff --git a/collab-user/Cargo.toml b/collab-user/Cargo.toml index b244cb7ec..56ede7463 100644 --- a/collab-user/Cargo.toml +++ b/collab-user/Cargo.toml @@ -26,6 +26,7 @@ fs_extra = "1.2.0" nanoid = "0.4.0" tempfile = "3.8.0" tokio = {version = "1.26", features = ["rt", "macros"]} +uuid = "1.3.3" [features] diff --git a/collab-user/tests/util.rs b/collab-user/tests/util.rs index f119b75ee..96db46be9 100644 --- a/collab-user/tests/util.rs +++ b/collab-user/tests/util.rs @@ -5,6 +5,7 @@ use std::time::Duration; use anyhow::Result; use collab::preclude::CollabBuilder; +use collab_entity::CollabType; use collab_plugins::local_storage::rocksdb::rocksdb_plugin::RocksdbDiskPlugin; use collab_plugins::CollabKVDB; use collab_user::core::{ @@ -37,7 +38,14 @@ impl UserAwarenessTest { let path = tempdir.into_path(); let db = Arc::new(CollabKVDB::open(path.clone()).unwrap()); - let disk_plugin = RocksdbDiskPlugin::new(uid, Arc::downgrade(&db)); + let id = uuid::Uuid::new_v4().to_string(); + let disk_plugin = RocksdbDiskPlugin::new( + uid, + id, + CollabType::UserAwareness, + Arc::downgrade(&db), + None, + ); let cleaner: Cleaner = Cleaner::new(path); let collab = CollabBuilder::new(1, uid.to_string()) From 6d9cb684cf6735981de8de3b59e547fa8abc57f7 Mon Sep 17 00:00:00 2001 From: "Nathan.fooo" <86001920+appflowy@users.noreply.github.com> Date: Sun, 7 Jan 2024 13:11:48 +0800 Subject: [PATCH 08/13] chore: Snapshot log (#146) * chore: refactor snapshot trait * chore: update * chore: update * refactor: snapshot * refactor: snapshot * refactor: remove old workspace and favrorite * refactor: remove old workspace and favrorite * fix: wasm build * chore: update log * chore: update log * chore: fix fmt --- collab-database/src/blocks/block.rs | 2 +- collab-plugins/src/local_storage/mod.rs | 2 +- .../local_storage/rocksdb/rocksdb_plugin.rs | 20 +++-- .../local_storage/rocksdb/snapshot_plugin.rs | 79 +++++++++---------- 4 files changed, 52 insertions(+), 51 deletions(-) diff --git a/collab-database/src/blocks/block.rs b/collab-database/src/blocks/block.rs index 5fbe4cc5e..7870d9485 100644 --- a/collab-database/src/blocks/block.rs +++ b/collab-database/src/blocks/block.rs @@ -239,7 +239,7 @@ impl Block { } fn collab_for_row(&self, row_id: &RowId) -> Arc { - let config = CollabPersistenceConfig::new().snapshot_per_update(5); + let config = CollabPersistenceConfig::new().snapshot_per_update(100); let doc_state = CollabDocState::default(); self.collab_service.build_collab_with_config( self.uid, diff --git a/collab-plugins/src/local_storage/mod.rs b/collab-plugins/src/local_storage/mod.rs index 019ea87b2..24ab602d8 100644 --- a/collab-plugins/src/local_storage/mod.rs +++ b/collab-plugins/src/local_storage/mod.rs @@ -8,7 +8,7 @@ pub struct CollabPersistenceConfig { /// Enable snapshot. Default is [false]. pub enable_snapshot: bool, /// Generate a snapshot every N updates - /// Default is 20. The value must be greater than 0. + /// Default is 100. The value must be greater than 0. pub snapshot_per_update: u32, } diff --git a/collab-plugins/src/local_storage/rocksdb/rocksdb_plugin.rs b/collab-plugins/src/local_storage/rocksdb/rocksdb_plugin.rs index 765557ca8..6450a9be7 100644 --- a/collab-plugins/src/local_storage/rocksdb/rocksdb_plugin.rs +++ b/collab-plugins/src/local_storage/rocksdb/rocksdb_plugin.rs @@ -10,7 +10,7 @@ use collab::core::collab_plugin::EncodedCollab; use collab::core::origin::CollabOrigin; use collab::preclude::CollabPlugin; use collab_entity::CollabType; -use tracing::{error, instrument}; +use tracing::{debug, error, instrument}; use yrs::updates::encoder::Encode; use yrs::{Doc, ReadTxn, StateVector, Transact, TransactionMut}; @@ -98,12 +98,18 @@ impl RocksdbDiskPlugin { fn create_snapshot_if_need(&self, update_count: u32) { if update_count != 0 && update_count % self.config.snapshot_per_update == 0 { if let Some(snapshot) = &self.snapshot { - snapshot.create_snapshot( - self.collab_db.clone(), - self.uid, - &self.object_id, - &self.collab_type, - ); + if snapshot.should_create_snapshot() { + debug!( + "create snapshot for {}, update_count:{}, snapshot_per_update:{}", + self.object_id, update_count, self.config.snapshot_per_update + ); + snapshot.create_snapshot( + self.collab_db.clone(), + self.uid, + &self.object_id, + &self.collab_type, + ); + } } self.update_count.store(0, SeqCst); } diff --git a/collab-plugins/src/local_storage/rocksdb/snapshot_plugin.rs b/collab-plugins/src/local_storage/rocksdb/snapshot_plugin.rs index 4d8047ce8..8443ec4e8 100644 --- a/collab-plugins/src/local_storage/rocksdb/snapshot_plugin.rs +++ b/collab-plugins/src/local_storage/rocksdb/snapshot_plugin.rs @@ -7,7 +7,6 @@ use crate::CollabKVDB; use collab::preclude::Collab; use collab_entity::CollabType; use parking_lot::RwLock; -use tracing::debug; use yrs::{ReadTxn, StateVector}; @@ -39,7 +38,7 @@ impl CollabSnapshot { } } - fn should_create_snapshot(&self) -> bool { + pub(crate) fn should_create_snapshot(&self) -> bool { if let Some(mut state) = self.state.try_write() { if !state.is_processing() { *state = SnapshotState::Processing; @@ -56,49 +55,45 @@ impl CollabSnapshot { object_id: &str, collab_type: &CollabType, ) { - let should_create_snapshot = self.should_create_snapshot(); - if should_create_snapshot { - debug!("{}: create snapshot", object_id); - let weak_state = Arc::downgrade(&self.state); - let weak_snapshot_persistence = Arc::downgrade(&self.snapshot_persistence); - let object_id = object_id.to_string(); - let collab_type = collab_type.clone(); + let weak_state = Arc::downgrade(&self.state); + let weak_snapshot_persistence = Arc::downgrade(&self.snapshot_persistence); + let object_id = object_id.to_string(); + let collab_type = collab_type.clone(); - // We use a blocking task to generate the snapshot - tokio::spawn(async move { - let _ = tokio::task::spawn_blocking(move || { - if let (Some(state), Some(collab_db), Some(snapshot_persistence)) = ( - weak_state.upgrade(), - weak_collab_db.upgrade(), - weak_snapshot_persistence.upgrade(), - ) { - let snapshot_collab = Collab::new(uid, object_id.clone(), "1", vec![]); - let mut txn = snapshot_collab.origin_transact_mut(); - if let Err(e) = collab_db - .read_txn() - .load_doc_with_txn(uid, &object_id, &mut txn) - { + // We use a blocking task to generate the snapshot + tokio::spawn(async move { + let _ = tokio::task::spawn_blocking(move || { + if let (Some(state), Some(collab_db), Some(snapshot_persistence)) = ( + weak_state.upgrade(), + weak_collab_db.upgrade(), + weak_snapshot_persistence.upgrade(), + ) { + let snapshot_collab = Collab::new(uid, object_id.clone(), "1", vec![]); + let mut txn = snapshot_collab.origin_transact_mut(); + if let Err(e) = collab_db + .read_txn() + .load_doc_with_txn(uid, &object_id, &mut txn) + { + tracing::error!("{} snapshot generation failed: {}", object_id, e); + *state.write() = SnapshotState::Fail; + return Ok::<(), PersistenceError>(()); + } + drop(txn); + + // Generate the snapshot + let txn = snapshot_collab.transact(); + let encoded_v1 = txn.encode_state_as_update_v1(&StateVector::default()); + match snapshot_persistence.create_snapshot(uid, &object_id, &collab_type, encoded_v1) { + Ok(_) => *state.write() = SnapshotState::Idle, + Err(e) => { tracing::error!("{} snapshot generation failed: {}", object_id, e); *state.write() = SnapshotState::Fail; - return Ok::<(), PersistenceError>(()); - } - drop(txn); - - // Generate the snapshot - let txn = snapshot_collab.transact(); - let encoded_v1 = txn.encode_state_as_update_v1(&StateVector::default()); - match snapshot_persistence.create_snapshot(uid, &object_id, &collab_type, encoded_v1) { - Ok(_) => *state.write() = SnapshotState::Idle, - Err(e) => { - tracing::error!("{} snapshot generation failed: {}", object_id, e); - *state.write() = SnapshotState::Fail; - }, - } + }, } - Ok::<(), PersistenceError>(()) - }) - .await; - }); - } + } + Ok::<(), PersistenceError>(()) + }) + .await; + }); } } From fe8f08fcc99ea56c78bfb746ccb0cd308126141d Mon Sep 17 00:00:00 2001 From: "Nathan.fooo" <86001920+appflowy@users.noreply.github.com> Date: Mon, 8 Jan 2024 04:03:33 +0800 Subject: [PATCH 09/13] refactor: Fine tune rocksdb (#147) * fix: flush * chore: update * chore: update * chore: update * chore: update * chore: update * chore: update * chore: update config * chore: clippy --- Cargo.toml | 1 + collab-plugins/src/local_storage/kv/doc.rs | 8 +--- .../src/local_storage/rocksdb/kv_impl.rs | 46 +++++++++++++++---- collab-plugins/tests/disk/insert_test.rs | 37 ++++++++++++++- collab-plugins/tests/disk/range_test.rs | 40 ++++++++++++++++ collab-plugins/tests/disk/script.rs | 4 +- collab/src/core/collab_plugin.rs | 38 +++++++++++++++ 7 files changed, 156 insertions(+), 18 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 30295e66a..8049ef5d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "collab-folder", "collab-plugins", ] +resolver = "2" [workspace.dependencies] collab = { workspace = true, path = "collab" } diff --git a/collab-plugins/src/local_storage/kv/doc.rs b/collab-plugins/src/local_storage/kv/doc.rs index 07207d087..49754a146 100644 --- a/collab-plugins/src/local_storage/kv/doc.rs +++ b/collab-plugins/src/local_storage/kv/doc.rs @@ -56,16 +56,12 @@ where doc_state: Vec, ) -> Result<(), PersistenceError> { let doc_id = get_or_create_did(uid, self, object_id)?; - tracing::debug!( - "[Client {}] => [{}:{:?}]: flush doc", - uid, - doc_id, - object_id - ); // Remove the updates let start = make_doc_start_key(doc_id); let end = make_doc_end_key(doc_id); + + tracing::debug!("[{}:{:?}]: flush doc", doc_id, object_id,); self.remove_range(start.as_ref(), end.as_ref())?; let doc_state_key = make_doc_state_key(doc_id); diff --git a/collab-plugins/src/local_storage/rocksdb/kv_impl.rs b/collab-plugins/src/local_storage/rocksdb/kv_impl.rs index 49ca7178e..4245a4b5c 100644 --- a/collab-plugins/src/local_storage/rocksdb/kv_impl.rs +++ b/collab-plugins/src/local_storage/rocksdb/kv_impl.rs @@ -29,16 +29,47 @@ impl RocksStore { // On the other hand, setting it too high could lead to excessive CPU and I/O usage, impacting the overall // performance of the system. db_opts.set_max_background_jobs(4); + db_opts.create_if_missing(true); + + // sst + db_opts.set_max_open_files(50); + + // compression db_opts.set_compression_type(rocksdb::DBCompressionType::Zstd); db_opts.set_blob_compression_type(rocksdb::DBCompressionType::Zstd); + db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level); + + // wal + // Can't set the wal because existing rocksdb databases don't have the wal directory + // It might cause data lost. + // db_opts.set_wal_dir(path.as_ref().join("wal")); + + db_opts.set_wal_bytes_per_sync(1024 * 1024); + db_opts.set_wal_size_limit_mb(2); + db_opts.set_max_total_wal_size(20 * 1024 * 1024); + + // write buffer + db_opts.set_bytes_per_sync(1024 * 1024); + db_opts.set_write_buffer_size(2 * 1024 * 1024); + db_opts.set_max_write_buffer_number(2); + db_opts.set_min_write_buffer_number_to_merge(1); + + // level 0 + db_opts.set_level_zero_file_num_compaction_trigger(2); + db_opts.set_level_zero_slowdown_writes_trigger(5); + db_opts.set_level_zero_stop_writes_trigger(10); + + // log db_opts.set_recycle_log_file_num(5); db_opts.set_keep_log_file_num(5); db_opts.set_db_log_dir(path.as_ref().join("logs")); - db_opts.create_if_missing(true); let open_result = TransactionDB::::open(&db_opts, &txn_db_opts, &path); let db = match open_result { - Ok(db) => Ok(db), + Ok(db) => { + // + Ok(db) + }, Err(e) => { tracing::error!("🔴open collab db error: {:?}", e); match e.kind() { @@ -92,6 +123,10 @@ impl RocksStore { /// Return a read transaction that accesses the database exclusively. pub fn read_txn(&self) -> impl CollabKVAction<'_, Error = PersistenceError> { let mut txn_options = TransactionOptions::default(); + // Use snapshot to provides a consistent view of the data. This snapshot can then be used + // to perform read operations, and the returned data will be consistent with the database + // state at the time the snapshot was created, regardless of any subsequent modifications + // made by other transactions. txn_options.set_snapshot(true); let txn = self .db @@ -105,12 +140,7 @@ impl RocksStore { where F: FnOnce(&RocksKVStoreImpl<'_, TransactionDB>) -> Result, { - let mut txn_options = TransactionOptions::default(); - // Use snapshot to provides a consistent view of the data. This snapshot can then be used - // to perform read operations, and the returned data will be consistent with the database - // state at the time the snapshot was created, regardless of any subsequent modifications - // made by other transactions. - txn_options.set_snapshot(true); + let txn_options = TransactionOptions::default(); let txn = self .db .transaction_opt(&WriteOptions::default(), &txn_options); diff --git a/collab-plugins/tests/disk/insert_test.rs b/collab-plugins/tests/disk/insert_test.rs index cf94cae28..f1d32560e 100644 --- a/collab-plugins/tests/disk/insert_test.rs +++ b/collab-plugins/tests/disk/insert_test.rs @@ -1,6 +1,11 @@ -use crate::disk::script::CollabPersistenceTest; use crate::disk::script::Script::*; +use crate::disk::script::{disk_plugin_with_db, CollabPersistenceTest}; +use assert_json_diff::assert_json_eq; +use collab::preclude::CollabBuilder; +use collab_entity::CollabType; +use collab_plugins::local_storage::kv::doc::CollabKVAction; use collab_plugins::local_storage::CollabPersistenceConfig; +use std::sync::Arc; #[tokio::test] async fn insert_single_change_and_restore_from_disk() { @@ -33,6 +38,34 @@ async fn insert_single_change_and_restore_from_disk() { .await; } +#[tokio::test] +async fn flush_test() { + let doc_id = "1".to_string(); + let test = CollabPersistenceTest::new(CollabPersistenceConfig::new()); + let disk_plugin = disk_plugin_with_db(test.uid, test.db.clone(), &doc_id, CollabType::Document); + let collab = Arc::new( + CollabBuilder::new(1, &doc_id) + .with_device_id("1") + .with_plugin(disk_plugin) + .build() + .unwrap(), + ); + collab.lock().initialize(); + for i in 0..100 { + collab.lock().insert(&i.to_string(), i.to_string()); + } + let before_flush_value = collab.to_json_value(); + + let read = test.db.read_txn(); + let before_flush_updates = read.get_all_updates(test.uid, &doc_id).unwrap(); + collab.lock().flush(); + let after_flush_updates = read.get_all_updates(test.uid, &doc_id).unwrap(); + let after_flush_value = collab.to_json_value(); + assert_eq!(before_flush_updates.len(), 100); + assert_eq!(after_flush_updates.len(), 0); + assert_json_eq!(before_flush_value, after_flush_value); +} + #[tokio::test] async fn insert_multiple_changes_and_restore_from_disk() { let mut test = CollabPersistenceTest::new(CollabPersistenceConfig::new()); @@ -64,7 +97,7 @@ async fn insert_multiple_changes_and_restore_from_disk() { key: "4".to_string(), value: "d".into(), }, - AssertNumOfUpdates { + AssertUpdateLen { id: doc_id.clone(), expected: 4, }, diff --git a/collab-plugins/tests/disk/range_test.rs b/collab-plugins/tests/disk/range_test.rs index 16b6d5440..1dc7ec6d4 100644 --- a/collab-plugins/tests/disk/range_test.rs +++ b/collab-plugins/tests/disk/range_test.rs @@ -194,6 +194,46 @@ async fn range_key_test() { assert!(iter.next().is_none()); } +#[tokio::test] +async fn delete_range_test() { + let db = rocks_db().1; + db.with_write_txn(|store| { + store.insert([0, 0, 0, 0, 0, 0, 0, 0], [0, 1, 1]).unwrap(); + store.insert([0, 0, 0, 0, 0, 0, 0, 1], [0, 1, 2]).unwrap(); + store.insert([0, 0, 0, 0, 0, 0, 0, 2], [0, 1, 3]).unwrap(); + + store.insert([0, 0, 1, 0, 0, 0, 0, 0], [0, 2, 1]).unwrap(); + store.insert([0, 0, 1, 0, 0, 0, 0, 1], [0, 2, 2]).unwrap(); + store.insert([0, 0, 1, 0, 0, 0, 0, 2], [0, 2, 3]).unwrap(); + + Ok(()) + }) + .unwrap(); + + let given_key: &[u8; 8] = &[0, 0, 0, 0, 0, 0, 0, u8::MAX]; + let store = db.read_txn(); + let iter = store + .range::<&[u8; 8], RangeTo<&[u8; 8]>>(..given_key) + .unwrap(); + assert_eq!(iter.count(), 3); + + // remove the keys + db.with_write_txn(|write_txn| { + write_txn + .remove_range(&[0, 0, 0, 0, 0, 0, 0, 0], given_key) + .unwrap(); + Ok(()) + }) + .unwrap(); + + // check that the keys are removed + let store = db.read_txn(); + let iter = store + .range::<&[u8; 8], RangeTo<&[u8; 8]>>(..given_key) + .unwrap(); + assert_eq!(iter.count(), 0); +} + #[repr(transparent)] #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct Key(pub SmallVec<[u8; N]>); diff --git a/collab-plugins/tests/disk/script.rs b/collab-plugins/tests/disk/script.rs index d115bec11..3706e9a22 100644 --- a/collab-plugins/tests/disk/script.rs +++ b/collab-plugins/tests/disk/script.rs @@ -44,7 +44,7 @@ pub enum Script { key: String, expected: Option, }, - AssertNumOfUpdates { + AssertUpdateLen { id: String, expected: usize, }, @@ -209,7 +209,7 @@ impl CollabPersistenceTest { .map(|value| Any::String(Arc::from(value))); assert_eq!(text, expected) }, - Script::AssertNumOfUpdates { id, expected } => { + Script::AssertUpdateLen { id, expected } => { let updates = self .db .read_txn() diff --git a/collab/src/core/collab_plugin.rs b/collab/src/core/collab_plugin.rs index 9d7aa0469..062027376 100644 --- a/collab/src/core/collab_plugin.rs +++ b/collab/src/core/collab_plugin.rs @@ -80,6 +80,25 @@ where fn receive_update(&self, object_id: &str, txn: &TransactionMut, update: &[u8]) { (**self).receive_update(object_id, txn, update) } + + fn receive_local_update(&self, origin: &CollabOrigin, object_id: &str, update: &[u8]) { + (**self).receive_local_update(origin, object_id, update) + } + + fn after_transaction(&self, object_id: &str, txn: &mut TransactionMut) { + (**self).after_transaction(object_id, txn) + } + fn plugin_type(&self) -> CollabPluginType { + (**self).plugin_type() + } + + fn reset(&self, object_id: &str) { + (**self).reset(object_id) + } + + fn flush(&self, object_id: &str, doc: &Doc) { + (**self).flush(object_id, doc) + } } #[async_trait] @@ -104,6 +123,25 @@ where fn receive_update(&self, object_id: &str, txn: &TransactionMut, update: &[u8]) { (**self).receive_update(object_id, txn, update) } + + fn receive_local_update(&self, origin: &CollabOrigin, object_id: &str, update: &[u8]) { + (**self).receive_local_update(origin, object_id, update) + } + + fn after_transaction(&self, object_id: &str, txn: &mut TransactionMut) { + (**self).after_transaction(object_id, txn) + } + fn plugin_type(&self) -> CollabPluginType { + (**self).plugin_type() + } + + fn reset(&self, object_id: &str) { + (**self).reset(object_id) + } + + fn flush(&self, object_id: &str, doc: &Doc) { + (**self).flush(object_id, doc) + } } #[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] From 3c2cb055e47ec9d6bff3d3aeb2a476b85d02cb80 Mon Sep 17 00:00:00 2001 From: "Nathan.fooo" <86001920+appflowy@users.noreply.github.com> Date: Wed, 10 Jan 2024 15:59:05 +0800 Subject: [PATCH 10/13] chore: return orphan views (#148) --- collab-folder/Cargo.toml | 2 +- collab-folder/src/folder.rs | 7 ++ collab-folder/src/view.rs | 18 ++++- .../tests/folder_test/child_views_test.rs | 75 +++++++++++++++++++ 4 files changed, 100 insertions(+), 2 deletions(-) diff --git a/collab-folder/Cargo.toml b/collab-folder/Cargo.toml index a858ace98..fc0ff5bac 100644 --- a/collab-folder/Cargo.toml +++ b/collab-folder/Cargo.toml @@ -28,7 +28,7 @@ collab-plugins = { workspace = true } fs_extra = "1.2.0" nanoid = "0.4.0" tempfile = "3.8.0" -tokio = {version = "1.26", features = ["rt"]} +tokio = {version = "1.26", features = ["rt", "macros"]} tracing-subscriber = {version = "0.3.3", features = ["env-filter"]} walkdir = "2.3.2" zip = "0.6.6" diff --git a/collab-folder/src/folder.rs b/collab-folder/src/folder.rs index 808ec584b..21b6fd116 100644 --- a/collab-folder/src/folder.rs +++ b/collab-folder/src/folder.rs @@ -182,9 +182,16 @@ impl Folder { let current_view = self.get_current_view_with_txn(&txn).unwrap_or_default(); let mut views = vec![]; + let orphan_views = self + .views + .get_orphan_views_with_txn(&txn) + .iter() + .map(|view| view.as_ref().clone()) + .collect::>(); for view in self.get_workspace_views_with_txn(&txn) { views.extend(self.get_view_recursively_with_txn(&txn, &view.id)); } + views.extend(orphan_views); let favorites = self .section diff --git a/collab-folder/src/view.rs b/collab-folder/src/view.rs index 5a475d180..97139838b 100644 --- a/collab-folder/src/view.rs +++ b/collab-folder/src/view.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use anyhow::bail; use collab::core::collab::IndexContentSender; use collab::preclude::{ - Any, DeepEventsSubscription, MapRef, MapRefExtension, MapRefWrapper, ReadTxn, TransactionMut, + Any, DeepEventsSubscription, Map, MapRef, MapRefExtension, MapRefWrapper, ReadTxn, TransactionMut, }; use parking_lot::RwLock; use serde::{Deserialize, Serialize}; @@ -219,6 +219,22 @@ impl ViewsMap { self.get_view_with_txn(&txn, view_id) } + pub fn get_orphan_views(&self) -> Vec> { + let txn = self.container.transact(); + self.get_orphan_views_with_txn(&txn) + } + + /// Return the orphan views. + /// The orphan views are the views that its parent_view_id equal to its view_id. + pub fn get_orphan_views_with_txn(&self, txn: &T) -> Vec> { + self + .container + .keys(txn) + .flat_map(|view_id| self.get_view_with_txn(txn, view_id)) + .filter(|view| view.parent_view_id == view.id) + .collect() + } + /// Return the view with the given view id. /// The view is support nested, by default, we only load the view and its children. pub fn get_view_with_txn(&self, txn: &T, view_id: &str) -> Option> { diff --git a/collab-folder/tests/folder_test/child_views_test.rs b/collab-folder/tests/folder_test/child_views_test.rs index 50cc5a52f..57caa5bdf 100644 --- a/collab-folder/tests/folder_test/child_views_test.rs +++ b/collab-folder/tests/folder_test/child_views_test.rs @@ -216,3 +216,78 @@ async fn delete_child_view_test() { let views = folder_test.views.get_views_belong_to("v1"); assert!(views.is_empty()); } + +#[tokio::test] +async fn create_orphan_child_views_test() { + let uid = UserId::from(1); + let folder_test = create_folder_with_workspace(uid.clone(), "fake_w_1").await; + let workspace_id = folder_test.get_workspace_id(); + let view_1 = make_test_view("1", "fake_w_1", vec![]); + + // The orphan view: the parent_view_id equal to the view_id + let view_2 = make_test_view("2", "2", vec![]); + + folder_test.insert_view(view_1.clone(), None); + folder_test.insert_view(view_2.clone(), None); + + let child_views = folder_test.views.get_views_belong_to(&workspace_id); + assert_eq!(child_views.len(), 1); + + let orphan_views = folder_test.views.get_orphan_views(); + assert_eq!(orphan_views.len(), 1); + + // The folder data should contains the orphan view + let folder_data = folder_test.get_folder_data().unwrap(); + assert_json_include!( + actual: json!(folder_data), + expected: json!({ + "current_view": "", + "favorites": {}, + "recent": {}, + "trash": {}, + "views": [ + { + "children": { + "items": [] + }, + "created_by": 1, + "desc": "", + "icon": null, + "id": "1", + "is_favorite": false, + "last_edited_by": 1, + "layout": 0, + "name": "", + "parent_view_id": "fake_w_1" + }, + { + "children": { + "items": [] + }, + "created_by": 1, + "desc": "", + "icon": null, + "id": "2", + "is_favorite": false, + "last_edited_by": 1, + "layout": 0, + "name": "", + "parent_view_id": "2" + } + ], + "workspace": { + "child_views": { + "items": [ + { + "id": "1" + } + ] + }, + "created_by": 1, + "id": "fake_w_1", + "last_edited_by": 1, + "name": "" + } + }) + ); +} From 3c6b188a2bcd25da2787d891a4faf6e085c8984c Mon Sep 17 00:00:00 2001 From: "Lucas.Xu" Date: Thu, 11 Jan 2024 14:12:33 +0700 Subject: [PATCH 11/13] fix: log file name too long (#149) --- collab-plugins/src/local_storage/rocksdb/kv_impl.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collab-plugins/src/local_storage/rocksdb/kv_impl.rs b/collab-plugins/src/local_storage/rocksdb/kv_impl.rs index 4245a4b5c..5b0a8a8d5 100644 --- a/collab-plugins/src/local_storage/rocksdb/kv_impl.rs +++ b/collab-plugins/src/local_storage/rocksdb/kv_impl.rs @@ -60,9 +60,9 @@ impl RocksStore { db_opts.set_level_zero_stop_writes_trigger(10); // log + // don't set the log dir (set_db_log_dir) because it will cause the 'file name too long' error on mobile platform db_opts.set_recycle_log_file_num(5); db_opts.set_keep_log_file_num(5); - db_opts.set_db_log_dir(path.as_ref().join("logs")); let open_result = TransactionDB::::open(&db_opts, &txn_db_opts, &path); let db = match open_result { From ae417ce8ccbc65e26062fb53690ebf0f7b2853e4 Mon Sep 17 00:00:00 2001 From: "Nathan.fooo" <86001920+appflowy@users.noreply.github.com> Date: Sun, 21 Jan 2024 12:39:49 +0800 Subject: [PATCH 12/13] feat: collab plugin wasm (#150) * refactor: KVTransactionDB * refactor: KVTransactionDB * chore: add indexeddb * test: add indexeddb test * chore: impl stream * chore: add test * chore: add test * ci: add test ci * ci: add test ci --- .github/workflows/wasm_test.yml | 41 ++ Cargo.lock | 275 ++++++++++-- Makefile.toml | 23 +- collab-database/Cargo.toml | 6 +- collab-database/src/blocks/block.rs | 1 + collab-database/src/blocks/task_controller.rs | 2 +- collab-database/src/id_gen/gen.rs | 16 +- collab-database/src/rows/row.rs | 1 + collab-database/src/user/user_db.rs | 1 + collab-database/tests/database_test/helper.rs | 2 +- .../tests/database_test/restore_test.rs | 2 +- collab-database/tests/helper/util.rs | 2 +- .../tests/user_test/async_test/script.rs | 3 +- collab-database/tests/user_test/helper.rs | 2 +- collab-document/Cargo.toml | 2 +- .../tests/document/restore_test.rs | 2 +- collab-document/tests/util.rs | 2 +- collab-folder/tests/folder_test/util.rs | 4 +- collab-plugins/Cargo.toml | 16 +- .../src/cloud_storage/postgres/plugin.rs | 2 +- collab-plugins/src/lib.rs | 25 +- .../indexeddb/indexeddb_plugin.rs | 177 ++++++++ .../src/local_storage/indexeddb/kv_impl.rs | 413 ++++++++++++++++++ .../src/local_storage/indexeddb/mod.rs | 2 + collab-plugins/src/local_storage/kv/db.rs | 19 +- collab-plugins/src/local_storage/kv/doc.rs | 27 +- collab-plugins/src/local_storage/kv/error.rs | 20 +- collab-plugins/src/local_storage/kv/mod.rs | 2 +- collab-plugins/src/local_storage/kv/oid.rs | 22 +- .../src/local_storage/kv/snapshot.rs | 2 +- collab-plugins/src/local_storage/mod.rs | 36 +- .../src/local_storage/rocksdb/kv_impl.rs | 85 ++-- .../local_storage/rocksdb/rocksdb_plugin.rs | 1 + .../local_storage/rocksdb/snapshot_plugin.rs | 2 +- .../src/local_storage/storage_config.rs | 34 ++ collab-plugins/tests/disk/insert_test.rs | 1 + collab-plugins/tests/disk/range_test.rs | 4 +- collab-plugins/tests/disk/restore_test.rs | 5 +- collab-plugins/tests/disk/script.rs | 3 +- collab-plugins/tests/disk/util.rs | 2 +- collab-plugins/tests/main.rs | 6 +- collab-plugins/tests/web/edit_collab_test.rs | 106 +++++ collab-plugins/tests/web/indexeddb_test.rs | 97 ++++ collab-plugins/tests/web/mod.rs | 5 + collab-plugins/tests/web/setup_tests.js | 6 + collab-plugins/tests/web/test.md | 17 + collab/Cargo.toml | 3 + collab/src/core/transaction.rs | 48 +- collab/src/lib.rs | 16 + 49 files changed, 1425 insertions(+), 166 deletions(-) create mode 100644 .github/workflows/wasm_test.yml create mode 100644 collab-plugins/src/local_storage/indexeddb/indexeddb_plugin.rs create mode 100644 collab-plugins/src/local_storage/indexeddb/kv_impl.rs create mode 100644 collab-plugins/src/local_storage/indexeddb/mod.rs create mode 100644 collab-plugins/src/local_storage/storage_config.rs create mode 100644 collab-plugins/tests/web/edit_collab_test.rs create mode 100644 collab-plugins/tests/web/indexeddb_test.rs create mode 100644 collab-plugins/tests/web/mod.rs create mode 100644 collab-plugins/tests/web/setup_tests.js create mode 100644 collab-plugins/tests/web/test.md diff --git a/.github/workflows/wasm_test.yml b/.github/workflows/wasm_test.yml new file mode 100644 index 000000000..2cc4387c0 --- /dev/null +++ b/.github/workflows/wasm_test.yml @@ -0,0 +1,41 @@ +name: Wasm Tests + +on: + pull_request: + +env: + RUST_TOOLCHAIN: "1.75" + CARGO_MAKE_VERSION: "0.36.6" + +jobs: + test: + name: Run Wasm Tests + runs-on: ubuntu-latest + + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + + - name: Install wasm-pack + uses: jetli/wasm-pack-action@v0.3.0 + with: + version: latest + + - uses: taiki-e/install-action@v2 + with: + tool: cargo-make@${{ env.CARGO_MAKE_VERSION }} + + - name: Install Node.js + uses: actions/setup-node@v2 + with: + node-version: '14' + + - name: Run Wasm Tests + run: cargo make wasm_test diff --git a/Cargo.lock b/Cargo.lock index eb678d22c..3f591c3bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,18 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "accessory" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "850bb534b9dc04744fbbb71d30ad6d25a7e4cf6dc33e223c81ef3a92ebab4e0b" +dependencies = [ + "macroific", + "proc-macro2", + "quote", + "syn 2.0.46", +] + [[package]] name = "adler" version = "1.0.2" @@ -67,6 +79,28 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.46", +] + [[package]] name = "async-trait" version = "0.1.73" @@ -278,6 +312,7 @@ dependencies = [ "tokio-stream", "tracing", "tracing-subscriber", + "web-sys", "yrs", ] @@ -294,6 +329,7 @@ dependencies = [ "collab-plugins", "futures", "getrandom 0.2.9", + "js-sys", "lazy_static", "lru", "nanoid", @@ -384,6 +420,7 @@ version = "0.1.0" dependencies = [ "anyhow", "assert-json-diff", + "async-stream", "async-trait", "bincode", "bytes", @@ -395,6 +432,8 @@ dependencies = [ "futures", "futures-util", "getrandom 0.2.9", + "indexed_db_futures", + "js-sys", "lazy_static", "parking_lot", "rand 0.8.5", @@ -411,7 +450,12 @@ dependencies = [ "tokio-util", "tracing", "tracing-subscriber", + "tracing-wasm", "uuid", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test", + "web-sys", "yrs", ] @@ -451,6 +495,16 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "console_error_panic_hook" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -544,6 +598,18 @@ dependencies = [ "syn 2.0.46", ] +[[package]] +name = "delegate-display" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98a85201f233142ac819bbf6226e36d0b5e129a47bd325084674261c82d4cd66" +dependencies = [ + "macroific", + "proc-macro2", + "quote", + "syn 2.0.46", +] + [[package]] name = "digest" version = "0.10.7" @@ -582,6 +648,18 @@ dependencies = [ "libc", ] +[[package]] +name = "fancy_constructor" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f71f317e4af73b2f8f608fac190c52eac4b1879d2145df1db2fe48881ca69435" +dependencies = [ + "macroific", + "proc-macro2", + "quote", + "syn 2.0.46", +] + [[package]] name = "fastrand" version = "2.0.0" @@ -606,9 +684,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -621,9 +699,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -631,15 +709,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -648,15 +726,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -665,21 +743,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -784,6 +862,23 @@ dependencies = [ "cxx-build", ] +[[package]] +name = "indexed_db_futures" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cc2083760572ee02385ab8b7c02c20925d2dd1f97a1a25a8737a238608f1152" +dependencies = [ + "accessory", + "cfg-if", + "delegate-display", + "fancy_constructor", + "js-sys", + "uuid", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "inout" version = "0.1.3" @@ -810,9 +905,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.61" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" dependencies = [ "wasm-bindgen", ] @@ -920,6 +1015,53 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "macroific" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05c00ac596022625d01047c421a0d97d7f09a18e429187b341c201cb631b9dd" +dependencies = [ + "macroific_attr_parse", + "macroific_core", + "macroific_macro", +] + +[[package]] +name = "macroific_attr_parse" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd94d5da95b30ae6e10621ad02340909346ad91661f3f8c0f2b62345e46a2f67" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 2.0.46", +] + +[[package]] +name = "macroific_core" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13198c120864097a565ccb3ff947672d969932b7975ebd4085732c9f09435e55" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.46", +] + +[[package]] +name = "macroific_macro" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c9853143cbed7f1e41dc39fee95f9b361bec65c8dc2a01bf609be01b61f5ae" +dependencies = [ + "macroific_attr_parse", + "macroific_core", + "proc-macro2", + "quote", + "syn 2.0.46", +] + [[package]] name = "matchers" version = "0.1.0" @@ -1293,6 +1435,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.1.0" @@ -1657,6 +1805,17 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "tracing-wasm" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4575c663a174420fa2d78f4108ff68f65bf2fbb7dd89f33749b6e826b3626e07" +dependencies = [ + "tracing", + "tracing-subscriber", + "wasm-bindgen", +] + [[package]] name = "typenum" version = "1.16.0" @@ -1677,12 +1836,13 @@ checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] name = "uuid" -version = "1.3.3" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "345444e32442451b267fc254ae85a209c64be56d2890e601a0c37ff0c3c5ecd2" +checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" dependencies = [ "getrandom 0.2.9", "sha1_smol", + "wasm-bindgen", ] [[package]] @@ -1727,9 +1887,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.84" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1737,24 +1897,36 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.84" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.46", "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" -version = "0.2.84" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1762,22 +1934,57 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.84" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.46", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.84" +version = "0.2.90" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" + +[[package]] +name = "wasm-bindgen-test" +version = "0.3.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139bd73305d50e1c1c4333210c0db43d989395b64a237bd35c10ef3832a7f70c" +dependencies = [ + "console_error_panic_hook", + "js-sys", + "scoped-tls", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test-macro", +] + +[[package]] +name = "wasm-bindgen-test-macro" +version = "0.3.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70072aebfe5da66d2716002c729a14e4aec4da0e23cc2ea66323dac541c93928" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.46", +] + +[[package]] +name = "web-sys" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" +dependencies = [ + "js-sys", + "wasm-bindgen", +] [[package]] name = "winapi" diff --git a/Makefile.toml b/Makefile.toml index c54f8f622..f4ccd8567 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -83,7 +83,7 @@ script = [ """ #!/bin/bash BASE_DIR=$(pwd) - crates=("collab" "collab-document" "collab-folder" "collab-user") + crates=("collab" "collab-document" "collab-folder" "collab-user" "collab-plugins") # Iterate over each crate and build it for crate in "${crates[@]}"; do @@ -94,8 +94,27 @@ script = [ # Build the crate wasm-pack build --features="wasm_build" || { echo "Build failed for $crate"; exit 1; } + done + """ +] + +[tasks.wasm_test] +script_runner = "bash" +script = [ + """ + #!/bin/bash + BASE_DIR=$(pwd) + crates=("collab-plugins") + + # Iterate over each crate and build it + for crate in "${crates[@]}"; do + echo "🔥🔥🔥 Running $crate tests with wasm-pack..." - # No need to cd back since we use absolute paths + # Navigate to the crate directory + cd "$BASE_DIR/$crate" || { echo "Failed to enter directory $crate"; exit 1; } + + # Build the crate + wasm-pack test --headless --firefox --features="wasm_build" done """ ] diff --git a/collab-database/Cargo.toml b/collab-database/Cargo.toml index d430ffd16..3ea90b721 100644 --- a/collab-database/Cargo.toml +++ b/collab-database/Cargo.toml @@ -30,6 +30,10 @@ strum = "0.25" strum_macros = "0.25" getrandom = { version = "0.2", optional = true } + +[target.'cfg(target_arch = "wasm32")'.dependencies] +js-sys = "0.3" + [dev-dependencies] collab-plugins = { workspace = true } tempfile = "3.8.0" @@ -38,7 +42,7 @@ assert-json-diff = "2.0.2" lazy_static = "1.4.0" tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } rand = "0.8.4" -futures = "0.3.18" +futures = "0.3.30" zip = "0.6.6" [features] diff --git a/collab-database/src/blocks/block.rs b/collab-database/src/blocks/block.rs index 7870d9485..6281d54b9 100644 --- a/collab-database/src/blocks/block.rs +++ b/collab-database/src/blocks/block.rs @@ -6,6 +6,7 @@ use collab::core::collab::{CollabDocState, MutexCollab}; use collab_entity::CollabType; use collab_plugins::local_storage::kv::doc::CollabKVAction; +use collab_plugins::local_storage::kv::KVTransactionDB; use collab_plugins::local_storage::CollabPersistenceConfig; use collab_plugins::CollabKVDB; use lru::LruCache; diff --git a/collab-database/src/blocks/task_controller.rs b/collab-database/src/blocks/task_controller.rs index b5cd7cc80..331e7245e 100644 --- a/collab-database/src/blocks/task_controller.rs +++ b/collab-database/src/blocks/task_controller.rs @@ -9,7 +9,7 @@ use collab::core::collab::{CollabDocState, MutexCollab}; use collab::core::origin::CollabOrigin; use collab_entity::CollabType; use collab_plugins::local_storage::kv::doc::CollabKVAction; -use collab_plugins::local_storage::kv::PersistenceError; +use collab_plugins::local_storage::kv::{KVTransactionDB, PersistenceError}; use collab_plugins::CollabKVDB; use tokio::sync::watch; diff --git a/collab-database/src/id_gen/gen.rs b/collab-database/src/id_gen/gen.rs index 92720e5bf..0f29c5354 100644 --- a/collab-database/src/id_gen/gen.rs +++ b/collab-database/src/id_gen/gen.rs @@ -1,4 +1,4 @@ -use std::time::SystemTime; +use collab_plugins::{if_native, if_wasm}; /// Equivalent to April 9, 2023 4:18:02 AM const EPOCH: u64 = 1637806706000; @@ -57,11 +57,19 @@ impl RowIDGen { } } - fn timestamp(&self) -> u64 { - SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) + if_native! { + fn timestamp(&self) -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::SystemTime::UNIX_EPOCH) .expect("Clock moved backwards!") .as_millis() as u64 + } + } + + if_wasm! { + fn timestamp(&self) -> u64 { + js_sys::Date::now() as u64 + } } } diff --git a/collab-database/src/rows/row.rs b/collab-database/src/rows/row.rs index 1659ef87e..86f46e922 100644 --- a/collab-database/src/rows/row.rs +++ b/collab-database/src/rows/row.rs @@ -10,6 +10,7 @@ use parking_lot::Mutex; use collab::core::value::YrsValueExtension; use collab_plugins::local_storage::kv::doc::CollabKVAction; +use collab_plugins::local_storage::kv::KVTransactionDB; use collab_plugins::CollabKVDB; use serde::{Deserialize, Serialize}; use tracing::error; diff --git a/collab-database/src/user/user_db.rs b/collab-database/src/user/user_db.rs index 78dcf0d57..599f3e3a0 100644 --- a/collab-database/src/user/user_db.rs +++ b/collab-database/src/user/user_db.rs @@ -12,6 +12,7 @@ use collab::preclude::{Collab, Update}; use collab_entity::CollabType; use collab_plugins::local_storage::kv::doc::CollabKVAction; use collab_plugins::local_storage::kv::snapshot::{CollabSnapshot, SnapshotAction}; +use collab_plugins::local_storage::kv::KVTransactionDB; use collab_plugins::local_storage::CollabPersistenceConfig; use collab_plugins::CollabKVDB; use parking_lot::Mutex; diff --git a/collab-database/tests/database_test/helper.rs b/collab-database/tests/database_test/helper.rs index 23ea8e9dd..66ddf6073 100644 --- a/collab-database/tests/database_test/helper.rs +++ b/collab-database/tests/database_test/helper.rs @@ -189,7 +189,7 @@ impl DatabaseTestBuilder { pub async fn build(self) -> DatabaseTest { let tempdir = TempDir::new().unwrap(); let path = tempdir.into_path(); - let collab_db = Arc::new(CollabKVDB::open_opt(path, false).unwrap()); + let collab_db = Arc::new(CollabKVDB::open(path).unwrap()); let collab = CollabBuilder::new(self.uid, &self.database_id) .with_device_id("1") .build() diff --git a/collab-database/tests/database_test/restore_test.rs b/collab-database/tests/database_test/restore_test.rs index 8b796575b..468fa17d1 100644 --- a/collab-database/tests/database_test/restore_test.rs +++ b/collab-database/tests/database_test/restore_test.rs @@ -127,7 +127,7 @@ const HISTORY_DOCUMENT_020: &str = "020_database"; #[tokio::test] async fn open_020_history_database_test() { let (_cleaner, db_path) = unzip_history_database_db(HISTORY_DOCUMENT_020).unwrap(); - let db = std::sync::Arc::new(CollabKVDB::open_opt(db_path, false).unwrap()); + let db = std::sync::Arc::new(CollabKVDB::open(db_path).unwrap()); let database_test = restore_database_from_db( 221439819971039232, "c0e69740-49f0-4790-a488-702e2750ba8d", diff --git a/collab-database/tests/helper/util.rs b/collab-database/tests/helper/util.rs index 9742a1fc1..58a2c1ac2 100644 --- a/collab-database/tests/helper/util.rs +++ b/collab-database/tests/helper/util.rs @@ -587,7 +587,7 @@ impl From for AnyMap { pub fn make_rocks_db() -> Arc { let tempdir = TempDir::new().unwrap(); let path = tempdir.into_path(); - Arc::new(CollabKVDB::open_opt(path, false).unwrap()) + Arc::new(CollabKVDB::open(path).unwrap()) } pub fn setup_log() { diff --git a/collab-database/tests/user_test/async_test/script.rs b/collab-database/tests/user_test/async_test/script.rs index 8dfc574f4..d671bed1d 100644 --- a/collab-database/tests/user_test/async_test/script.rs +++ b/collab-database/tests/user_test/async_test/script.rs @@ -10,6 +10,7 @@ use collab_database::rows::{Cells, CellsBuilder, RowId}; use collab_database::user::WorkspaceDatabase; use collab_database::views::{CreateDatabaseParams, OrderObjectPosition}; use collab_plugins::local_storage::kv::doc::CollabKVAction; +use collab_plugins::local_storage::kv::KVTransactionDB; use collab_plugins::local_storage::CollabPersistenceConfig; use collab_plugins::CollabKVDB; use serde_json::Value; @@ -72,7 +73,7 @@ impl DatabaseTest { pub async fn new(config: CollabPersistenceConfig) -> Self { let tempdir = TempDir::new().unwrap(); let db_path = tempdir.into_path(); - let collab_db = Arc::new(CollabKVDB::open_opt(db_path.clone(), false).unwrap()); + let collab_db = Arc::new(CollabKVDB::open(db_path.clone()).unwrap()); let workspace_database = workspace_database_with_db(1, Arc::downgrade(&collab_db), Some(config.clone())).await; Self { diff --git a/collab-database/tests/user_test/helper.rs b/collab-database/tests/user_test/helper.rs index 49df3101e..a893165ad 100644 --- a/collab-database/tests/user_test/helper.rs +++ b/collab-database/tests/user_test/helper.rs @@ -162,7 +162,7 @@ pub async fn user_database_test_with_db( pub async fn user_database_test_with_default_data(uid: i64) -> WorkspaceDatabaseTest { let tempdir = TempDir::new().unwrap(); let path = tempdir.into_path(); - let db = Arc::new(CollabKVDB::open_opt(path, false).unwrap()); + let db = Arc::new(CollabKVDB::open(path).unwrap()); let w_database = user_database_test_with_db(uid, db).await; w_database diff --git a/collab-document/Cargo.toml b/collab-document/Cargo.toml index 302c4d9bc..2556a0361 100644 --- a/collab-document/Cargo.toml +++ b/collab-document/Cargo.toml @@ -28,7 +28,7 @@ tempfile = "3.8.0" tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } collab-plugins = { workspace = true } zip = "0.6.6" -futures = "0.3.17" +futures = "0.3.30" [features] diff --git a/collab-document/tests/document/restore_test.rs b/collab-document/tests/document/restore_test.rs index 01cc52085..d3c4332b7 100644 --- a/collab-document/tests/document/restore_test.rs +++ b/collab-document/tests/document/restore_test.rs @@ -94,7 +94,7 @@ const HISTORY_DOCUMENT_020: &str = "020_document"; #[tokio::test] async fn open_020_history_document_test() { let (_cleaner, db_path) = unzip_history_document_db(HISTORY_DOCUMENT_020).unwrap(); - let db = std::sync::Arc::new(CollabKVDB::open_opt(db_path, false).unwrap()); + let db = std::sync::Arc::new(CollabKVDB::open(db_path).unwrap()); let document = open_document_with_db( 221439819971039232, "631584ec-af71-42c3-94f4-89dcfdafb988", diff --git a/collab-document/tests/util.rs b/collab-document/tests/util.rs index 2a62c66e8..1ece44c32 100644 --- a/collab-document/tests/util.rs +++ b/collab-document/tests/util.rs @@ -130,7 +130,7 @@ pub async fn open_document_with_db(uid: i64, doc_id: &str, db: Arc) pub fn document_storage() -> Arc { let tempdir = TempDir::new().unwrap(); let path = tempdir.into_path(); - Arc::new(CollabKVDB::open_opt(path, false).unwrap()) + Arc::new(CollabKVDB::open(path).unwrap()) } fn setup_log() { diff --git a/collab-folder/tests/folder_test/util.rs b/collab-folder/tests/folder_test/util.rs index f2bada3b8..8b0120345 100644 --- a/collab-folder/tests/folder_test/util.rs +++ b/collab-folder/tests/folder_test/util.rs @@ -52,7 +52,7 @@ pub async fn create_folder_with_data( let tempdir = TempDir::new().unwrap(); let path = tempdir.into_path(); - let db = Arc::new(CollabKVDB::open_opt(path.clone(), false).unwrap()); + let db = Arc::new(CollabKVDB::open(path.clone()).unwrap()); let disk_plugin = RocksdbDiskPlugin::new( uid.as_i64(), workspace_id.to_string(), @@ -86,7 +86,7 @@ pub async fn create_folder_with_data( } pub async fn open_folder_with_db(uid: UserId, object_id: &str, db_path: PathBuf) -> FolderTest { - let db = Arc::new(CollabKVDB::open_opt(db_path.clone(), false).unwrap()); + let db = Arc::new(CollabKVDB::open(db_path.clone()).unwrap()); let disk_plugin = RocksdbDiskPlugin::new( uid.as_i64(), object_id.to_string(), diff --git a/collab-plugins/Cargo.toml b/collab-plugins/Cargo.toml index 35324706f..4f71ef27c 100644 --- a/collab-plugins/Cargo.toml +++ b/collab-plugins/Cargo.toml @@ -47,10 +47,22 @@ assert-json-diff = "2.0.2" tokio-util = { version = "0.7", features = ["codec"] } config = { version = "0.13.3", default-features = false, features = ["yaml"] } dotenv = "0.15.0" -futures = "0.3.17" +futures = "0.3" +[target.'cfg(target_arch = "wasm32")'.dependencies] +indexed_db_futures = { version = "0.4.1" } +js-sys = "0.3" +async-stream = "0.3.4" +futures = "0.3" +wasm-bindgen = "0.2" +web-sys = { version = "0.3.67", features = ["console", "Window"] } +wasm-bindgen-futures = "0.4.40" +tracing-wasm = "0.2" + +[target.'cfg(target_arch = "wasm32")'.dev-dependencies] +wasm-bindgen-test = "0.3.40" [features] default = [] postgres_plugin = ["rand"] -wasm_build = ["getrandom/js"] \ No newline at end of file +wasm_build = ["getrandom/js", "collab/async-plugin"] \ No newline at end of file diff --git a/collab-plugins/src/cloud_storage/postgres/plugin.rs b/collab-plugins/src/cloud_storage/postgres/plugin.rs index d98907c91..465250891 100644 --- a/collab-plugins/src/cloud_storage/postgres/plugin.rs +++ b/collab-plugins/src/cloud_storage/postgres/plugin.rs @@ -24,7 +24,7 @@ use crate::cloud_storage::remote_collab::{ }; use crate::cloud_storage::sink::{SinkConfig, SinkStrategy}; use crate::local_storage::kv::doc::CollabKVAction; -use crate::local_storage::kv::TransactionMutExt; +use crate::local_storage::kv::{KVTransactionDB, TransactionMutExt}; use crate::CollabKVDB; pub struct SupabaseDBPlugin { diff --git a/collab-plugins/src/lib.rs b/collab-plugins/src/lib.rs index b90190e98..351858836 100644 --- a/collab-plugins/src/lib.rs +++ b/collab-plugins/src/lib.rs @@ -1,8 +1,29 @@ pub mod local_storage; +#[macro_export] +macro_rules! if_native { + ($($item:item)*) => {$( + #[cfg(not(target_arch = "wasm32"))] + $item + )*} +} + +#[macro_export] +macro_rules! if_wasm { + ($($item:item)*) => {$( + #[cfg(target_arch = "wasm32")] + $item + )*} +} + #[cfg(all(feature = "postgres_plugin", not(target_arch = "wasm32")))] pub mod cloud_storage; pub mod connect_state; -#[cfg(not(target_arch = "wasm32"))] -pub type CollabKVDB = local_storage::rocksdb::kv_impl::RocksStore; +if_native! { + pub type CollabKVDB = local_storage::rocksdb::kv_impl::KVTransactionDBRocksdbImpl; +} + +if_wasm! { + pub type CollabKVDB = local_storage::indexeddb::kv_impl::CollabIndexeddb; +} diff --git a/collab-plugins/src/local_storage/indexeddb/indexeddb_plugin.rs b/collab-plugins/src/local_storage/indexeddb/indexeddb_plugin.rs new file mode 100644 index 000000000..c316e915a --- /dev/null +++ b/collab-plugins/src/local_storage/indexeddb/indexeddb_plugin.rs @@ -0,0 +1,177 @@ +use crate::local_storage::indexeddb::kv_impl::CollabIndexeddb; +use crate::local_storage::kv::keys::{make_doc_state_key, make_state_vector_key}; + +use async_stream::stream; +use async_trait::async_trait; +use collab::core::awareness::Awareness; + +use collab::core::origin::CollabOrigin; +use collab::preclude::CollabPlugin; +use collab_entity::CollabType; +use futures::stream::StreamExt; + +use crate::local_storage::kv::PersistenceError; +use collab::core::collab::make_yrs_doc; +use collab::core::transaction::DocTransactionExtension; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering::SeqCst; +use std::sync::{Arc, Weak}; +use tracing::{error, instrument}; +use wasm_bindgen_futures::spawn_local; +use yrs::{Doc, TransactionMut}; + +pub struct IndexeddbDiskPlugin { + uid: i64, + #[allow(dead_code)] + object_id: String, + #[allow(dead_code)] + collab_type: CollabType, + collab_db: Weak, + did_load: Arc, + edit_sender: DocEditStreamSender, +} + +impl IndexeddbDiskPlugin { + pub fn new( + uid: i64, + object_id: String, + collab_type: CollabType, + collab_db: Weak, + ) -> Self { + let did_load = Arc::new(AtomicBool::new(false)); + let (edit_sender, rx) = tokio::sync::mpsc::unbounded_channel(); + let edit_stream = DocEditStream::new(uid, &object_id, collab_db.clone(), rx); + spawn_local(edit_stream.run()); + Self { + uid, + object_id, + collab_type, + did_load, + collab_db, + edit_sender, + } + } + + #[instrument(skip_all)] + fn flush_doc(&self, db: Arc, object_id: &str) { + let uid = self.uid; + let object_id = object_id.to_string(); + spawn_local(async move { + let doc = make_yrs_doc(); + db.load_doc(uid, &object_id, doc.clone()).await.unwrap(); + let encoded_collab = doc.get_encoded_collab_v1(); + db.flush_doc(uid, &object_id, &encoded_collab) + .await + .unwrap(); + }); + } +} + +#[async_trait] +impl CollabPlugin for IndexeddbDiskPlugin { + async fn init(&self, object_id: &str, _origin: &CollabOrigin, doc: &Doc) { + if let Some(db) = self.collab_db.upgrade() { + let object_id = object_id.to_string(); + let doc = doc.clone(); + let uid = self.uid; + + spawn_local(async move { + match db.load_doc(uid, &object_id, doc.clone()).await { + Ok(_) => {}, + Err(err) => { + if err.is_record_not_found() { + let encoded_collab = doc.get_encoded_collab_v1(); + let f = || async move { + let doc_id = db.create_doc_id(uid, object_id).await?; + let doc_state_key = make_doc_state_key(doc_id); + let sv_key = make_state_vector_key(doc_id); + db.set_data(doc_state_key, encoded_collab.doc_state).await?; + db.set_data(sv_key, encoded_collab.state_vector).await?; + Ok::<(), PersistenceError>(()) + }; + if let Err(err) = f().await { + error!("failed to create doc_id: {:?}", err); + } + } else { + error!("failed to get encoded collab: {:?}", err); + } + }, + } + }); + } else { + tracing::warn!("collab_db is dropped"); + } + } + + fn receive_update(&self, _object_id: &str, _txn: &TransactionMut, update: &[u8]) { + // Only push update if the doc is loaded + if !self.did_load.load(SeqCst) { + return; + } + if let Err(err) = self.edit_sender.send(DocUpdate::Update(update.to_vec())) { + error!("failed to send update: {}", err); + } + } + + fn did_init(&self, _awareness: &Awareness, _object_id: &str, _last_sync_at: i64) { + self.did_load.store(true, SeqCst); + } + + fn flush(&self, object_id: &str, _doc: &Doc) { + if let Some(db) = self.collab_db.upgrade() { + self.flush_doc(db, object_id); + } + } +} + +type DocEditStreamSender = tokio::sync::mpsc::UnboundedSender; +type DocEditStreamReceiver = tokio::sync::mpsc::UnboundedReceiver; +struct DocEditStream { + uid: i64, + object_id: String, + collab_db: Weak, + receiver: Option, +} + +#[derive(Clone)] +enum DocUpdate { + Update(Vec), +} + +impl DocEditStream { + fn new( + uid: i64, + object_id: &str, + collab_db: Weak, + receiver: DocEditStreamReceiver, + ) -> Self { + Self { + uid, + object_id: object_id.to_string(), + collab_db, + receiver: Some(receiver), + } + } + + async fn run(mut self) { + let mut receiver = self.receiver.take().expect("Only take once"); + let stream = stream! { + while let Some(data) = receiver.recv().await { + yield data; + } + }; + stream + .for_each(|data| async { + match data { + DocUpdate::Update(update) => { + if let Some(db) = self.collab_db.upgrade() { + if let Err(err) = db.push_update(self.uid, &self.object_id, &update).await { + error!("failed to push update: {}", err); + } + } + }, + } + }) + .await; + } +} diff --git a/collab-plugins/src/local_storage/indexeddb/kv_impl.rs b/collab-plugins/src/local_storage/indexeddb/kv_impl.rs new file mode 100644 index 000000000..3d9ebadf8 --- /dev/null +++ b/collab-plugins/src/local_storage/indexeddb/kv_impl.rs @@ -0,0 +1,413 @@ +use crate::local_storage::kv::PersistenceError; +use collab::core::collab_plugin::EncodedCollab; +use indexed_db_futures::js_sys::wasm_bindgen::JsValue; +use indexed_db_futures::prelude::*; +use js_sys::{ArrayBuffer, Uint8Array}; + +use crate::local_storage::kv::keys::{ + clock_from_key, make_doc_end_key, make_doc_id_key, make_doc_start_key, make_doc_state_key, + make_doc_update_key, make_state_vector_key, Clock, DocID, DOC_ID_LEN, +}; +use crate::local_storage::kv::oid::{LOCAL_DOC_ID_GEN, OID}; +use anyhow::anyhow; +use collab::core::collab::TransactionMutExt; +use indexed_db_futures::web_sys::IdbKeyRange; +use std::sync::Arc; +use tokio::sync::RwLock; +use tracing::error; +use wasm_bindgen::JsCast; +use web_sys::console; +use yrs::updates::decoder::Decode; +use yrs::{Doc, Transact, Update}; + +pub struct CollabIndexeddb { + db: Arc>, +} + +unsafe impl Send for CollabIndexeddb {} +unsafe impl Sync for CollabIndexeddb {} + +const COLLAB_KV_STORE: &str = "collab_kv"; +impl CollabIndexeddb { + pub async fn new() -> Result { + let mut db_req = IdbDatabase::open_u32("appflowy_indexeddb", 1)?; + db_req.set_on_upgrade_needed(Some(|evt: &IdbVersionChangeEvent| -> Result<(), JsValue> { + if evt + .db() + .object_store_names() + .find(|n| n == COLLAB_KV_STORE) + .is_none() + { + evt.db().create_object_store(COLLAB_KV_STORE)?; + } + Ok(()) + })); + let db = Arc::new(RwLock::new(db_req.await?)); + Ok(Self { db }) + } + + pub async fn with_write_transaction( + &self, + f: impl FnOnce(&IdbTransactionActionImpl<'_>) -> Result, + ) -> Result { + let db_write_guard = self.db.write().await; + let txn = db_write_guard + .transaction_on_one_with_mode(COLLAB_KV_STORE, IdbTransactionMode::Readwrite)?; + let action_impl = IdbTransactionActionImpl::new(txn)?; + let output = f(&action_impl)?; + action_impl.tx.await.into_result()?; + Ok(output) + } + + pub async fn get_data( + &self, + store: &IdbObjectStore<'_>, + key: K, + ) -> Result, PersistenceError> + where + K: AsRef<[u8]>, + { + let js_key = to_js_key(key.as_ref()); + match store.get(&js_key)?.await? { + None => Err(PersistenceError::RecordNotFound(format!( + "object with given key:{:?} is not found", + js_key + ))), + Some(value) => Ok(Uint8Array::new(&value).to_vec()), + } + } + + pub async fn set_data(&self, key: K, value: V) -> Result<(), PersistenceError> + where + K: AsRef<[u8]>, + V: AsRef<[u8]>, + { + let write_guard = self.db.write().await; + let transaction = + write_guard.transaction_on_one_with_mode(COLLAB_KV_STORE, IdbTransactionMode::Readwrite)?; + let store = store_from_transaction(&transaction)?; + self.set_data_with_store(&store, key, value).await?; + transaction_result_to_result(transaction.await)?; + Ok(()) + } + + pub async fn set_data_with_store( + &self, + store: &IdbObjectStore<'_>, + key: K, + value: V, + ) -> Result<(), PersistenceError> + where + K: AsRef<[u8]>, + V: AsRef<[u8]>, + { + let js_key = to_js_key(key.as_ref()); + let js_value = to_js_key(value.as_ref()); + store.put_key_val(&js_key, &js_value)?.await?; + Ok(()) + } + + pub async fn create_doc( + &self, + uid: i64, + object_id: &str, + encoded_collab: &EncodedCollab, + ) -> Result<(), PersistenceError> { + let doc_id = self.create_doc_id(uid, object_id).await?; + let doc_state_key = make_doc_state_key(doc_id); + let sv_key = make_state_vector_key(doc_id); + + let read_guard = self.db.write().await; + let transaction = + read_guard.transaction_on_one_with_mode(COLLAB_KV_STORE, IdbTransactionMode::Readwrite)?; + let store = store_from_transaction(&transaction)?; + self + .set_data_with_store(&store, doc_state_key, &encoded_collab.doc_state) + .await?; + self + .set_data_with_store(&store, sv_key, &encoded_collab.state_vector) + .await?; + + transaction_result_to_result(transaction.await)?; + Ok(()) + } + + pub async fn load_doc( + &self, + uid: i64, + object_id: &str, + doc: Doc, + ) -> Result<(), PersistenceError> { + let read_guard = self.db.read().await; + let transaction = + read_guard.transaction_on_one_with_mode(COLLAB_KV_STORE, IdbTransactionMode::Readonly)?; + let store = store_from_transaction(&transaction)?; + let doc_id = self + .get_doc_id(&store, uid, object_id) + .await + .ok_or_else(|| { + PersistenceError::RecordNotFound(format!("doc_id for object_id:{} is not found", object_id)) + })?; + + let doc_state_key = make_doc_state_key(doc_id); + let doc_state = self.get_data(&store, doc_state_key).await?; + let updates = fetch_updates(&store, doc_id).await?; + + let mut txn = doc + .try_transact_mut() + .map_err(|err| PersistenceError::Internal(anyhow!("Transact mut fail. error: {:?}", err)))?; + let doc_state_update = Update::decode_v1(doc_state.as_ref()).map_err(PersistenceError::Yrs)?; + txn.try_apply_update(doc_state_update)?; + + for update in updates { + if let Ok(update) = Update::decode_v1(update.as_ref()) { + txn.try_apply_update(update)?; + } + } + + drop(txn); + Ok(()) + } + + pub async fn get_encoded_collab( + &self, + uid: i64, + object_id: &str, + ) -> Result { + let read_guard = self.db.read().await; + let transaction = + read_guard.transaction_on_one_with_mode(COLLAB_KV_STORE, IdbTransactionMode::Readonly)?; + let store = store_from_transaction(&transaction)?; + + let doc_id = self + .get_doc_id(&store, uid, object_id) + .await + .ok_or_else(|| { + PersistenceError::RecordNotFound(format!("doc_id for object_id:{} is not found", object_id)) + })?; + + let doc_state_key = make_doc_state_key(doc_id); + let sv_key = make_state_vector_key(doc_id); + + let doc_stata = self.get_data(&store, doc_state_key).await?; + let sv = self.get_data(&store, sv_key).await?; + + Ok(EncodedCollab::new_v1(sv, doc_stata)) + } + + pub async fn flush_doc( + &self, + uid: i64, + object_id: &str, + encoded: &EncodedCollab, + ) -> Result<(), PersistenceError> { + let read_guard = self.db.write().await; + let transaction = + read_guard.transaction_on_one_with_mode(COLLAB_KV_STORE, IdbTransactionMode::Readwrite)?; + let store = store_from_transaction(&transaction)?; + let doc_id = self + .get_doc_id(&store, uid, object_id) + .await + .ok_or_else(|| { + PersistenceError::RecordNotFound(format!("doc_id for object_id:{} is not found", object_id)) + })?; + + let start = to_js_key(make_doc_start_key(doc_id)); + let end = to_js_key(make_doc_end_key(doc_id)); + let key_range = IdbKeyRange::bound(&start, &end).map_err(|err| { + PersistenceError::Internal(anyhow!("Get last update key fail. error: {:?}", err)) + })?; + + let cursor_request = store + .open_cursor_with_range(&key_range)? + .await? + .ok_or_else(|| { + PersistenceError::Internal(anyhow!("Open cursor fail. error: {:?}", "cursor is none")) + })?; + + // Delete the first key + let _ = cursor_request.delete(); + while cursor_request.continue_cursor()?.await? { + console::log_1(&JsValue::from_str("delete cursor")); + if let Err(err) = cursor_request.delete() { + error!("failed to delete cursor: {:?}", err) + } + } + + let doc_state_key = make_doc_state_key(doc_id); + let sv_key = make_state_vector_key(doc_id); + self + .set_data_with_store(&store, doc_state_key, &encoded.doc_state) + .await?; + self + .set_data_with_store(&store, sv_key, &encoded.state_vector) + .await?; + + transaction_result_to_result(transaction.await)?; + Ok(()) + } + + pub async fn push_update( + &self, + uid: i64, + object_id: &str, + update: &[u8], + ) -> Result<(), PersistenceError> { + let write_guard = self.db.write().await; + let transaction = + write_guard.transaction_on_one_with_mode(COLLAB_KV_STORE, IdbTransactionMode::Readwrite)?; + let store = store_from_transaction(&transaction)?; + let doc_id = self + .get_doc_id(&store, uid, object_id) + .await + .ok_or_else(|| { + PersistenceError::RecordNotFound(format!("doc_id for object_id:{} is not found", object_id)) + })?; + self.put_update(&store, doc_id, update).await?; + transaction_result_to_result(transaction.await)?; + Ok(()) + } + + pub async fn get_all_updates( + &self, + uid: i64, + object_id: &str, + ) -> Result>, PersistenceError> { + let read_guard = self.db.read().await; + let transaction = + read_guard.transaction_on_one_with_mode(COLLAB_KV_STORE, IdbTransactionMode::Readonly)?; + let store = store_from_transaction(&transaction)?; + let doc_id = self + .get_doc_id(&store, uid, object_id) + .await + .ok_or_else(|| { + PersistenceError::RecordNotFound(format!("doc_id for object_id:{} is not found", object_id)) + })?; + + fetch_updates(&store, doc_id).await + } + + async fn put_update( + &self, + store: &IdbObjectStore<'_>, + id: OID, + update: &[u8], + ) -> Result<(), PersistenceError> { + let max_key = JsValue::from(Uint8Array::from( + make_doc_update_key(id, Clock::MAX).as_ref(), + )); + + let key_range = IdbKeyRange::upper_bound(&max_key).map_err(|err| { + PersistenceError::Internal(anyhow!("Get last update key fail. error: {:?}", err)) + })?; + let cursor = store + .open_cursor_with_range_and_direction(&key_range, IdbCursorDirection::Prev)? + .await? + .ok_or_else(|| { + PersistenceError::Internal(anyhow!("Open cursor fail. error: {:?}", "cursor is none")) + })?; + + let clock = cursor + .key() + .map(|key| { + let array_buffer = key.dyn_into::().unwrap(); + let uint8_array = Uint8Array::new(&array_buffer); + let mut vec = vec![0; uint8_array.length() as usize]; + uint8_array.copy_to(&mut vec); + let clock_byte = clock_from_key(&vec); + Clock::from_be_bytes(clock_byte.try_into().unwrap()) + }) + .unwrap_or_else(|| 0); + + let next_clock = clock + 1; + let update_key = make_doc_update_key(id, next_clock); + self.set_data_with_store(store, update_key, update).await?; + Ok(()) + } + + pub async fn get_doc_id( + &self, + store: &IdbObjectStore<'_>, + uid: i64, + object_id: &K, + ) -> Option + where + K: AsRef<[u8]> + ?Sized, + { + let uid_id_bytes = &uid.to_be_bytes(); + let key = make_doc_id_key(uid_id_bytes, object_id.as_ref()); + let value = self.get_data(store, key).await.ok()?; + let mut bytes = [0; DOC_ID_LEN]; + bytes[0..DOC_ID_LEN].copy_from_slice(value.as_ref()); + Some(OID::from_be_bytes(bytes)) + } + + pub async fn create_doc_id(&self, uid: i64, object_id: I) -> Result + where + I: AsRef<[u8]>, + { + let new_id = LOCAL_DOC_ID_GEN.lock().next_id(); + let key = make_doc_id_key(&uid.to_be_bytes(), object_id.as_ref()); + self.set_data(key, new_id.to_be_bytes()).await?; + Ok(new_id) + } +} + +fn to_js_key>(key: K) -> JsValue { + JsValue::from(Uint8Array::from(key.as_ref())) +} + +fn store_from_transaction<'a>( + txn: &'a IdbTransaction<'a>, +) -> Result, PersistenceError> { + txn + .object_store(COLLAB_KV_STORE) + .map_err(PersistenceError::from) +} + +pub struct IdbTransactionActionImpl<'a> { + tx: IdbTransaction<'a>, +} + +impl<'a> IdbTransactionActionImpl<'a> { + fn new(tx: IdbTransaction<'a>) -> Result { + Ok(Self { tx }) + } +} + +fn transaction_result_to_result(result: IdbTransactionResult) -> Result<(), PersistenceError> { + match result { + IdbTransactionResult::Success => Ok(()), + IdbTransactionResult::Error(err) => Err(PersistenceError::from(err)), + IdbTransactionResult::Abort => Err(PersistenceError::Internal(anyhow!("Transaction aborted"))), + } +} + +async fn fetch_updates( + store: &IdbObjectStore<'_>, + doc_id: DocID, +) -> Result>, PersistenceError> { + let start = to_js_key(make_doc_update_key(doc_id, 0).as_ref()); + let end = to_js_key(make_doc_update_key(doc_id, Clock::MAX).as_ref()); + let key_range = IdbKeyRange::bound(&start, &end).map_err(|err| { + PersistenceError::Internal(anyhow!("Get last update key fail. error: {:?}", err)) + })?; + let cursor_request = store.open_cursor_with_range(&key_range)?.await?; + if cursor_request.is_none() { + return Ok(Vec::new()); + } + + let cursor_request = cursor_request.unwrap(); + let mut js_values = Vec::new(); + js_values.push(cursor_request.value()); + while cursor_request.continue_cursor()?.await? { + js_values.push(cursor_request.value()); + } + + Ok( + js_values + .into_iter() + .map(|js_value| js_value.dyn_into::().unwrap().to_vec()) + .collect(), + ) +} diff --git a/collab-plugins/src/local_storage/indexeddb/mod.rs b/collab-plugins/src/local_storage/indexeddb/mod.rs new file mode 100644 index 000000000..819c785c9 --- /dev/null +++ b/collab-plugins/src/local_storage/indexeddb/mod.rs @@ -0,0 +1,2 @@ +pub mod indexeddb_plugin; +pub mod kv_impl; diff --git a/collab-plugins/src/local_storage/kv/db.rs b/collab-plugins/src/local_storage/kv/db.rs index 0722e7b33..d347cea0b 100644 --- a/collab-plugins/src/local_storage/kv/db.rs +++ b/collab-plugins/src/local_storage/kv/db.rs @@ -12,6 +12,23 @@ use crate::local_storage::kv::PersistenceError; use smallvec::SmallVec; use yrs::{TransactionMut, Update}; +pub trait KVTransactionDB: Send + Sync + 'static { + type TransactionAction<'a>; + + fn read_txn<'a, 'b>(&'b self) -> Self::TransactionAction<'a> + where + 'b: 'a; + + fn with_write_txn<'a, 'b, Output>( + &'b self, + f: impl FnOnce(&Self::TransactionAction<'a>) -> Result, + ) -> Result + where + 'b: 'a; + + fn flush(&self) -> Result<(), PersistenceError>; +} + pub trait KVStore<'a> { type Range: Iterator; type Entry: KVEntry; @@ -172,7 +189,7 @@ where Some(OID::from_be_bytes(bytes)) } -pub fn make_doc_id_for_key<'a, S>(store: &S, key: Key<20>) -> Result +pub fn insert_doc_id_for_key<'a, S>(store: &S, key: Key<20>) -> Result where S: KVStore<'a>, PersistenceError: From<>::Error>, diff --git a/collab-plugins/src/local_storage/kv/doc.rs b/collab-plugins/src/local_storage/kv/doc.rs index 49754a146..700b665ae 100644 --- a/collab-plugins/src/local_storage/kv/doc.rs +++ b/collab-plugins/src/local_storage/kv/doc.rs @@ -72,8 +72,8 @@ where Ok(()) } - fn is_exist + ?Sized + Debug>(&self, collab_id: i64, object_id: &K) -> bool { - get_doc_id(collab_id, self, object_id).is_some() + fn is_exist + ?Sized + Debug>(&self, uid: i64, object_id: &K) -> bool { + get_doc_id(uid, self, object_id).is_some() } /// Load the document from the database and apply the updates to the transaction. @@ -130,7 +130,10 @@ where Ok(update_count) } else { tracing::trace!("[Client] => {:?} not exist", object_id); - Err(PersistenceError::DocumentNotExist) + Err(PersistenceError::RecordNotFound(format!( + "doc with given object id: {:?} is not found", + object_id + ))) } } @@ -193,7 +196,10 @@ where "🔴Insert update failed. Can't find the doc for {:?}", object_id ); - Err(PersistenceError::DocumentNotExist) + Err(PersistenceError::RecordNotFound(format!( + "doc with given object id: {:?} is not found", + object_id + ))) }, Some(doc_id) => insert_doc_update(self, doc_id, object_id, update.to_vec()), } @@ -323,7 +329,10 @@ where } Ok(updates) } else { - Err(PersistenceError::DocumentNotExist) + Err(PersistenceError::RecordNotFound(format!( + "The document with given object id: {:?} is not found", + object_id.as_ref(), + ))) } } @@ -373,18 +382,18 @@ where Ok(did) } else { let key = make_doc_id_key(&uid.to_be_bytes(), object_id.as_ref()); - let new_did = make_doc_id_for_key(store, key)?; + let new_did = insert_doc_id_for_key(store, key)?; Ok(new_did) } } -fn get_doc_id<'a, K, S>(collab_id: i64, store: &S, object_id: &K) -> Option +fn get_doc_id<'a, K, S>(uid: i64, store: &S, object_id: &K) -> Option where S: KVStore<'a>, K: AsRef<[u8]> + ?Sized, { - let collab_id_bytes = &collab_id.to_be_bytes(); - let key = make_doc_id_key(collab_id_bytes, object_id.as_ref()); + let uid_id_bytes = &uid.to_be_bytes(); + let key = make_doc_id_key(uid_id_bytes, object_id.as_ref()); get_id_for_key(store, key) } diff --git a/collab-plugins/src/local_storage/kv/error.rs b/collab-plugins/src/local_storage/kv/error.rs index 873f87521..708c6c1e9 100644 --- a/collab-plugins/src/local_storage/kv/error.rs +++ b/collab-plugins/src/local_storage/kv/error.rs @@ -21,8 +21,8 @@ pub enum PersistenceError { #[error(transparent)] Bincode(#[from] bincode::Error), - #[error("The document is not exist")] - DocumentNotExist, + #[error("{0}")] + RecordNotFound(String), #[error("The document already exist")] DocumentAlreadyExist, @@ -42,10 +42,26 @@ pub enum PersistenceError { #[error("Can't find the latest update key")] LatestUpdateKeyNotExist, + #[error(transparent)] + Collab(#[from] collab::error::CollabError), + #[error(transparent)] Internal(#[from] anyhow::Error), } +impl PersistenceError { + pub fn is_record_not_found(&self) -> bool { + matches!(self, PersistenceError::RecordNotFound(_)) + } +} + +#[cfg(target_arch = "wasm32")] +impl From for PersistenceError { + fn from(value: indexed_db_futures::web_sys::DomException) -> Self { + PersistenceError::Internal(anyhow::anyhow!("DOMException: {:?}", value)) + } +} + #[cfg(not(target_arch = "wasm32"))] impl From for PersistenceError { fn from(value: rocksdb::Error) -> Self { diff --git a/collab-plugins/src/local_storage/kv/mod.rs b/collab-plugins/src/local_storage/kv/mod.rs index 59e3a1f5d..a4aecf66b 100644 --- a/collab-plugins/src/local_storage/kv/mod.rs +++ b/collab-plugins/src/local_storage/kv/mod.rs @@ -6,6 +6,6 @@ mod db; pub mod doc; pub mod error; pub mod keys; -mod oid; +pub mod oid; mod range; pub mod snapshot; diff --git a/collab-plugins/src/local_storage/kv/oid.rs b/collab-plugins/src/local_storage/kv/oid.rs index b39a2d5d4..d96ea3b15 100644 --- a/collab-plugins/src/local_storage/kv/oid.rs +++ b/collab-plugins/src/local_storage/kv/oid.rs @@ -1,8 +1,8 @@ #![allow(clippy::upper_case_acronyms)] +use crate::{if_native, if_wasm}; use lazy_static::lazy_static; use parking_lot::Mutex; -use std::time::SystemTime; const EPOCH: u64 = 1637806706000; const NODE_BITS: u64 = 8; @@ -25,6 +25,12 @@ pub struct DocIDGen { last_timestamp: u64, } +impl Default for DocIDGen { + fn default() -> Self { + Self::new() + } +} + impl DocIDGen { #[allow(dead_code)] pub fn new() -> DocIDGen { @@ -64,11 +70,19 @@ impl DocIDGen { } } - fn timestamp(&self) -> u64 { - SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) + if_wasm! { + fn timestamp(&self) -> u64 { + js_sys::Date::now() as u64 + } + } + + if_native! { + fn timestamp(&self) -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::SystemTime::UNIX_EPOCH) .expect("Clock moved backwards!") .as_millis() as u64 + } } } diff --git a/collab-plugins/src/local_storage/kv/snapshot.rs b/collab-plugins/src/local_storage/kv/snapshot.rs index 28f88f59c..156fae0bc 100644 --- a/collab-plugins/src/local_storage/kv/snapshot.rs +++ b/collab-plugins/src/local_storage/kv/snapshot.rs @@ -143,7 +143,7 @@ where Ok(snapshot_id) } else { let key = make_snapshot_id_key(&uid.to_be_bytes(), object_id.as_ref()); - let new_snapshot_id = make_doc_id_for_key(self, key)?; + let new_snapshot_id = insert_doc_id_for_key(self, key)?; Ok(new_snapshot_id) } } diff --git a/collab-plugins/src/local_storage/mod.rs b/collab-plugins/src/local_storage/mod.rs index 24ab602d8..7aa724de4 100644 --- a/collab-plugins/src/local_storage/mod.rs +++ b/collab-plugins/src/local_storage/mod.rs @@ -3,37 +3,9 @@ pub mod kv; #[cfg(not(target_arch = "wasm32"))] pub mod rocksdb; -#[derive(Clone)] -pub struct CollabPersistenceConfig { - /// Enable snapshot. Default is [false]. - pub enable_snapshot: bool, - /// Generate a snapshot every N updates - /// Default is 100. The value must be greater than 0. - pub snapshot_per_update: u32, -} +#[cfg(target_arch = "wasm32")] +pub mod indexeddb; -impl CollabPersistenceConfig { - pub fn new() -> Self { - Self::default() - } +mod storage_config; - pub fn enable_snapshot(mut self, enable_snapshot: bool) -> Self { - self.enable_snapshot = enable_snapshot; - self - } - - pub fn snapshot_per_update(mut self, snapshot_per_update: u32) -> Self { - debug_assert!(snapshot_per_update > 0); - self.snapshot_per_update = snapshot_per_update; - self - } -} - -impl Default for CollabPersistenceConfig { - fn default() -> Self { - Self { - enable_snapshot: true, - snapshot_per_update: 100, - } - } -} +pub use storage_config::*; diff --git a/collab-plugins/src/local_storage/rocksdb/kv_impl.rs b/collab-plugins/src/local_storage/rocksdb/kv_impl.rs index 5b0a8a8d5..5d2e8eec8 100644 --- a/collab-plugins/src/local_storage/rocksdb/kv_impl.rs +++ b/collab-plugins/src/local_storage/rocksdb/kv_impl.rs @@ -3,8 +3,7 @@ use std::ops::RangeBounds; use std::path::Path; use std::sync::Arc; -use crate::local_storage::kv::doc::CollabKVAction; -use crate::local_storage::kv::{KVEntry, KVStore, PersistenceError}; +use crate::local_storage::kv::{KVEntry, KVStore, KVTransactionDB, PersistenceError}; use rocksdb::Direction::Forward; use rocksdb::{ DBIteratorWithThreadMode, Direction, ErrorKind, IteratorMode, Options, ReadOptions, @@ -13,14 +12,15 @@ use rocksdb::{ }; #[derive(Clone)] -pub struct RocksStore { +pub struct KVTransactionDBRocksdbImpl { db: Arc, } -impl RocksStore { +impl KVTransactionDBRocksdbImpl { /// Open a new RocksDB database at the given path. /// If the database is corrupted, try to repair it. If it cannot be repaired, return an error. - pub fn open_opt(path: impl AsRef, auto_repair: bool) -> Result { + pub fn open(path: impl AsRef) -> Result { + let auto_repair = false; let txn_db_opts = TransactionDBOptions::default(); let mut db_opts = Options::default(); // This option sets the upper limit for the total number of background jobs (both flushes and compactions) @@ -109,19 +109,15 @@ impl RocksStore { Ok(Self { db: Arc::new(db) }) } +} - /// Open a new RocksDB database at the given path. - /// If the database is corrupted, try to repair it. If it cannot be repaired, return an error. - pub fn open(path: impl AsRef) -> Result { - Self::open_opt(path, false) - } +impl KVTransactionDB for KVTransactionDBRocksdbImpl { + type TransactionAction<'a> = RocksdbKVStoreImpl<'a, TransactionDB>; - pub fn flush(&self) -> Result<(), PersistenceError> { - Ok(()) - } - - /// Return a read transaction that accesses the database exclusively. - pub fn read_txn(&self) -> impl CollabKVAction<'_, Error = PersistenceError> { + fn read_txn<'a, 'b>(&'b self) -> Self::TransactionAction<'a> + where + 'b: 'a, + { let mut txn_options = TransactionOptions::default(); // Use snapshot to provides a consistent view of the data. This snapshot can then be used // to perform read operations, and the returned data will be consistent with the database @@ -131,33 +127,38 @@ impl RocksStore { let txn = self .db .transaction_opt(&WriteOptions::default(), &txn_options); - RocksKVStoreImpl::new(txn) + RocksdbKVStoreImpl::new(txn) } - /// Create a write transaction that accesses the database exclusively. - /// The transaction will be committed when the closure [F] returns. - pub fn with_write_txn(&self, f: F) -> Result + fn with_write_txn<'a, 'b, Output>( + &'b self, + f: impl FnOnce(&Self::TransactionAction<'a>) -> Result, + ) -> Result where - F: FnOnce(&RocksKVStoreImpl<'_, TransactionDB>) -> Result, + 'b: 'a, { let txn_options = TransactionOptions::default(); let txn = self .db .transaction_opt(&WriteOptions::default(), &txn_options); - let store = RocksKVStoreImpl::new(txn); + let store = RocksdbKVStoreImpl::new(txn); let result = f(&store)?; store.0.commit()?; Ok(result) } + + fn flush(&self) -> Result<(), PersistenceError> { + Ok(()) + } } -/// Implementation of [KVStore] for [RocksStore]. This is a wrapper around [Transaction]. +/// Implementation of [KVStore] for [KVTransactionDBRocksdbImpl]. This is a wrapper around [Transaction]. // pub struct RocksKVStoreImpl<'a, DB: Send + Sync>(Transaction<'a, DB>); -pub struct RocksKVStoreImpl<'a, DB: Send>(Transaction<'a, DB>); +pub struct RocksdbKVStoreImpl<'a, DB: Send>(Transaction<'a, DB>); -unsafe impl<'a, DB: Send> Send for RocksKVStoreImpl<'a, DB> {} +unsafe impl<'a, DB: Send> Send for RocksdbKVStoreImpl<'a, DB> {} -impl<'a, DB: Send + Sync> RocksKVStoreImpl<'a, DB> { +impl<'a, DB: Send + Sync> RocksdbKVStoreImpl<'a, DB> { pub fn new(txn: Transaction<'a, DB>) -> Self { Self(txn) } @@ -168,10 +169,10 @@ impl<'a, DB: Send + Sync> RocksKVStoreImpl<'a, DB> { } } -impl<'a, DB: Send + Sync> KVStore<'a> for RocksKVStoreImpl<'a, DB> { - type Range = RocksDBRange<'a, DB>; - type Entry = RocksDBEntry; - type Value = RocksDBVec; +impl<'a, DB: Send + Sync> KVStore<'a> for RocksdbKVStoreImpl<'a, DB> { + type Range = RocksdbRange<'a, DB>; + type Entry = RocksdbEntry; + type Value = Vec; type Error = PersistenceError; fn get>(&self, key: K) -> Result, Self::Error> { @@ -235,7 +236,7 @@ impl<'a, DB: Send + Sync> KVStore<'a> for RocksKVStoreImpl<'a, DB> { }; let iterator_mode = IteratorMode::From(from, Forward); let iter = self.0.iterator_opt(iterator_mode, opt); - Ok(RocksDBRange { + Ok(RocksdbRange { // Safe to transmute because the lifetime of the iterator is the same as the lifetime of the // transaction. inner: unsafe { std::mem::transmute(iter) }, @@ -248,29 +249,27 @@ impl<'a, DB: Send + Sync> KVStore<'a> for RocksKVStoreImpl<'a, DB> { let mut raw = self.0.raw_iterator_opt(opt); raw.seek_for_prev(key); if let Some((key, value)) = raw.item() { - Ok(Some(RocksDBEntry::new(key.to_vec(), value.to_vec()))) + Ok(Some(RocksdbEntry::new(key.to_vec(), value.to_vec()))) } else { Ok(None) } } } -impl<'a, DB: Send + Sync> From> for RocksKVStoreImpl<'a, DB> { +impl<'a, DB: Send + Sync> From> for RocksdbKVStoreImpl<'a, DB> { #[inline(always)] fn from(txn: Transaction<'a, DB>) -> Self { - RocksKVStoreImpl::new(txn) + RocksdbKVStoreImpl::new(txn) } } -pub type RocksDBVec = Vec; - -pub struct RocksDBRange<'a, DB> { +pub struct RocksdbRange<'a, DB> { inner: DBIteratorWithThreadMode<'a, Transaction<'a, DB>>, to: Vec, } -impl<'a, DB: Send + Sync> Iterator for RocksDBRange<'a, DB> { - type Item = RocksDBEntry; +impl<'a, DB: Send + Sync> Iterator for RocksdbRange<'a, DB> { + type Item = RocksdbEntry; fn next(&mut self) -> Option { let n = self.inner.next()?; @@ -278,7 +277,7 @@ impl<'a, DB: Send + Sync> Iterator for RocksDBRange<'a, DB> { if key.as_ref() >= self.to.as_slice() { None } else { - Some(RocksDBEntry::new(key.to_vec(), value.to_vec())) + Some(RocksdbEntry::new(key.to_vec(), value.to_vec())) } } else { None @@ -286,18 +285,18 @@ impl<'a, DB: Send + Sync> Iterator for RocksDBRange<'a, DB> { } } -pub struct RocksDBEntry { +pub struct RocksdbEntry { key: Vec, value: Vec, } -impl RocksDBEntry { +impl RocksdbEntry { pub fn new(key: Vec, value: Vec) -> Self { Self { key, value } } } -impl KVEntry for RocksDBEntry { +impl KVEntry for RocksdbEntry { fn key(&self) -> &[u8] { self.key.as_ref() } diff --git a/collab-plugins/src/local_storage/rocksdb/rocksdb_plugin.rs b/collab-plugins/src/local_storage/rocksdb/rocksdb_plugin.rs index 6450a9be7..fc2405a3c 100644 --- a/collab-plugins/src/local_storage/rocksdb/rocksdb_plugin.rs +++ b/collab-plugins/src/local_storage/rocksdb/rocksdb_plugin.rs @@ -16,6 +16,7 @@ use yrs::{Doc, ReadTxn, StateVector, Transact, TransactionMut}; use crate::local_storage::kv::doc::CollabKVAction; use crate::local_storage::kv::snapshot::SnapshotPersistence; +use crate::local_storage::kv::KVTransactionDB; use crate::local_storage::rocksdb::snapshot_plugin::CollabSnapshot; use crate::local_storage::CollabPersistenceConfig; diff --git a/collab-plugins/src/local_storage/rocksdb/snapshot_plugin.rs b/collab-plugins/src/local_storage/rocksdb/snapshot_plugin.rs index 8443ec4e8..22511ad9e 100644 --- a/collab-plugins/src/local_storage/rocksdb/snapshot_plugin.rs +++ b/collab-plugins/src/local_storage/rocksdb/snapshot_plugin.rs @@ -2,7 +2,7 @@ use std::sync::{Arc, Weak}; use crate::local_storage::kv::doc::CollabKVAction; use crate::local_storage::kv::snapshot::SnapshotPersistence; -use crate::local_storage::kv::PersistenceError; +use crate::local_storage::kv::{KVTransactionDB, PersistenceError}; use crate::CollabKVDB; use collab::preclude::Collab; use collab_entity::CollabType; diff --git a/collab-plugins/src/local_storage/storage_config.rs b/collab-plugins/src/local_storage/storage_config.rs new file mode 100644 index 000000000..478a1e9e0 --- /dev/null +++ b/collab-plugins/src/local_storage/storage_config.rs @@ -0,0 +1,34 @@ +#[derive(Clone)] +pub struct CollabPersistenceConfig { + /// Enable snapshot. Default is [false]. + pub enable_snapshot: bool, + /// Generate a snapshot every N updates + /// Default is 100. The value must be greater than 0. + pub snapshot_per_update: u32, +} + +impl CollabPersistenceConfig { + pub fn new() -> Self { + Self::default() + } + + pub fn enable_snapshot(mut self, enable_snapshot: bool) -> Self { + self.enable_snapshot = enable_snapshot; + self + } + + pub fn snapshot_per_update(mut self, snapshot_per_update: u32) -> Self { + debug_assert!(snapshot_per_update > 0); + self.snapshot_per_update = snapshot_per_update; + self + } +} + +impl Default for CollabPersistenceConfig { + fn default() -> Self { + Self { + enable_snapshot: true, + snapshot_per_update: 100, + } + } +} diff --git a/collab-plugins/tests/disk/insert_test.rs b/collab-plugins/tests/disk/insert_test.rs index f1d32560e..cbc275cc2 100644 --- a/collab-plugins/tests/disk/insert_test.rs +++ b/collab-plugins/tests/disk/insert_test.rs @@ -4,6 +4,7 @@ use assert_json_diff::assert_json_eq; use collab::preclude::CollabBuilder; use collab_entity::CollabType; use collab_plugins::local_storage::kv::doc::CollabKVAction; +use collab_plugins::local_storage::kv::KVTransactionDB; use collab_plugins::local_storage::CollabPersistenceConfig; use std::sync::Arc; diff --git a/collab-plugins/tests/disk/range_test.rs b/collab-plugins/tests/disk/range_test.rs index 1dc7ec6d4..e6e5ed772 100644 --- a/collab-plugins/tests/disk/range_test.rs +++ b/collab-plugins/tests/disk/range_test.rs @@ -4,7 +4,7 @@ use std::thread; use crate::disk::util::rocks_db; use collab_plugins::local_storage::kv::keys::{clock_from_key, make_doc_update_key, Clock}; -use collab_plugins::local_storage::kv::{KVEntry, KVStore}; +use collab_plugins::local_storage::kv::{KVEntry, KVStore, KVTransactionDB}; use smallvec::SmallVec; #[tokio::test] @@ -59,7 +59,7 @@ async fn rocks_id_test() { let txn = rocks_db.read_txn(); let value = txn.get([0, 0, 0, 0, 0, 0, 0, 2]).unwrap().unwrap(); - assert_eq!(value.as_ref(), &[0, 1, 3]); + assert_eq!(&value, &[0, 1, 3]); } #[tokio::test] diff --git a/collab-plugins/tests/disk/restore_test.rs b/collab-plugins/tests/disk/restore_test.rs index e5d8670b6..d52a4451f 100644 --- a/collab-plugins/tests/disk/restore_test.rs +++ b/collab-plugins/tests/disk/restore_test.rs @@ -2,6 +2,7 @@ use std::thread; use crate::disk::util::rocks_db; use collab_plugins::local_storage::kv::doc::CollabKVAction; +use collab_plugins::local_storage::kv::KVTransactionDB; use collab_plugins::CollabKVDB; use yrs::{Doc, GetString, Text, Transact}; @@ -33,7 +34,7 @@ async fn single_thread_test() { } drop(db); - let db = CollabKVDB::open_opt(path, false).unwrap(); + let db = CollabKVDB::open(path).unwrap(); for i in 0..100 { let oid = format!("doc_{}", i); let doc = Doc::new(); @@ -80,7 +81,7 @@ async fn rocks_multiple_thread_test() { } drop(db); - let db = CollabKVDB::open_opt(path, false).unwrap(); + let db = CollabKVDB::open(path).unwrap(); for i in 0..100 { let oid = format!("doc_{}", i); let doc = Doc::new(); diff --git a/collab-plugins/tests/disk/script.rs b/collab-plugins/tests/disk/script.rs index 3706e9a22..993264a97 100644 --- a/collab-plugins/tests/disk/script.rs +++ b/collab-plugins/tests/disk/script.rs @@ -11,6 +11,7 @@ use collab_plugins::local_storage::kv::doc::CollabKVAction; use collab_plugins::local_storage::rocksdb::rocksdb_plugin::RocksdbDiskPlugin; use collab_entity::CollabType; +use collab_plugins::local_storage::kv::KVTransactionDB; use collab_plugins::CollabKVDB; use tempfile::TempDir; @@ -70,7 +71,7 @@ impl CollabPersistenceTest { let tempdir = TempDir::new().unwrap(); let db_path = tempdir.into_path(); let uid = 1; - let db = Arc::new(CollabKVDB::open_opt(db_path.clone(), false).unwrap()); + let db = Arc::new(CollabKVDB::open(db_path.clone()).unwrap()); let cleaner = Cleaner::new(db_path); Self { uid, diff --git a/collab-plugins/tests/disk/util.rs b/collab-plugins/tests/disk/util.rs index a98c407db..b913252f7 100644 --- a/collab-plugins/tests/disk/util.rs +++ b/collab-plugins/tests/disk/util.rs @@ -7,5 +7,5 @@ pub fn rocks_db() -> (PathBuf, CollabKVDB) { let tempdir = TempDir::new().unwrap(); let path = tempdir.into_path(); let cloned_path = path.clone(); - (path, CollabKVDB::open_opt(cloned_path, false).unwrap()) + (path, CollabKVDB::open(cloned_path).unwrap()) } diff --git a/collab-plugins/tests/main.rs b/collab-plugins/tests/main.rs index 15fe9d24f..525c0ad0b 100644 --- a/collab-plugins/tests/main.rs +++ b/collab-plugins/tests/main.rs @@ -1,10 +1,12 @@ -use tracing_subscriber::util::SubscriberInitExt; - #[cfg(not(target_arch = "wasm32"))] mod disk; +#[cfg(target_arch = "wasm32")] +mod web; + #[cfg(not(target_arch = "wasm32"))] pub fn setup_log() { + use tracing_subscriber::util::SubscriberInitExt; static START: std::sync::Once = std::sync::Once::new(); START.call_once(|| { let level = "trace"; diff --git a/collab-plugins/tests/web/edit_collab_test.rs b/collab-plugins/tests/web/edit_collab_test.rs new file mode 100644 index 000000000..c19966a16 --- /dev/null +++ b/collab-plugins/tests/web/edit_collab_test.rs @@ -0,0 +1,106 @@ +use assert_json_diff::assert_json_eq; +use collab::core::collab::MutexCollab; +use collab::preclude::CollabBuilder; +use collab_entity::CollabType; +use collab_plugins::local_storage::indexeddb::indexeddb_plugin::IndexeddbDiskPlugin; +use collab_plugins::local_storage::indexeddb::kv_impl::CollabIndexeddb; +use js_sys::Promise; +use serde_json::json; +use std::sync::{Arc, Once}; +use uuid::Uuid; +use wasm_bindgen::prelude::*; +use wasm_bindgen_futures::JsFuture; +use wasm_bindgen_test::wasm_bindgen_test; +use web_sys::window; + +#[wasm_bindgen_test] +async fn edit_collab_with_indexeddb_test() { + setup_log(); + let object_id = Uuid::new_v4().to_string(); + let uid: i64 = 1; + let db = Arc::new(CollabIndexeddb::new().await.unwrap()); + let collab = create_collab(uid, object_id.clone(), &db).await; + collab.lock().insert("message", "hello world"); + let json_1 = collab.to_json_value(); + drop(collab); + + // sleep 2 secs to wait for the disk plugin to flush the data + sleep(2000).await; + let collab_from_disk = create_collab(uid, object_id.clone(), &db).await; + let json_2 = collab_from_disk.to_json_value(); + assert_json_eq!( + json_2, + json!({ + "message": "hello world" + }) + ); + assert_json_eq!(json_1, json_2); +} + +#[wasm_bindgen_test] +async fn flush_collab_with_indexeddb_test() { + setup_log(); + let object_id = Uuid::new_v4().to_string(); + let uid: i64 = 1; + let db = Arc::new(CollabIndexeddb::new().await.unwrap()); + let collab = create_collab(uid, object_id.clone(), &db).await; + collab.lock().insert("1", "a"); + sleep(100).await; + collab.lock().insert("2", "b"); + sleep(100).await; + collab.lock().insert("3", "c"); + sleep(100).await; + let json_1 = collab.to_json_value(); + collab.lock().flush(); + + // sleep 2 secs to wait for the disk plugin to flush the data + sleep(2000).await; + + // after flush, all the updates will be removed. Only the final doc state will be saved to disk + let updates = db.get_all_updates(uid, &object_id).await.unwrap(); + assert_eq!(updates.len(), 0); + + let collab_from_disk = create_collab(uid, object_id.clone(), &db).await; + let json_2 = collab_from_disk.to_json_value(); + assert_json_eq!(json_1, json_2); +} + +pub fn setup_log() { + static START: Once = Once::new(); + START.call_once(|| { + tracing_wasm::set_as_global_default(); + }); +} + +pub async fn create_collab( + uid: i64, + doc_id: String, + db: &Arc, +) -> Arc { + let collab = Arc::new( + CollabBuilder::new(uid, &doc_id) + .with_device_id("1") + .build() + .unwrap(), + ); + let disk_plugin = IndexeddbDiskPlugin::new(uid, doc_id, CollabType::Document, Arc::downgrade(db)); + collab.lock().add_plugin(Arc::new(disk_plugin)); + collab.lock().initialize().await; + sleep(1000).await; + collab +} + +async fn sleep(ms: i32) { + let promise = Promise::new(&mut |resolve, _| { + let closure = Closure::once_into_js(move || { + resolve.call0(&JsValue::NULL).unwrap(); + }); + + window() + .unwrap() + .set_timeout_with_callback_and_timeout_and_arguments_0(closure.as_ref().unchecked_ref(), ms) + .unwrap(); + }); + + JsFuture::from(promise).await.unwrap(); +} diff --git a/collab-plugins/tests/web/indexeddb_test.rs b/collab-plugins/tests/web/indexeddb_test.rs new file mode 100644 index 000000000..0e651f7c6 --- /dev/null +++ b/collab-plugins/tests/web/indexeddb_test.rs @@ -0,0 +1,97 @@ +use collab::core::collab_plugin::EncodedCollab; +use collab_plugins::local_storage::indexeddb::kv_impl::CollabIndexeddb; +use uuid::Uuid; +use wasm_bindgen_test::*; +use yrs::Doc; + +#[wasm_bindgen_test] +async fn indexeddb_put_and_get_encoded_collab_test() { + let db = CollabIndexeddb::new().await.unwrap(); + let object_id = Uuid::new_v4().to_string(); + let uid: i64 = 1; + let encoded_collab = EncodedCollab { + state_vector: vec![1, 2, 3].into(), + doc_state: vec![4, 5, 6].into(), + version: collab::core::collab_plugin::EncoderVersion::V1, + }; + + db.create_doc(uid, &object_id, &encoded_collab) + .await + .unwrap(); + let encoded_collab_from_db = db.get_encoded_collab(uid, &object_id).await.unwrap(); + + assert_eq!( + encoded_collab.state_vector, + encoded_collab_from_db.state_vector + ); + assert_eq!(encoded_collab.doc_state, encoded_collab_from_db.doc_state); +} + +#[wasm_bindgen_test] +async fn indexeddb_get_non_exist_encoded_collab_test() { + let db = CollabIndexeddb::new().await.unwrap(); + let object_id = Uuid::new_v4().to_string(); + let doc = Doc::new(); + let uid: i64 = 1; + let error = db.load_doc(uid, &object_id, doc).await.unwrap_err(); + assert!(error.is_record_not_found()); +} + +#[wasm_bindgen_test] +async fn indexeddb_push_update_test() { + let db = CollabIndexeddb::new().await.unwrap(); + let object_id = Uuid::new_v4().to_string(); + let uid: i64 = 1; + + db.create_doc_id(uid, &object_id).await.unwrap(); + let update_1 = vec![1, 2, 3]; + db.push_update(uid, &object_id, &update_1).await.unwrap(); + + let update_2 = vec![4, 5, 6]; + db.push_update(uid, &object_id, &update_2).await.unwrap(); + + let update_3 = vec![7, 8, 9]; + db.push_update(uid, &object_id, &update_3).await.unwrap(); + + let update_4 = vec![10, 11, 12]; + db.push_update(uid, &object_id, &update_4).await.unwrap(); + + let updates = db.get_all_updates(uid, &object_id).await.unwrap(); + assert_eq!(updates.len(), 4); + assert_eq!(updates[0], update_1); + assert_eq!(updates[1], update_2); + assert_eq!(updates[2], update_3); + assert_eq!(updates[3], update_4); +} + +#[wasm_bindgen_test] +async fn indexeddb_flush_doc_test() { + let db = CollabIndexeddb::new().await.unwrap(); + let object_id = Uuid::new_v4().to_string(); + let uid: i64 = 1; + + db.create_doc_id(uid, &object_id).await.unwrap(); + let update_1 = vec![1, 2, 3]; + db.push_update(uid, &object_id, &update_1).await.unwrap(); + + let update_2 = vec![4, 5, 6]; + db.push_update(uid, &object_id, &update_2).await.unwrap(); + + let update_3 = vec![7, 8, 9]; + db.push_update(uid, &object_id, &update_3).await.unwrap(); + + let update_4 = vec![10, 11, 12]; + db.push_update(uid, &object_id, &update_4).await.unwrap(); + + let encoded_collab = EncodedCollab { + state_vector: vec![1, 2, 3].into(), + doc_state: vec![4, 5, 6].into(), + version: collab::core::collab_plugin::EncoderVersion::V1, + }; + db.flush_doc(uid, &object_id, &encoded_collab) + .await + .unwrap(); + + let updates = db.get_all_updates(uid, &object_id).await.unwrap(); + assert_eq!(updates.len(), 0); +} diff --git a/collab-plugins/tests/web/mod.rs b/collab-plugins/tests/web/mod.rs new file mode 100644 index 000000000..821bd1efd --- /dev/null +++ b/collab-plugins/tests/web/mod.rs @@ -0,0 +1,5 @@ +use wasm_bindgen_test::wasm_bindgen_test_configure; +wasm_bindgen_test_configure!(run_in_browser); + +mod edit_collab_test; +mod indexeddb_test; diff --git a/collab-plugins/tests/web/setup_tests.js b/collab-plugins/tests/web/setup_tests.js new file mode 100644 index 000000000..5ea989ae2 --- /dev/null +++ b/collab-plugins/tests/web/setup_tests.js @@ -0,0 +1,6 @@ +function get_current_timestamp() { + return Date.now(); +} + +// Expose the function to the global scope so it's accessible to the WASM module +global.get_current_timestamp = get_current_timestamp; diff --git a/collab-plugins/tests/web/test.md b/collab-plugins/tests/web/test.md new file mode 100644 index 000000000..76a158426 --- /dev/null +++ b/collab-plugins/tests/web/test.md @@ -0,0 +1,17 @@ + +## Run clippy for web + +```shell +cargo clippy --target=wasm32-unknown-unknown --fix --allow-dirty --features="wasm_build" +``` + +## Run tests in Chrome +```shell +wasm-pack test --chrome --features="wasm_build" +``` + +## Build for web + +```shell +wasm-pack build --features="wasm_build" +``` \ No newline at end of file diff --git a/collab/Cargo.toml b/collab/Cargo.toml index 89a9b8e97..d3c31fd2f 100644 --- a/collab/Cargo.toml +++ b/collab/Cargo.toml @@ -22,6 +22,9 @@ async-trait.workspace = true bincode = "1.3.3" serde_repr = "0.1" +[target.'cfg(target_arch = "wasm32")'.dependencies] +web-sys = { version = "0.3.67"} + [dev-dependencies] tokio = { version = "1.26", features = ["rt"] } tempfile = "3.8.0" diff --git a/collab/src/core/transaction.rs b/collab/src/core/transaction.rs index c03aab815..2918a6d90 100644 --- a/collab/src/core/transaction.rs +++ b/collab/src/core/transaction.rs @@ -1,5 +1,5 @@ use std::thread::sleep; -use std::time::{Duration, Instant}; +use std::time::Duration; use crate::core::collab_plugin::EncodedCollab; use yrs::updates::encoder::Encode; @@ -15,7 +15,7 @@ use crate::error::CollabError; pub struct TransactionRetry<'a> { timeout: Duration, doc: &'a Doc, - start: Instant, + timer: Timer, retry_interval: Duration, } @@ -25,12 +25,12 @@ impl<'a> TransactionRetry<'a> { timeout: Duration::from_secs(2), retry_interval: Duration::from_millis(50), doc, - start: Instant::now(), + timer: Timer::start(), } } pub fn get_read_txn(&mut self) -> Transaction<'a> { - while self.start.elapsed() < self.timeout { + while self.timer.elapsed() < self.timeout { match self.doc.try_transact() { Ok(txn) => { return txn; @@ -45,7 +45,7 @@ impl<'a> TransactionRetry<'a> { } pub fn try_get_write_txn(&mut self) -> Result, CollabError> { - while self.start.elapsed() < self.timeout { + while self.timer.elapsed() < self.timeout { match self.doc.try_transact_mut() { Ok(txn) => { return Ok(txn); @@ -60,7 +60,7 @@ impl<'a> TransactionRetry<'a> { } pub fn get_write_txn_with(&mut self, origin: CollabOrigin) -> TransactionMut<'a> { - while self.start.elapsed() < self.timeout { + while self.timer.elapsed() < self.timeout { match self.doc.try_transact_mut_with(origin.clone()) { Ok(txn) => { return txn; @@ -78,7 +78,7 @@ impl<'a> TransactionRetry<'a> { &mut self, origin: CollabOrigin, ) -> Result, CollabError> { - while self.start.elapsed() < self.timeout { + while self.timer.elapsed() < self.timeout { match self.doc.try_transact_mut_with(origin.clone()) { Ok(txn) => { return Ok(txn); @@ -122,3 +122,37 @@ impl DocTransactionExtension for Doc { self.transact_mut() } } + +if_native! { + struct Timer { + start: std::time::Instant, + } + + impl Timer { + fn start() -> Self { + Self { start: std::time::Instant::now() } + } + + fn elapsed(&self) -> Duration { + self.start.elapsed() + } + } +} + +if_wasm! { + struct Timer { + start: f64, + } + + impl Timer { + fn start() -> Self { + Self { start: web_sys::js_sys::Date::now() } + } + + fn elapsed(&self) -> Duration { + let now = web_sys::js_sys::Date::now(); + let elapsed_ms = now - self.start; + Duration::from_millis(elapsed_ms as u64) + } + } +} diff --git a/collab/src/lib.rs b/collab/src/lib.rs index 84d8fa191..2a0c41cd6 100644 --- a/collab/src/lib.rs +++ b/collab/src/lib.rs @@ -1,3 +1,19 @@ +#[macro_export] +macro_rules! if_native { + ($($item:item)*) => {$( + #[cfg(not(target_arch = "wasm32"))] + $item + )*} +} + +#[macro_export] +macro_rules! if_wasm { + ($($item:item)*) => {$( + #[cfg(target_arch = "wasm32")] + $item + )*} +} + pub mod core; pub mod error; pub mod sync_protocol; From 3eef93f35f606edef2541888cb91cd6686d77225 Mon Sep 17 00:00:00 2001 From: "Nathan.fooo" <86001920+appflowy@users.noreply.github.com> Date: Mon, 22 Jan 2024 01:01:25 +0800 Subject: [PATCH 13/13] chore: Wasm deps (#151) * chore: fix deps * chore: fix deps * chore: fix deps * chore: fix deps * chore: fix deps * chore: fix deps * chore: fix deps * chore: fix deps * chore: fix deps * chore: fix deps * chore: fix deps * chore: fix deps * chore: fix deps * chore: fix deps * chore: fix deps --- Cargo.lock | 1 + collab-database/Cargo.toml | 2 +- collab-plugins/Cargo.toml | 8 +- collab-plugins/src/lib.rs | 2 +- .../src/local_storage/indexeddb/kv_impl.rs | 93 +++++++++++++------ .../src/local_storage/indexeddb/mod.rs | 7 +- .../src/local_storage/rocksdb/kv_impl.rs | 12 +++ collab-plugins/tests/web/edit_collab_test.rs | 4 +- collab-plugins/tests/web/indexeddb_test.rs | 2 +- collab/Cargo.toml | 3 +- collab/src/core/transaction.rs | 4 +- 11 files changed, 96 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f591c3bd..29715e596 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -301,6 +301,7 @@ dependencies = [ "bytes", "chrono", "collab", + "js-sys", "nanoid", "parking_lot", "serde", diff --git a/collab-database/Cargo.toml b/collab-database/Cargo.toml index 3ea90b721..1211f3504 100644 --- a/collab-database/Cargo.toml +++ b/collab-database/Cargo.toml @@ -46,5 +46,5 @@ futures = "0.3.30" zip = "0.6.6" [features] -wasm_build = ["getrandom/js"] +wasm_build = ["getrandom/js", "collab-plugins/wasm_build"] diff --git a/collab-plugins/Cargo.toml b/collab-plugins/Cargo.toml index 4f71ef27c..b4a52df69 100644 --- a/collab-plugins/Cargo.toml +++ b/collab-plugins/Cargo.toml @@ -50,13 +50,13 @@ dotenv = "0.15.0" futures = "0.3" [target.'cfg(target_arch = "wasm32")'.dependencies] -indexed_db_futures = { version = "0.4.1" } +indexed_db_futures = { version = "0.4" } js-sys = "0.3" -async-stream = "0.3.4" +async-stream = "0.3" futures = "0.3" wasm-bindgen = "0.2" -web-sys = { version = "0.3.67", features = ["console", "Window"] } -wasm-bindgen-futures = "0.4.40" +web-sys = { version = "0.3", features = ["console", "Window"] } +wasm-bindgen-futures = "0.4" tracing-wasm = "0.2" [target.'cfg(target_arch = "wasm32")'.dev-dependencies] diff --git a/collab-plugins/src/lib.rs b/collab-plugins/src/lib.rs index 351858836..290c3d242 100644 --- a/collab-plugins/src/lib.rs +++ b/collab-plugins/src/lib.rs @@ -25,5 +25,5 @@ if_native! { } if_wasm! { - pub type CollabKVDB = local_storage::indexeddb::kv_impl::CollabIndexeddb; + pub type CollabKVDB = local_storage::indexeddb::CollabIndexeddb; } diff --git a/collab-plugins/src/local_storage/indexeddb/kv_impl.rs b/collab-plugins/src/local_storage/indexeddb/kv_impl.rs index 3d9ebadf8..903d9aea4 100644 --- a/collab-plugins/src/local_storage/indexeddb/kv_impl.rs +++ b/collab-plugins/src/local_storage/indexeddb/kv_impl.rs @@ -1,6 +1,5 @@ use crate::local_storage::kv::PersistenceError; use collab::core::collab_plugin::EncodedCollab; -use indexed_db_futures::js_sys::wasm_bindgen::JsValue; use indexed_db_futures::prelude::*; use js_sys::{ArrayBuffer, Uint8Array}; @@ -15,8 +14,7 @@ use indexed_db_futures::web_sys::IdbKeyRange; use std::sync::Arc; use tokio::sync::RwLock; use tracing::error; -use wasm_bindgen::JsCast; -use web_sys::console; +use wasm_bindgen::{JsCast, JsValue}; use yrs::updates::decoder::Decode; use yrs::{Doc, Transact, Update}; @@ -195,15 +193,46 @@ impl CollabIndexeddb { Ok(EncodedCollab::new_v1(sv, doc_stata)) } + pub async fn is_exist(&self, uid: i64, object_id: &str) -> Result { + let read_guard = self.db.read().await; + let transaction = + read_guard.transaction_on_one_with_mode(COLLAB_KV_STORE, IdbTransactionMode::Readonly)?; + let store = store_from_transaction(&transaction)?; + Ok(self.get_doc_id(&store, uid, object_id).await.is_some()) + } + + pub async fn delete_doc(&self, uid: i64, object_id: &str) -> Result<(), PersistenceError> { + let write_guard = self.db.write().await; + let transaction = + write_guard.transaction_on_one_with_mode(COLLAB_KV_STORE, IdbTransactionMode::Readwrite)?; + let store = store_from_transaction(&transaction)?; + let doc_id = self + .get_doc_id(&store, uid, object_id) + .await + .ok_or_else(|| { + PersistenceError::RecordNotFound(format!("doc_id for object_id:{} is not found", object_id)) + })?; + + self.delete_all_updates(&store, doc_id).await?; + + // delete the doc state and state vector + let doc_state_key = make_doc_state_key(doc_id); + let sv_key = make_state_vector_key(doc_id); + store.delete(&to_js_key(doc_state_key.as_ref()))?; + store.delete(&to_js_key(sv_key.as_ref()))?; + transaction_result_to_result(transaction.await)?; + Ok(()) + } + pub async fn flush_doc( &self, uid: i64, object_id: &str, encoded: &EncodedCollab, ) -> Result<(), PersistenceError> { - let read_guard = self.db.write().await; + let write_guard = self.db.write().await; let transaction = - read_guard.transaction_on_one_with_mode(COLLAB_KV_STORE, IdbTransactionMode::Readwrite)?; + write_guard.transaction_on_one_with_mode(COLLAB_KV_STORE, IdbTransactionMode::Readwrite)?; let store = store_from_transaction(&transaction)?; let doc_id = self .get_doc_id(&store, uid, object_id) @@ -211,29 +240,9 @@ impl CollabIndexeddb { .ok_or_else(|| { PersistenceError::RecordNotFound(format!("doc_id for object_id:{} is not found", object_id)) })?; + self.delete_all_updates(&store, doc_id).await?; - let start = to_js_key(make_doc_start_key(doc_id)); - let end = to_js_key(make_doc_end_key(doc_id)); - let key_range = IdbKeyRange::bound(&start, &end).map_err(|err| { - PersistenceError::Internal(anyhow!("Get last update key fail. error: {:?}", err)) - })?; - - let cursor_request = store - .open_cursor_with_range(&key_range)? - .await? - .ok_or_else(|| { - PersistenceError::Internal(anyhow!("Open cursor fail. error: {:?}", "cursor is none")) - })?; - - // Delete the first key - let _ = cursor_request.delete(); - while cursor_request.continue_cursor()?.await? { - console::log_1(&JsValue::from_str("delete cursor")); - if let Err(err) = cursor_request.delete() { - error!("failed to delete cursor: {:?}", err) - } - } - + // save the new doc state and state vector let doc_state_key = make_doc_state_key(doc_id); let sv_key = make_state_vector_key(doc_id); self @@ -242,7 +251,6 @@ impl CollabIndexeddb { self .set_data_with_store(&store, sv_key, &encoded.state_vector) .await?; - transaction_result_to_result(transaction.await)?; Ok(()) } @@ -268,6 +276,35 @@ impl CollabIndexeddb { Ok(()) } + async fn delete_all_updates( + &self, + store: &IdbObjectStore<'_>, + doc_id: DocID, + ) -> Result<(), PersistenceError> { + let start = to_js_key(make_doc_start_key(doc_id)); + let end = to_js_key(make_doc_end_key(doc_id)); + let key_range = IdbKeyRange::bound(&start, &end).map_err(|err| { + PersistenceError::Internal(anyhow!("Get last update key fail. error: {:?}", err)) + })?; + + let cursor_request = store + .open_cursor_with_range(&key_range)? + .await? + .ok_or_else(|| { + PersistenceError::Internal(anyhow!("Open cursor fail. error: {:?}", "cursor is none")) + })?; + + // Delete the first key + let _ = cursor_request.delete(); + while cursor_request.continue_cursor()?.await? { + if let Err(err) = cursor_request.delete() { + error!("failed to delete cursor: {:?}", err) + } + } + + Ok(()) + } + pub async fn get_all_updates( &self, uid: i64, diff --git a/collab-plugins/src/local_storage/indexeddb/mod.rs b/collab-plugins/src/local_storage/indexeddb/mod.rs index 819c785c9..47c970dc3 100644 --- a/collab-plugins/src/local_storage/indexeddb/mod.rs +++ b/collab-plugins/src/local_storage/indexeddb/mod.rs @@ -1,2 +1,5 @@ -pub mod indexeddb_plugin; -pub mod kv_impl; +mod indexeddb_plugin; +mod kv_impl; + +pub use indexeddb_plugin::*; +pub use kv_impl::*; diff --git a/collab-plugins/src/local_storage/rocksdb/kv_impl.rs b/collab-plugins/src/local_storage/rocksdb/kv_impl.rs index 5d2e8eec8..a7f6089e5 100644 --- a/collab-plugins/src/local_storage/rocksdb/kv_impl.rs +++ b/collab-plugins/src/local_storage/rocksdb/kv_impl.rs @@ -3,6 +3,8 @@ use std::ops::RangeBounds; use std::path::Path; use std::sync::Arc; +use crate::local_storage::kv::doc::CollabKVAction; + use crate::local_storage::kv::{KVEntry, KVStore, KVTransactionDB, PersistenceError}; use rocksdb::Direction::Forward; use rocksdb::{ @@ -109,6 +111,16 @@ impl KVTransactionDBRocksdbImpl { Ok(Self { db: Arc::new(db) }) } + + pub async fn is_exist(&self, uid: i64, object_id: &str) -> Result { + let read_txn = self.read_txn(); + Ok(read_txn.is_exist(uid, object_id)) + } + + pub async fn delete_doc(&self, uid: i64, doc_id: &str) -> Result<(), PersistenceError> { + self.with_write_txn(|txn| txn.delete_doc(uid, doc_id))?; + Ok(()) + } } impl KVTransactionDB for KVTransactionDBRocksdbImpl { diff --git a/collab-plugins/tests/web/edit_collab_test.rs b/collab-plugins/tests/web/edit_collab_test.rs index c19966a16..991363709 100644 --- a/collab-plugins/tests/web/edit_collab_test.rs +++ b/collab-plugins/tests/web/edit_collab_test.rs @@ -2,8 +2,8 @@ use assert_json_diff::assert_json_eq; use collab::core::collab::MutexCollab; use collab::preclude::CollabBuilder; use collab_entity::CollabType; -use collab_plugins::local_storage::indexeddb::indexeddb_plugin::IndexeddbDiskPlugin; -use collab_plugins::local_storage::indexeddb::kv_impl::CollabIndexeddb; +use collab_plugins::local_storage::indexeddb::CollabIndexeddb; +use collab_plugins::local_storage::indexeddb::IndexeddbDiskPlugin; use js_sys::Promise; use serde_json::json; use std::sync::{Arc, Once}; diff --git a/collab-plugins/tests/web/indexeddb_test.rs b/collab-plugins/tests/web/indexeddb_test.rs index 0e651f7c6..a069993bf 100644 --- a/collab-plugins/tests/web/indexeddb_test.rs +++ b/collab-plugins/tests/web/indexeddb_test.rs @@ -1,5 +1,5 @@ use collab::core::collab_plugin::EncodedCollab; -use collab_plugins::local_storage::indexeddb::kv_impl::CollabIndexeddb; +use collab_plugins::local_storage::indexeddb::CollabIndexeddb; use uuid::Uuid; use wasm_bindgen_test::*; use yrs::Doc; diff --git a/collab/Cargo.toml b/collab/Cargo.toml index d3c31fd2f..2b3f49d92 100644 --- a/collab/Cargo.toml +++ b/collab/Cargo.toml @@ -23,7 +23,8 @@ bincode = "1.3.3" serde_repr = "0.1" [target.'cfg(target_arch = "wasm32")'.dependencies] -web-sys = { version = "0.3.67"} +web-sys = { version = "0.3"} +js-sys = "0.3" [dev-dependencies] tokio = { version = "1.26", features = ["rt"] } diff --git a/collab/src/core/transaction.rs b/collab/src/core/transaction.rs index 2918a6d90..5b6bfb78c 100644 --- a/collab/src/core/transaction.rs +++ b/collab/src/core/transaction.rs @@ -146,11 +146,11 @@ if_wasm! { impl Timer { fn start() -> Self { - Self { start: web_sys::js_sys::Date::now() } + Self { start: js_sys::Date::now() } } fn elapsed(&self) -> Duration { - let now = web_sys::js_sys::Date::now(); + let now = js_sys::Date::now(); let elapsed_ms = now - self.start; Duration::from_millis(elapsed_ms as u64) }