Data sync refactoring (#65)

Data Sync refactoring: changing synchronization protocol

meta-secret/Earthfile

@@ -1,6 +1,6 @@
 VERSION 0.8
 FROM scratch
-ENV RUST_VERSION="1.82.0"
+ENV RUST_VERSION="1.84.0"
 
 generate-cargo-chef-recipe:
     FROM lukemathwalker/cargo-chef:latest-rust-${RUST_VERSION}-bookworm

meta-secret/cli/Dockerfile

@@ -1,5 +1,5 @@
 # https://docs.docker.com/develop/develop-images/multistage-build/
-FROM rust:1.80.1 AS build
+FROM rust:1.84.0-bookworm AS build
 COPY ./ /build
 WORKDIR /build
 RUN cargo test

meta-secret/core/src/meta_tests/fixture_util.rs

@@ -2,12 +2,13 @@
 pub mod fixture {
     use crate::meta_tests::fixture_util::fixture::states::{BaseState, EmptyState, ExtendedState};
     use crate::node::app::meta_app::meta_client_service::fixture::MetaClientServiceFixture;
+    use crate::node::app::sync::sync_protocol::fixture::SyncProtocolFixture;
     use crate::node::common::model::device::device_creds::fixture::DeviceCredentialsFixture;
     use crate::node::common::model::user::user_creds::fixture::UserCredentialsFixture;
-    use crate::node::db::objects::global_index::fixture::ServerPersistentGlobalIndexFixture;
     use crate::node::db::objects::persistent_object::fixture::PersistentObjectFixture;
     use crate::node::db::repo::persistent_credentials::fixture::PersistentCredentialsFixture;
-    use crate::node::server::server_app::fixture::{ServerAppFixture, ServerDataTransferFixture};
+    use crate::node::server::server_app::fixture::ServerAppFixture;
+    use std::sync::Arc;
 
     pub struct FixtureRegistry<S> {
         pub state: S,
@@ -31,13 +32,10 @@ pub mod fixture {
         pub async fn base() -> anyhow::Result<FixtureRegistry<BaseState>> {
             let empty = FixtureRegistry::empty();
             let p_creds = PersistentCredentialsFixture::init(&empty.state).await?;
-            let server_p_gi = ServerPersistentGlobalIndexFixture::from(&empty);
 
             let base = BaseState {
                 empty: empty.state,
                 p_creds,
-                server_p_gi,
-                server_dt: ServerDataTransferFixture::generate(),
             };
 
             Ok(FixtureRegistry { state: base })
@@ -47,28 +45,29 @@ pub mod fixture {
         pub async fn extended() -> anyhow::Result<FixtureRegistry<ExtendedState>> {
             let base = FixtureRegistry::base().await?;
 
-            let server_app = ServerAppFixture::try_from(&base)?;
-            let meta_client_service = MetaClientServiceFixture::from(&base);
+            let server_app = Arc::new(ServerAppFixture::try_from(&base)?);
+            let sync = SyncProtocolFixture::new(server_app.clone());
+            let meta_client_service = MetaClientServiceFixture::from(&base.state, &sync);
 
             let state = ExtendedState {
                 base: base.state,
                 server_app,
                 meta_client_service,
+                sync,
             };
             Ok(FixtureRegistry { state })
         }
     }
 
     pub mod states {
         use crate::node::app::meta_app::meta_client_service::fixture::MetaClientServiceFixture;
+        use crate::node::app::sync::sync_protocol::fixture::SyncProtocolFixture;
         use crate::node::common::model::device::device_creds::fixture::DeviceCredentialsFixture;
         use crate::node::common::model::user::user_creds::fixture::UserCredentialsFixture;
-        use crate::node::db::objects::global_index::fixture::ServerPersistentGlobalIndexFixture;
         use crate::node::db::objects::persistent_object::fixture::PersistentObjectFixture;
         use crate::node::db::repo::persistent_credentials::fixture::PersistentCredentialsFixture;
-        use crate::node::server::server_app::fixture::{
-            ServerAppFixture, ServerDataTransferFixture,
-        };
+        use crate::node::server::server_app::fixture::ServerAppFixture;
+        use std::sync::Arc;
 
         pub enum Fixture {
             Empty(EmptyState),
@@ -85,17 +84,14 @@ pub mod fixture {
 
         pub struct BaseState {
             pub empty: EmptyState,
-
             pub p_creds: PersistentCredentialsFixture,
-            pub server_p_gi: ServerPersistentGlobalIndexFixture,
-
-            pub server_dt: ServerDataTransferFixture,
         }
 
         pub struct ExtendedState {
             pub base: BaseState,
-            pub server_app: ServerAppFixture,
+            pub server_app: Arc<ServerAppFixture>,
             pub meta_client_service: MetaClientServiceFixture,
+            pub sync: SyncProtocolFixture,
         }
     }
 }

meta-secret/core/src/node/app/meta_app/meta_client_service.rs

@@ -5,6 +5,7 @@ use tracing::{error, info, instrument};
 
 use crate::node::app::meta_app::messaging::{GenericAppStateRequest, GenericAppStateResponse};
 use crate::node::app::sync::sync_gateway::SyncGateway;
+use crate::node::app::sync::sync_protocol::SyncProtocol;
 use crate::node::common::actor::ServiceState;
 use crate::node::common::data_transfer::MpscDataTransfer;
 use crate::node::common::model::device::common::DeviceName;
@@ -20,17 +21,17 @@ use crate::node::db::repo::generic_db::KvLogEventRepo;
 use crate::node::db::repo::persistent_credentials::PersistentCredentials;
 use crate::secret::MetaDistributor;
 
-pub struct MetaClientService<Repo: KvLogEventRepo> {
-    pub data_transfer: Arc<MetaClientDataTransfer>,
-    pub sync_gateway: Arc<SyncGateway<Repo>>,
-    pub state_provider: Arc<MetaClientStateProvider>,
+pub(crate) struct MetaClientService<Repo: KvLogEventRepo, Sync: SyncProtocol> {
+    data_transfer: Arc<MetaClientDataTransfer>,
+    sync_gateway: Arc<SyncGateway<Repo, Sync>>,
+    state_provider: Arc<MetaClientStateProvider>,
 }
 
 pub struct MetaClientDataTransfer {
     pub dt: MpscDataTransfer<GenericAppStateRequest, GenericAppStateResponse>,
 }
 
-impl<Repo: KvLogEventRepo> MetaClientService<Repo> {
+impl<Repo: KvLogEventRepo, Sync: SyncProtocol> MetaClientService<Repo, Sync> {
     #[instrument(skip_all)]
     pub async fn run(&self) -> anyhow::Result<()> {
         info!("Run meta_app service");
@@ -210,7 +211,7 @@ impl MetaClientStateProvider {
 
 #[cfg(test)]
 pub mod fixture {
-    use crate::meta_tests::fixture_util::fixture::states::BaseState;
+    use crate::meta_tests::fixture_util::fixture::states::{BaseState, ExtendedState};
     use crate::meta_tests::fixture_util::fixture::FixtureRegistry;
     use crate::node::app::meta_app::meta_client_service::{
         MetaClientDataTransfer, MetaClientService, MetaClientStateProvider,
@@ -219,10 +220,12 @@ pub mod fixture {
     use crate::node::common::data_transfer::MpscDataTransfer;
     use crate::node::db::in_mem_db::InMemKvLogEventRepo;
     use std::sync::Arc;
+    use crate::node::app::sync::sync_protocol::EmbeddedSyncProtocol;
+    use crate::node::app::sync::sync_protocol::fixture::SyncProtocolFixture;
 
     pub struct MetaClientServiceFixture {
-        pub client: Arc<MetaClientService<InMemKvLogEventRepo>>,
-        pub vd: Arc<MetaClientService<InMemKvLogEventRepo>>,
+        pub client: Arc<MetaClientService<InMemKvLogEventRepo, EmbeddedSyncProtocol>>,
+        pub vd: Arc<MetaClientService<InMemKvLogEventRepo, EmbeddedSyncProtocol>>,
 
         pub state_provider: MetaClientStateProviderFixture,
         pub data_transfer: MetaClientDataTransferFixture,
@@ -231,11 +234,11 @@ pub mod fixture {
     }
 
     impl MetaClientServiceFixture {
-        pub fn from(base: &FixtureRegistry<BaseState>) -> Self {
+        pub fn from(base: &BaseState, sync: &SyncProtocolFixture) -> Self {
             let state_provider = MetaClientStateProviderFixture::generate();
             let dt_fxr = MetaClientDataTransferFixture::generate();
 
-            let sync_gateway = SyncGatewayFixture::from(base);
+            let sync_gateway = SyncGatewayFixture::from(&base.empty, sync);
 
             let client = Arc::new(MetaClientService {
                 data_transfer: dt_fxr.client.clone(),

meta-secret/core/src/node/app/sync/mod.rs

@@ -1,2 +1,2 @@
-mod global_index;
 pub mod sync_gateway;
+pub mod sync_protocol;