From d4ce05605786775598cf755d9d92dc2f24bfaaa4 Mon Sep 17 00:00:00 2001
From: lhsoft <liuhuwork@gmail.com>
Date: Tue, 2 Apr 2024 17:38:58 +0800
Subject: [PATCH 1/2] add on_install_snapshot callback for statemachine

---
 src/braft/fsm_caller.cpp | 4 ++++
 src/braft/fsm_caller.h   | 1 +
 src/braft/node.cpp       | 5 +++++
 src/braft/node.h         | 3 +++
 src/braft/raft.cpp       | 1 +
 src/braft/raft.h         | 5 +++++
 src/braft/replicator.cpp | 2 ++
 7 files changed, 21 insertions(+)

diff --git a/src/braft/fsm_caller.cpp b/src/braft/fsm_caller.cpp
index 98913eea..50b9f753 100644
--- a/src/braft/fsm_caller.cpp
+++ b/src/braft/fsm_caller.cpp
@@ -496,6 +496,10 @@ void FSMCaller::do_stop_following(const LeaderChangeContext& stop_following_cont
     _fsm->on_stop_following(stop_following_context);
 }
 
+void FSMCaller::on_install_snapshot_start(const PeerId& peer_id) {
+    _fsm->on_install_snapshot_start(peer_id);
+}
+
 void FSMCaller::describe(std::ostream &os, bool use_html) {
     const char* newline = (use_html) ? "<br>" : "\n";
     TaskType cur_task = _cur_task;
diff --git a/src/braft/fsm_caller.h b/src/braft/fsm_caller.h
index 897a50f9..11dfe5f3 100644
--- a/src/braft/fsm_caller.h
+++ b/src/braft/fsm_caller.h
@@ -116,6 +116,7 @@ class BAIDU_CACHELINE_ALIGNMENT FSMCaller {
     int on_leader_start(int64_t term, int64_t lease_epoch);
     int on_start_following(const LeaderChangeContext& start_following_context);
     int on_stop_following(const LeaderChangeContext& stop_following_context);
+    void on_install_snapshot_start(const PeerId& peer_id);
     BRAFT_MOCK int on_error(const Error& e);
     int64_t last_applied_index() const {
         return _last_applied_index.load(butil::memory_order_relaxed);
diff --git a/src/braft/node.cpp b/src/braft/node.cpp
index b5ba39eb..50073be1 100644
--- a/src/braft/node.cpp
+++ b/src/braft/node.cpp
@@ -1365,6 +1365,11 @@ void NodeImpl::on_error(const Error& e) {
     lck.unlock();
 }
 
+void NodeImpl::install_snapshot_start(const PeerId& peer_id) {
+    _fsm_caller->on_install_snapshot_start(peer_id);
+}
+
+
 void NodeImpl::handle_vote_timeout() {
     std::unique_lock<raft_mutex_t> lck(_mutex);
 
diff --git a/src/braft/node.h b/src/braft/node.h
index b9dd3e82..0d239d07 100644
--- a/src/braft/node.h
+++ b/src/braft/node.h
@@ -241,6 +241,9 @@ friend class VoteBallotCtx;
 
     bool disable_cli() const { return _options.disable_cli; }
     bool is_witness() const { return _options.witness; }
+
+    // Called when leader start to install snapshot to remote peer
+    void install_snapshot_start(const PeerId& peer_id);
 private:
 friend class butil::RefCountedThreadSafe<NodeImpl>;
 
diff --git a/src/braft/raft.cpp b/src/braft/raft.cpp
index 6069f706..9906dd90 100644
--- a/src/braft/raft.cpp
+++ b/src/braft/raft.cpp
@@ -313,6 +313,7 @@ void StateMachine::on_configuration_committed(const Configuration& conf, int64_t
 
 void StateMachine::on_stop_following(const LeaderChangeContext&) {}
 void StateMachine::on_start_following(const LeaderChangeContext&) {}
+void StateMachine::on_install_snapshot_start(const PeerId& peer_id) {}
 
 BootstrapOptions::BootstrapOptions()
     : last_log_index(0)
diff --git a/src/braft/raft.h b/src/braft/raft.h
index cb80163a..a56b8d83 100644
--- a/src/braft/raft.h
+++ b/src/braft/raft.h
@@ -265,6 +265,11 @@ class StateMachine {
     // the very leader whom the follower starts to follow.
     // User can reset the node's information as it starts to follow some leader.
     virtual void on_start_following(const ::braft::LeaderChangeContext& ctx);
+
+    // Invoked when the leader start to send snapshot to |peer_id|
+    // Default: Do nothing
+    virtual void on_install_snapshot_start(const PeerId& peer_id);
+
 };
 
 enum State {
diff --git a/src/braft/replicator.cpp b/src/braft/replicator.cpp
index f2e2bb5c..7c14cbd2 100644
--- a/src/braft/replicator.cpp
+++ b/src/braft/replicator.cpp
@@ -789,6 +789,8 @@ void Replicator::_install_snapshot() {
                                             add_one_more_task(true)) {
         return _block(butil::gettimeofday_us(), EBUSY);
     }
+
+    node_impl->install_snapshot_start(_options.peer_id);
     
     // pre-set replicator state to INSTALLING_SNAPSHOT, so replicator could be
     // blocked if something is wrong, such as throttled for a period of time 

From 5317991f43121690b1a992ab1caf98be05741841 Mon Sep 17 00:00:00 2001
From: lhsoft <liuhuwork@gmail.com>
Date: Thu, 16 May 2024 10:59:25 +0800
Subject: [PATCH 2/2] fix comments

---
 src/braft/fsm_caller.cpp  | 4 ++--
 src/braft/fsm_caller.h    | 2 +-
 src/braft/log_manager.cpp | 4 ++++
 src/braft/node.cpp        | 4 ++--
 src/braft/node.h          | 4 ++--
 src/braft/raft.cpp        | 2 +-
 src/braft/raft.h          | 2 +-
 src/braft/replicator.cpp  | 2 +-
 8 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/src/braft/fsm_caller.cpp b/src/braft/fsm_caller.cpp
index 50b9f753..8be576b6 100644
--- a/src/braft/fsm_caller.cpp
+++ b/src/braft/fsm_caller.cpp
@@ -496,8 +496,8 @@ void FSMCaller::do_stop_following(const LeaderChangeContext& stop_following_cont
     _fsm->on_stop_following(stop_following_context);
 }
 
-void FSMCaller::on_install_snapshot_start(const PeerId& peer_id) {
-    _fsm->on_install_snapshot_start(peer_id);
+void FSMCaller::on_pre_send_snapshot(const PeerId& peer_id) {
+    _fsm->on_pre_send_snapshot(peer_id);
 }
 
 void FSMCaller::describe(std::ostream &os, bool use_html) {
diff --git a/src/braft/fsm_caller.h b/src/braft/fsm_caller.h
index 11dfe5f3..6ad1c84e 100644
--- a/src/braft/fsm_caller.h
+++ b/src/braft/fsm_caller.h
@@ -116,7 +116,7 @@ class BAIDU_CACHELINE_ALIGNMENT FSMCaller {
     int on_leader_start(int64_t term, int64_t lease_epoch);
     int on_start_following(const LeaderChangeContext& start_following_context);
     int on_stop_following(const LeaderChangeContext& stop_following_context);
-    void on_install_snapshot_start(const PeerId& peer_id);
+    void on_pre_send_snapshot(const PeerId& peer_id);
     BRAFT_MOCK int on_error(const Error& e);
     int64_t last_applied_index() const {
         return _last_applied_index.load(butil::memory_order_relaxed);
diff --git a/src/braft/log_manager.cpp b/src/braft/log_manager.cpp
index cf077751..cc0d14a3 100644
--- a/src/braft/log_manager.cpp
+++ b/src/braft/log_manager.cpp
@@ -668,6 +668,10 @@ void LogManager::set_snapshot(const SnapshotMeta* meta) {
             // We have last snapshot index
             _virtual_first_log_id = last_but_one_snapshot_id;
             truncate_prefix(last_but_one_snapshot_id.index + 1, lck);
+        } else {
+            // after restart, no followers, we can truncate log safely
+            _virtual_first_log_id = _last_snapshot_id;
+            truncate_prefix(meta->last_included_index() + 1, lck);
         }
         return;
     } else {
diff --git a/src/braft/node.cpp b/src/braft/node.cpp
index 50073be1..30e67a34 100644
--- a/src/braft/node.cpp
+++ b/src/braft/node.cpp
@@ -1365,8 +1365,8 @@ void NodeImpl::on_error(const Error& e) {
     lck.unlock();
 }
 
-void NodeImpl::install_snapshot_start(const PeerId& peer_id) {
-    _fsm_caller->on_install_snapshot_start(peer_id);
+void NodeImpl::pre_send_snapshot(const PeerId& peer_id) {
+    _fsm_caller->on_pre_send_snapshot(peer_id);
 }
 
 
diff --git a/src/braft/node.h b/src/braft/node.h
index 0d239d07..e6230f02 100644
--- a/src/braft/node.h
+++ b/src/braft/node.h
@@ -242,8 +242,8 @@ friend class VoteBallotCtx;
     bool disable_cli() const { return _options.disable_cli; }
     bool is_witness() const { return _options.witness; }
 
-    // Called when leader start to install snapshot to remote peer
-    void install_snapshot_start(const PeerId& peer_id);
+    // Called when leader start to send snapshot to remote peer
+    void pre_send_snapshot(const PeerId& peer_id);
 private:
 friend class butil::RefCountedThreadSafe<NodeImpl>;
 
diff --git a/src/braft/raft.cpp b/src/braft/raft.cpp
index 9906dd90..c7c66ef6 100644
--- a/src/braft/raft.cpp
+++ b/src/braft/raft.cpp
@@ -313,7 +313,7 @@ void StateMachine::on_configuration_committed(const Configuration& conf, int64_t
 
 void StateMachine::on_stop_following(const LeaderChangeContext&) {}
 void StateMachine::on_start_following(const LeaderChangeContext&) {}
-void StateMachine::on_install_snapshot_start(const PeerId& peer_id) {}
+void StateMachine::on_pre_send_snapshot(const PeerId& peer_id) {}
 
 BootstrapOptions::BootstrapOptions()
     : last_log_index(0)
diff --git a/src/braft/raft.h b/src/braft/raft.h
index a56b8d83..335e5e58 100644
--- a/src/braft/raft.h
+++ b/src/braft/raft.h
@@ -268,7 +268,7 @@ class StateMachine {
 
     // Invoked when the leader start to send snapshot to |peer_id|
     // Default: Do nothing
-    virtual void on_install_snapshot_start(const PeerId& peer_id);
+    virtual void on_pre_send_snapshot(const PeerId& peer_id);
 
 };
 
diff --git a/src/braft/replicator.cpp b/src/braft/replicator.cpp
index 7c14cbd2..faec0977 100644
--- a/src/braft/replicator.cpp
+++ b/src/braft/replicator.cpp
@@ -790,7 +790,7 @@ void Replicator::_install_snapshot() {
         return _block(butil::gettimeofday_us(), EBUSY);
     }
 
-    node_impl->install_snapshot_start(_options.peer_id);
+    node_impl->pre_send_snapshot(_options.peer_id);
     
     // pre-set replicator state to INSTALLING_SNAPSHOT, so replicator could be
     // blocked if something is wrong, such as throttled for a period of time