From 2f4df04f506d9f4d4acef0c101431dead9f304be Mon Sep 17 00:00:00 2001
From: srlch <111035020+srlch@users.noreply.github.com>
Date: Thu, 27 Feb 2025 14:21:35 +0800
Subject: [PATCH] [Enhancement] Wait apply finish out of tablet meta lock when
 set tablet state as SHUTDOWN (#56266)

Signed-off-by: srlch <linzichao@starrocks.com>
(cherry picked from commit 76db0ef14c916c7fffa6ecf1a87971b7b5dfc17f)
---
 be/src/storage/tablet.cpp         |  2 +-
 be/src/storage/tablet_manager.cpp | 21 +++++++++++++++++++++
 be/src/storage/tablet_updates.cpp |  8 ++++----
 be/src/storage/tablet_updates.h   |  3 ++-
 4 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/be/src/storage/tablet.cpp b/be/src/storage/tablet.cpp
index 5198420c00dfe7..8a8e5c3a100351 100644
--- a/be/src/storage/tablet.cpp
+++ b/be/src/storage/tablet.cpp
@@ -1571,7 +1571,7 @@ Status Tablet::rowset_commit(int64_t version, const RowsetSharedPtr& rowset, uin
 
 void Tablet::on_shutdown() {
     if (_updates) {
-        _updates->_stop_and_wait_apply_done();
+        _updates->stop_and_wait_apply_done();
     }
 }
 
diff --git a/be/src/storage/tablet_manager.cpp b/be/src/storage/tablet_manager.cpp
index 2c8cc5a4e8236c..bf44826630a36a 100644
--- a/be/src/storage/tablet_manager.cpp
+++ b/be/src/storage/tablet_manager.cpp
@@ -407,6 +407,13 @@ Status TabletManager::drop_tablet(TTabletId tablet_id, TabletDropFlag flag) {
 
     DroppedTabletInfo drop_info{.tablet = dropped_tablet, .flag = flag};
 
+    // meta lock free shutdown for primary key table. In current impl, TabletUpdates's context
+    // is protected by specified lock defined in TabletUpdates itself but not tablet meta lock
+    // It is safe call stop_and_wait_apply_done out of tablet meta lock
+    if (dropped_tablet->updates() != nullptr) {
+        dropped_tablet->updates()->stop_and_wait_apply_done();
+    }
+
     if (flag == kDeleteFiles) {
         {
             // NOTE: Other threads may save the tablet meta back to storage again after we
@@ -492,6 +499,13 @@ Status TabletManager::drop_tablets_on_error_root_path(const std::vector<TabletIn
     }
 
     for (const auto& dropped_tablet : dropped_tablets) {
+        // meta lock free shutdown for primary key table. In current impl, TabletUpdates's context
+        // is protected by specified lock defined in TabletUpdates itself but not tablet meta lock
+        // It is safe call stop_and_wait_apply_done out of tablet meta lock
+        if (dropped_tablet->updates() != nullptr) {
+            dropped_tablet->updates()->stop_and_wait_apply_done();
+        }
+
         // make sure dropped tablet state is TABLET_SHUTDOWN IN MEMORY ONLY!
         // any persistent operation is useless because the disk has failed
         // and the IO operation should be always failed in this case.
@@ -1520,6 +1534,13 @@ Status TabletManager::_drop_tablet_unlocked(TTabletId tablet_id, TabletDropFlag
 
     DroppedTabletInfo drop_info{.tablet = dropped_tablet, .flag = flag};
 
+    // meta lock free shutdown for primary key table. In current impl, TabletUpdates's context
+    // is protected by specified lock defined in TabletUpdates itself but not tablet meta lock
+    // It is safe call stop_and_wait_apply_done out of tablet meta lock
+    if (dropped_tablet->updates() != nullptr) {
+        dropped_tablet->updates()->stop_and_wait_apply_done();
+    }
+
     if (flag == kDeleteFiles) {
         {
             // NOTE: Other threads may save the tablet meta back to storage again after we
diff --git a/be/src/storage/tablet_updates.cpp b/be/src/storage/tablet_updates.cpp
index 42dc44c12fbaf4..92ace8b8648c4f 100644
--- a/be/src/storage/tablet_updates.cpp
+++ b/be/src/storage/tablet_updates.cpp
@@ -83,7 +83,7 @@ std::string EditVersion::to_string() const {
 TabletUpdates::TabletUpdates(Tablet& tablet) : _tablet(tablet), _unused_rowsets(UINT64_MAX) {}
 
 TabletUpdates::~TabletUpdates() {
-    _stop_and_wait_apply_done();
+    stop_and_wait_apply_done();
 }
 
 template <class Itr1, class Itr2>
@@ -1022,7 +1022,7 @@ void TabletUpdates::_wait_apply_done() {
     }
 }
 
-void TabletUpdates::_stop_and_wait_apply_done() {
+void TabletUpdates::stop_and_wait_apply_done() {
     _apply_stopped = true;
     _wait_apply_done();
 }
@@ -4843,7 +4843,7 @@ Status TabletUpdates::load_snapshot(const SnapshotMeta& snapshot_meta, bool rest
             RETURN_IF_ERROR(check_rowset_files(rowset_meta_pb));
         }
         // Stop apply thread.
-        _stop_and_wait_apply_done();
+        stop_and_wait_apply_done();
 
         DeferOp defer([&]() {
             if (!_error.load()) {
@@ -5570,7 +5570,7 @@ Status TabletUpdates::recover() {
     }
     LOG(INFO) << "Tablet " << _tablet.tablet_id() << " begin do recover: " << _error_msg;
     // Stop apply thread.
-    _stop_and_wait_apply_done();
+    stop_and_wait_apply_done();
 
     DeferOp defer([&]() {
         if (!_error.load()) {
diff --git a/be/src/storage/tablet_updates.h b/be/src/storage/tablet_updates.h
index 4e053a2da6d9bf..d24bdb90923466 100644
--- a/be/src/storage/tablet_updates.h
+++ b/be/src/storage/tablet_updates.h
@@ -381,6 +381,8 @@ class TabletUpdates {
 
     void rewrite_rs_meta(bool is_fatal);
 
+    void stop_and_wait_apply_done();
+
 private:
     friend class Tablet;
     friend class PrimaryIndex;
@@ -435,7 +437,6 @@ class TabletUpdates {
                               EditVersion* commit_version);
 
     void _wait_apply_done();
-    void _stop_and_wait_apply_done();
 
     Status _do_compaction(std::unique_ptr<CompactionInfo>* pinfo);