branch-2.1:[fix] (inverted index) Fix UTF-8 4-byte truncation issue a…

…nd add configuration to control correct term writing (#48657) (#48741) Cherry-picked from #48657
apache · Mar 6, 2025 · 3f684f2 · 3f684f2
1 parent 661eba5
commit 3f684f2
Show file tree

Hide file tree

Showing 5 changed files with 822 additions and 1 deletion.
diff --git a/be/src/clucene b/be/src/clucene
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
@@ -1089,6 +1089,7 @@ DEFINE_String(inverted_index_searcher_cache_limit, "10%");
 // set `true` to enable insert searcher into cache when write inverted index data
 DEFINE_Bool(enable_write_index_searcher_cache, "true");
 DEFINE_Bool(enable_inverted_index_cache_check_timestamp, "true");
+DEFINE_mBool(enable_inverted_index_correct_term_write, "true");
 DEFINE_Int32(inverted_index_fd_number_limit_percent, "20"); // 20%
 DEFINE_Int32(inverted_index_query_cache_shards, "256");
 

diff --git a/be/src/common/config.h b/be/src/common/config.h
@@ -1122,6 +1122,7 @@ DECLARE_String(inverted_index_searcher_cache_limit);
 // set `true` to enable insert searcher into cache when write inverted index data
 DECLARE_Bool(enable_write_index_searcher_cache);
 DECLARE_Bool(enable_inverted_index_cache_check_timestamp);
+DECLARE_mBool(enable_inverted_index_correct_term_write);
 DECLARE_Int32(inverted_index_fd_number_limit_percent); // 50%
 DECLARE_Int32(inverted_index_query_cache_shards);
 

diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
@@ -211,6 +211,7 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
         index_writer->setMaxFieldLength(MAX_FIELD_LEN);
         index_writer->setMergeFactor(MERGE_FACTOR);
         index_writer->setUseCompoundFile(false);
+        index_writer->setEnableCorrectTermWrite(config::enable_inverted_index_correct_term_write);
 
         return Status::OK();
     }
+1 −1		.github/workflows/build.yml
+3 −1		.github/workflows/clucene-ut.yml
+1 −0		CMakeLists.txt
+10 −0		src/core/CLucene/index/IndexWriter.cpp
+4 −1		src/core/CLucene/index/IndexWriter.h
+1 −1		src/core/CLucene/index/SDocumentWriter.cpp
+18 −5		src/core/CLucene/index/TermInfosWriter.cpp
+3 −0		src/core/CLucene/index/_TermInfosWriter.h
+20 −19		src/core/CLucene/store/IndexInput.cpp
+67 −21		src/core/CLucene/store/IndexOutput.cpp
+3 −0		src/core/CLucene/store/IndexOutput.h
+10,101 −0		src/ext/sse2neon/sse2neon.h
+1 −0		src/test/CMakeLists.txt
+1,095 −0		src/test/store/TestUTF8Chars.cpp
+1 −0		src/test/test.h
+1 −0		src/test/tests.cpp