Merge pull request #42 from myscale/sync/myscaledb-oss

MyscaleDB 1.7.1 Release

README.md

@@ -80,7 +80,7 @@ The simplest way to use MyScaleDB is to create an instance on MyScale Cloud serv
 To quickly get a MyScaleDB instance up and running, simply pull and run the latest Docker image:
 
 ```bash
-docker run --name myscaledb --net=host myscale/myscaledb:1.6.4
+docker run --name myscaledb --net=host myscale/myscaledb:1.7.1
 ```
 
 >Note: Myscale's default configuration only allows localhost ip access. For the docker run startup method, you need to specify `--net=host` to access services deployed in docker mode on the current node.
@@ -114,7 +114,7 @@ version: '3.7'
 
 services:
   myscaledb:
-    image: myscale/myscaledb:1.6.4
+    image: myscale/myscaledb:1.7.1
     tty: true
     ports:
       - '8123:8123'

cmake/autogenerated_myscale_versions.txt

@@ -3,9 +3,9 @@
 # NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
 # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
 SET(MYSCALE_VERSION_MAJOR 1)
-SET(MYSCALE_VERSION_MINOR 6)
-SET(MYSCALE_VERSION_PATCH 4)
-SET(MYSCALE_VERSION_DESCRIBE myscale-v1.6.4)
-SET(MYSCALE_VERSION_STRING 1.6.4)
+SET(MYSCALE_VERSION_MINOR 7)
+SET(MYSCALE_VERSION_PATCH 1)
+SET(MYSCALE_VERSION_DESCRIBE myscale-v1.7.1)
+SET(MYSCALE_VERSION_STRING 1.7.1)
 # end of autochange
 

src/Core/Settings.h

@@ -921,6 +921,7 @@ class IColumn;
     M(UInt64, hybrid_search_top_k_multiple_base, 3, "Default multiple base on top k for num_candidates in hybrid search", 0) \
     M(Bool, optimize_prefilter_in_search, true, "Enable prewhere optimization for vector or text search if some partition columns in prewhere condition.", 0) \
     M(UInt64, max_search_result_window, 10000, "The maximum value of n + m in limit clause for pagination in vector/text/hybrid search", 0) \
+    M(Bool, dfs_query_then_fetch, false, "Enable Distributed Frequency Search (DFS) query to gather global statistical info for accurate BM25 calculation.", 0) \
     // End of COMMON_SETTINGS
     // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS and move obsolete settings to OBSOLETE_SETTINGS.
 

src/Databases/DatabasesCommon.cpp

@@ -290,7 +290,8 @@ StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_n
             }
             // clean stores
             // TODO needs refine TantivyIndexStoreFactory, the remove func is only for data part relative path.
-            TantivyIndexStoreFactory::instance().remove(table_relative_path);
+            auto index_names = res->getInMemoryMetadataPtr()->getSecondaryIndices().getAllRegisteredNames();
+            TantivyIndexStoreFactory::instance().remove(table_relative_path, index_names);
         }
     }
 #endif

src/Interpreters/ExpressionAnalyzer.cpp

@@ -594,8 +594,9 @@ void ExpressionAnalyzer::analyzeTextSearch(ActionsDAGPtr & temp_actions)
         }
     }
 
-    /// text search cannot be performed when no fts index exists
-    if (has_text_search)
+    /// Text search cannot be performed when no fts index exists
+    /// Skip the fts index check when table is distributed.
+    if (!syntax->is_remote_storage && has_text_search)
         checkTantivyIndex(syntax->storage_snapshot, text_search_info->text_column_name);
 }
 
@@ -615,8 +616,8 @@ void ExpressionAnalyzer::analyzeHybridSearch(ActionsDAGPtr & temp_actions)
 
     if (has_hybrid_search && hybrid_search_info)
     {
-        /// check fts index
-        if (hybrid_search_info->text_search_info)
+        /// Skip the fts index check when table is distributed.
+        if (!syntax->is_remote_storage && hybrid_search_info->text_search_info)
             checkTantivyIndex(syntax->storage_snapshot, hybrid_search_info->text_search_info->text_column_name);
 
         /// Get vector search type and dim from metadata, check paramaters in vector scan and add to vector_paramters
@@ -1095,7 +1096,7 @@ TextSearchInfoPtr ExpressionAnalyzer::commonMakeTextSearchInfo(
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown parameter {} for TextSearch", param_key);
     }
 
-    return std::make_shared<TextSearchInfo>(text_column_name, query_text_value, function_col_name, topk, syntax->direction, text_operator, enable_natural_language_query);
+    return std::make_shared<TextSearchInfo>(text_column_name, query_text_value, function_col_name, topk, text_operator, enable_natural_language_query);
 }
 
 bool ExpressionAnalyzer::makeTextSearchInfo(ActionsDAGPtr & actions)
@@ -1110,7 +1111,6 @@ bool ExpressionAnalyzer::makeTextSearchInfo(ActionsDAGPtr & actions)
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "Wrong argument number in TextSearch function: expected 2, got {}", arguments.size());
         }
 
-
         Array parameters = (node->parameters) ? getAggregateFunctionParametersArray(node->parameters, "", getContext()) : Array();
 
         /// Only need actions for the second argument, the first argument is used for search index.
@@ -1253,11 +1253,10 @@ bool ExpressionAnalyzer::makeHybridSearchInfo(ActionsDAGPtr & actions)
                     "Wrong HybridSearch parameter for Relative Score Fusion(RSF), valid value is in interval [0.0f, 1.0f]");
             }
 
-            auto metric = Search::getMetricType(syntax->vector_scan_metric_type, vector_scan_descriptions[0].vector_search_type);
             hybrid_search_info = std::make_shared<HybridSearchInfo>(
                 std::make_shared<VectorScanInfo>(vector_scan_descriptions),
                 tmp_text_search_info,
-                function_column_name, static_cast<int>(syntax->limit_length), hybrid_fusion_type, hybrid_fusion_weight, metric);
+                function_column_name, static_cast<int>(syntax->limit_length), hybrid_fusion_type, hybrid_fusion_weight);
         }
         else if (isRankFusion(hybrid_fusion_type))
         {

src/Interpreters/InterpreterSelectQuery.cpp

@@ -101,6 +101,9 @@
 #include "config_version.h"
 #include <Interpreters/Context.h>
 
+#if USE_TANTIVY_SEARCH
+#    include <VectorIndex/Utils/CommonUtils.h>
+#endif
 
 namespace DB
 {
@@ -641,14 +644,29 @@ InterpreterSelectQuery::InterpreterSelectQuery(
                 current_info.query = query_ptr;
                 current_info.syntax_analyzer_result = syntax_analyzer_result;
 
+                /// Support full text search table function
+                NameSet table_columns;
+                bool from_table_function = false;
+                if (storage->getName() == "FullTextSearch")
+                {
+                    from_table_function = true;
+                    table_columns = {collections::map<std::unordered_set>(
+                                    metadata_snapshot->getColumns().getAllPhysical(), [](const NameAndTypePair & col) { return col.name; })};
+                    table_columns.erase(SCORE_COLUMN_NAME);
+
+                    current_info.has_hybrid_search = true;
+                }
+                else if (syntax_analyzer_result && !syntax_analyzer_result->hybrid_search_funcs.empty())
+                    current_info.has_hybrid_search = true;
+
                 Names queried_columns = syntax_analyzer_result->requiredSourceColumns();
                 const auto & supported_prewhere_columns = storage->supportedPrewhereColumns();
 
                 MergeTreeWhereOptimizer where_optimizer{
                     std::move(column_compressed_sizes),
                     metadata_snapshot,
                     queried_columns,
-                    supported_prewhere_columns,
+                    from_table_function ? table_columns : supported_prewhere_columns,
                     log};
 
                 where_optimizer.optimize(current_info, context);
@@ -762,6 +780,23 @@ InterpreterSelectQuery::InterpreterSelectQuery(
 
     analyze(shouldMoveToPrewhere());
 
+#if USE_TANTIVY_SEARCH
+    if (!options.only_analyze && storage && query_analyzer->getAnalyzedData().text_search_info && context->getSettingsRef().dfs_query_then_fetch)
+    {
+        /// Collect global statistics information of all shards used in BM25 calculation when text search is distributed
+        if (auto distributed_storage = std::dynamic_pointer_cast<StorageDistributed>(storage))
+        {
+            collectStatisticForBM25Calculation(
+                context,
+                distributed_storage->getClusterName(),
+                distributed_storage->getRemoteDatabaseName(),
+                distributed_storage->getRemoteTableName(),
+                query_analyzer->getAnalyzedData().text_search_info->text_column_name,
+                query_analyzer->getAnalyzedData().text_search_info->query_text);
+        }
+    }
+#endif
+
     bool need_analyze_again = false;
     bool can_analyze_again = false;
     if (context->hasQueryContext())