Greedy query planning for large connected components

src/engine/QueryPlanner.cpp

@@ -33,6 +33,7 @@
 #include "engine/TransitivePathBase.h"
 #include "engine/Union.h"
 #include "engine/Values.h"
+#include "global/RuntimeParameters.h"
 #include "parser/Alias.h"
 #include "parser/SparqlParserHelpers.h"
 
@@ -1193,6 +1194,19 @@
   row.insert(row.end(), addedPlans.begin(), addedPlans.end());
 }
 
+// _____________________________________________________________________________
+size_t QueryPlanner::findUniqueNodeIds(
+    const std::vector<SubtreePlan>& connectedComponent) {
+  ad_utility::HashSet<uint64_t> uniqueNodeIds;
+  // TODO<joka921> Assert that all the node IDs only consist of a single
+  // element. Or even better: Do a bitwise or and then count the number of set
+  // bits...
+  std::ranges::copy(connectedComponent | std::views::transform(
+                                             &SubtreePlan::_idsOfIncludedNodes),
+                    std::inserter(uniqueNodeIds, uniqueNodeIds.end()));
+  return uniqueNodeIds.size();
+}
+
 // _____________________________________________________________________________
 std::vector<QueryPlanner::SubtreePlan>
 QueryPlanner::runDynamicProgrammingOnConnectedComponent(
@@ -1206,16 +1220,12 @@
   dpTab.push_back(std::move(connectedComponent));
   applyFiltersIfPossible<false>(dpTab.back(), filters);
   applyTextLimitsIfPossible(dpTab.back(), textLimits, false);
-  ad_utility::HashSet<uint64_t> uniqueNodeIds;
-  std::ranges::copy(
-      dpTab.back() | std::views::transform(&SubtreePlan::_idsOfIncludedNodes),
-      std::inserter(uniqueNodeIds, uniqueNodeIds.end()));
-  size_t numSeeds = uniqueNodeIds.size();
+  size_t numSeeds = findUniqueNodeIds(dpTab.back());
 
   for (size_t k = 2; k <= numSeeds; ++k) {
     LOG(TRACE) << "Producing plans that unite " << k << " triples."
                << std::endl;
-    dpTab.emplace_back(vector<SubtreePlan>());
+    dpTab.emplace_back();
     for (size_t i = 1; i * 2 <= k; ++i) {
       checkCancellation();
       auto newPlans = merge(dpTab[i - 1], dpTab[k - i - 1], tg);
@@ -1230,6 +1240,38 @@
   return std::move(dpTab.back());
 }
 
+// _____________________________________________________________________________
+std::vector<QueryPlanner::SubtreePlan>
+QueryPlanner::runGreedyPlanningOnConnectedComponent(
+    std::vector<SubtreePlan> connectedComponent,
+    const vector<SparqlFilter>& filters, const TextLimitMap& textLimits,
+    const TripleGraph& tg) const {
+  auto& result = connectedComponent;
+  applyFiltersIfPossible<true>(result, filters);
+  applyTextLimitsIfPossible(result, textLimits, true);
+  size_t numSeeds = findUniqueNodeIds(result);
+
+  while (numSeeds > 1) {
+    checkCancellation();
+    auto newPlans = merge(result, result, tg);
+    applyFiltersIfPossible<true>(newPlans, filters);
+    applyTextLimitsIfPossible(newPlans, textLimits, true);
+    auto smallestIdx = findSmallestExecutionTree(newPlans);
+    auto& cheapestNewTree = newPlans.at(smallestIdx);
+    size_t oldSize = result.size();
+    std::erase_if(result, [&cheapestNewTree](const auto& plan) {
+      // TODO<joka921> We can also assert some other invariants here.
+      return (cheapestNewTree._idsOfIncludedNodes & plan._idsOfIncludedNodes) !=
+             0;
+    });
+    result.push_back(std::move(cheapestNewTree));
+    AD_CORRECTNESS_CHECK(result.size() < oldSize);
+    numSeeds--;
+  }
+  // TODO<joka921> Assert that all seeds are covered by the result.
+  return std::move(result);
+}
+
 // _____________________________________________________________________________
 vector<vector<QueryPlanner::SubtreePlan>> QueryPlanner::fillDpTab(
     const QueryPlanner::TripleGraph& tg, vector<SparqlFilter> filters,
@@ -1247,9 +1289,13 @@
     components[componentIndices.at(i)].push_back(std::move(initialPlans.at(i)));
   }
   vector<vector<SubtreePlan>> lastDpRowFromComponents;
+  bool useGreedyPlanning = RuntimeParameters().get<"use-greedy-planning">();
   for (auto& component : components | std::views::values) {
-    lastDpRowFromComponents.push_back(runDynamicProgrammingOnConnectedComponent(
-        std::move(component), filters, textLimits, tg));
+    auto impl = useGreedyPlanning
+                    ? &QueryPlanner::runGreedyPlanningOnConnectedComponent
+                    : &QueryPlanner::runDynamicProgrammingOnConnectedComponent;
+    lastDpRowFromComponents.push_back(
+        std::invoke(impl, this, std::move(component), filters, textLimits, tg));
     checkCancellation();
   }
   size_t numConnectedComponents = lastDpRowFromComponents.size();
@@ -1593,6 +1639,21 @@
          lastRow.begin();
 };
 
+// _________________________________________________________________________________
+size_t QueryPlanner::findSmallestExecutionTree(
+    const std::vector<SubtreePlan>& lastRow) {
+  AD_CONTRACT_CHECK(!lastRow.empty());
+  // TODO<joka921> Precompute the sizes and costs. (or check if they are
+  // cached).
+  auto compare = [](const auto& a, const auto& b) {
+    auto aSize = a.getSizeEstimate(), bSize = b.getSizeEstimate();
+    auto aCost = a.getCostEstimate(), bCost = b.getCostEstimate();
+    return std::tie(aSize, aCost) < std::tie(bSize, bCost);
+  };
+  return std::min_element(lastRow.begin(), lastRow.end(), compare) -
+         lastRow.begin();
+};
+
 // _____________________________________________________________________________
 std::vector<QueryPlanner::SubtreePlan> QueryPlanner::createJoinCandidates(
     const SubtreePlan& ain, const SubtreePlan& bin,

src/engine/QueryPlanner.h

@@ -445,6 +445,11 @@ class QueryPlanner {
       const vector<SparqlFilter>& filters, const TextLimitMap& textLimits,
       const TripleGraph& tg) const;
 
+  std::vector<QueryPlanner::SubtreePlan> runGreedyPlanningOnConnectedComponent(
+      std::vector<SubtreePlan> connectedComponent,
+      const vector<SparqlFilter>& filters, const TextLimitMap& textLimits,
+      const TripleGraph& tg) const;
+
   // Creates a SubtreePlan for the given text leaf node in the triple graph.
   // While doing this the TextLimitMetaObjects are created and updated according
   // to the text leaf node.
@@ -529,8 +534,12 @@ class QueryPlanner {
    * sorting by the cache key when comparing equally cheap indices, else the
    * first element that has the minimum index is returned.
    */
-  [[nodiscard]] size_t findCheapestExecutionTree(
+  size_t findCheapestExecutionTree(
       const std::vector<SubtreePlan>& lastRow) const;
+  static size_t findSmallestExecutionTree(
+      const std::vector<SubtreePlan>& lastRow);
+  static size_t findUniqueNodeIds(
+      const std::vector<SubtreePlan>& connectedComponent);
 
   /// if this Planner is not associated with a queryExecutionContext we are only
   /// in the unit test mode

src/global/RuntimeParameters.h

@@ -47,7 +47,8 @@ inline auto& RuntimeParameters() {
         SizeT<"lazy-index-scan-max-size-materialization">{1'000'000},
         Bool<"use-binsearch-transitive-path">{true},
         Bool<"group-by-hash-map-enabled">{false},
-        SizeT<"service-max-value-rows">{100}};
+        SizeT<"service-max-value-rows">{100},
+        Bool<"use-greedy-planning">{false}};
   }();
   return params;
 }