From 41d9205099a69381261b5b6b031774d0d13402b8 Mon Sep 17 00:00:00 2001 From: minghong Date: Mon, 13 Jan 2025 11:47:01 +0800 Subject: [PATCH] move 3 topn rules from rbo to cbo --- .../main/java/org/apache/doris/nereids/rules/RuleSet.java | 6 ++++++ .../rules/rewrite/PushDownLimitDistinctThroughJoin.java | 5 +++++ .../rules/rewrite/PushDownTopNDistinctThroughJoin.java | 5 +++++ .../nereids/rules/rewrite/PushDownTopNThroughJoin.java | 6 +++++- 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index bcd12ac17d2579..e86727a25cde76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -107,7 +107,10 @@ import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughSort; import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughWindow; import org.apache.doris.nereids.rules.rewrite.PushDownJoinOtherCondition; +import org.apache.doris.nereids.rules.rewrite.PushDownLimitDistinctThroughJoin; import org.apache.doris.nereids.rules.rewrite.PushDownProjectThroughLimit; +import org.apache.doris.nereids.rules.rewrite.PushDownTopNDistinctThroughJoin; +import org.apache.doris.nereids.rules.rewrite.PushDownTopNThroughJoin; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; @@ -132,6 +135,9 @@ public class RuleSet { .add(PushDownProjectThroughInnerOuterJoin.INSTANCE) .add(PushDownProjectThroughSemiJoin.INSTANCE) .add(TransposeAggSemiJoinProject.INSTANCE) + .addAll(new PushDownTopNThroughJoin().buildRules()) + .addAll(new PushDownLimitDistinctThroughJoin().buildRules()) + .addAll(new PushDownTopNDistinctThroughJoin().buildRules()) .build(); public static final List PUSH_DOWN_FILTERS = ImmutableList.of( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoin.java index 21f777204b2a0e..590e20a776366a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoin.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; @@ -42,6 +43,10 @@ public List buildRules() { // limit -> distinct -> join logicalLimit(logicalAggregate(logicalJoin()) .when(LogicalAggregate::isDistinct)) + .when(limit -> + ConnectContext.get() != null + && ConnectContext.get().getSessionVariable().topnOptLimitThreshold + >= limit.getLimit() + limit.getOffset()) .then(limit -> { LogicalAggregate> agg = limit.child(); LogicalJoin join = agg.child(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNDistinctThroughJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNDistinctThroughJoin.java index f2dde7ba2a8be5..6c280146c8ab2c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNDistinctThroughJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNDistinctThroughJoin.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; import org.apache.doris.nereids.util.PlanUtils; +import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; @@ -45,6 +46,10 @@ public List buildRules() { // topN -> join logicalTopN(logicalAggregate(logicalJoin()).when(LogicalAggregate::isDistinct)) // TODO: complex order by + .when(topn -> + ConnectContext.get() != null + && ConnectContext.get().getSessionVariable().topnOptLimitThreshold + >= topn.getLimit() + topn.getOffset()) .when(topN -> topN.getOrderKeys().stream().map(OrderKey::getExpr) .allMatch(Slot.class::isInstance)) .then(topN -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoin.java index 28a7f2688be8cd..4d15dae59e76da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoin.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; +import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; @@ -43,6 +44,10 @@ public List buildRules() { // topN -> join logicalTopN(logicalJoin()) // TODO: complex orderby + .when(topn -> + ConnectContext.get() != null + && ConnectContext.get().getSessionVariable().topnOptLimitThreshold + >= topn.getLimit() + topn.getOffset()) .when(topN -> topN.getOrderKeys().stream().map(OrderKey::getExpr) .allMatch(Slot.class::isInstance)) .then(topN -> { @@ -102,7 +107,6 @@ private Plan pushLimitThroughJoin(LogicalTopN topN, LogicalJoin< } return null; case CROSS_JOIN: - if (join.left().getOutputSet().containsAll(orderbySlots)) { return join.withChildren( topN.withLimitChild(topN.getLimit() + topN.getOffset(), 0, join.left()),