From c538a68f28f1c9ab10b59037381b2ceffda1f37d Mon Sep 17 00:00:00 2001 From: Chengpeng Yan <41809508+Reminiscent@users.noreply.github.com> Date: Mon, 20 Feb 2023 22:09:05 +0800 Subject: [PATCH 1/2] This is an automated cherry-pick of #41440 Signed-off-by: ti-chi-bot --- planner/core/exhaust_physical_plans.go | 3 + planner/core/integration_test.go | 18 ++++ planner/core/logical_plan_builder.go | 119 +++++++++++++++++++++++-- planner/core/logical_plans.go | 30 +++++-- sessionctx/variable/session.go | 3 + sessionctx/variable/sysvar.go | 4 + sessionctx/variable/tidb_vars.go | 8 ++ 7 files changed, 171 insertions(+), 14 deletions(-) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 27867f444331d..5f2020a521693 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -2010,6 +2010,9 @@ func (p *LogicalJoin) canPushToCop(storeTp kv.StoreType) bool { // If the hint is not matched, it will get other candidates. // If the hint is not figured, we will pick all candidates. func (p *LogicalJoin) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool, error) { + if p.ctx.GetSessionVars().EnableAdvancedJoinHint { + p.setPreferredJoinType4PhysicalOp() + } failpoint.Inject("MockOnlyEnableIndexHashJoin", func(val failpoint.Value) { if val.(bool) && !p.ctx.GetSessionVars().InRestrictedSQL { indexJoins, _ := p.tryToGetIndexJoin(prop) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 117090b897107..9428b7c43a28f 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -1373,6 +1373,24 @@ func TestKeepOrderHint(t *testing.T) { } } +func TestSplitJoinHint(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec(`create table t(a int, b int, index idx_a(a), index idx_b(b));`) + + tk.MustExec(`set @@tidb_opt_advanced_join_hint=0`) + tk.MustExec("select /*+ hash_join(t1) merge_join(t2) */ * from t t1 join t t2 join t t3 where t1.a = t2.a and t2.a=t3.a") + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1815 Join hints are conflict, you can only specify one type of join")) + + tk.MustExec(`set @@tidb_opt_advanced_join_hint=1`) + tk.MustExec("select /*+ hash_join(t1) merge_join(t2) */ * from t t1 join t t2 join t t3 where t1.a = t2.a and t2.a=t3.a") + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1815 Join hints conflict after join reorder phase, you can only specify one type of join")) + + tk.MustExec(`set @@tidb_opt_advanced_join_hint=0`) +} + func TestKeepOrderHintWithBinding(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 5e5e2edc01b0e..352564b6292a4 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -591,49 +591,89 @@ func (p *LogicalJoin) setPreferredJoinTypeAndOrder(hintInfo *tableHintInfo) { lhsAlias := extractTableAlias(p.children[0], p.blockOffset) rhsAlias := extractTableAlias(p.children[1], p.blockOffset) - if hintInfo.ifPreferMergeJoin(lhsAlias, rhsAlias) { + if hintInfo.ifPreferMergeJoin(lhsAlias) { p.preferJoinType |= preferMergeJoin + p.leftPreferJoinType |= preferMergeJoin } - if hintInfo.ifPreferBroadcastJoin(lhsAlias, rhsAlias) { + if hintInfo.ifPreferMergeJoin(rhsAlias) { + p.preferJoinType |= preferMergeJoin + p.rightPreferJoinType |= preferMergeJoin + } + if hintInfo.ifPreferBroadcastJoin(lhsAlias) { p.preferJoinType |= preferBCJoin + p.leftPreferJoinType |= preferBCJoin } - if hintInfo.ifPreferShuffleJoin(lhsAlias, rhsAlias) { + if hintInfo.ifPreferBroadcastJoin(rhsAlias) { + p.preferJoinType |= preferBCJoin + p.rightPreferJoinType |= preferBCJoin + } + if hintInfo.ifPreferShuffleJoin(lhsAlias) { p.preferJoinType |= preferShuffleJoin + p.leftPreferJoinType |= preferShuffleJoin } - if hintInfo.ifPreferHashJoin(lhsAlias, rhsAlias) { + if hintInfo.ifPreferShuffleJoin(rhsAlias) { + p.preferJoinType |= preferShuffleJoin + p.rightPreferJoinType |= preferShuffleJoin + } + if hintInfo.ifPreferHashJoin(lhsAlias) { + p.preferJoinType |= preferHashJoin + p.leftPreferJoinType |= preferHashJoin + } + if hintInfo.ifPreferHashJoin(rhsAlias) { p.preferJoinType |= preferHashJoin + p.rightPreferJoinType |= preferHashJoin } if hintInfo.ifPreferINLJ(lhsAlias) { p.preferJoinType |= preferLeftAsINLJInner + p.leftPreferJoinType |= preferINLJ } if hintInfo.ifPreferINLJ(rhsAlias) { p.preferJoinType |= preferRightAsINLJInner + p.rightPreferJoinType |= preferINLJ } if hintInfo.ifPreferINLHJ(lhsAlias) { p.preferJoinType |= preferLeftAsINLHJInner + p.leftPreferJoinType |= preferINLHJ } if hintInfo.ifPreferINLHJ(rhsAlias) { p.preferJoinType |= preferRightAsINLHJInner + p.rightPreferJoinType |= preferINLHJ } if hintInfo.ifPreferINLMJ(lhsAlias) { p.preferJoinType |= preferLeftAsINLMJInner + p.leftPreferJoinType |= preferINLMJ } if hintInfo.ifPreferINLMJ(rhsAlias) { p.preferJoinType |= preferRightAsINLMJInner + p.rightPreferJoinType |= preferINLMJ } if hintInfo.ifPreferHJBuild(lhsAlias) { p.preferJoinType |= preferLeftAsHJBuild + p.leftPreferJoinType |= preferHJBuild } if hintInfo.ifPreferHJBuild(rhsAlias) { p.preferJoinType |= preferRightAsHJBuild + p.rightPreferJoinType |= preferHJBuild } if hintInfo.ifPreferHJProbe(lhsAlias) { p.preferJoinType |= preferLeftAsHJProbe + p.leftPreferJoinType |= preferHJProbe } if hintInfo.ifPreferHJProbe(rhsAlias) { p.preferJoinType |= preferRightAsHJProbe + p.rightPreferJoinType |= preferHJProbe } - if containDifferentJoinTypes(p.preferJoinType) { + hasConflict := false + if !p.ctx.GetSessionVars().EnableAdvancedJoinHint || p.ctx.GetSessionVars().StmtCtx.StraightJoinOrder { + if containDifferentJoinTypes(p.preferJoinType) { + hasConflict = true + } + } else if p.ctx.GetSessionVars().EnableAdvancedJoinHint { + if containDifferentJoinTypes(p.leftPreferJoinType) || containDifferentJoinTypes(p.rightPreferJoinType) { + hasConflict = true + } + } + if hasConflict { errMsg := "Join hints are conflict, you can only specify one type of join" warning := ErrInternal.GenWithStack(errMsg) p.ctx.GetSessionVars().StmtCtx.AppendWarning(warning) @@ -649,6 +689,75 @@ func (p *LogicalJoin) setPreferredJoinTypeAndOrder(hintInfo *tableHintInfo) { } } +// setPreferredJoinType4PhysicalOp generates hint information for the logicalJoin based on the hint information of its left and right children. +// This information is used for selecting the physical operator. +func (p *LogicalJoin) setPreferredJoinType4PhysicalOp() { + leftHintInfo := p.leftPreferJoinType + rightHintInfo := p.rightPreferJoinType + if leftHintInfo == 0 && rightHintInfo == 0 { + return + } + if leftHintInfo != 0 && rightHintInfo != 0 && leftHintInfo != rightHintInfo { + // The hint information on the left and right child nodes is different. It causes the conflict. + errMsg := "Join hints conflict after join reorder phase, you can only specify one type of join" + warning := ErrInternal.GenWithStack(errMsg) + p.ctx.GetSessionVars().StmtCtx.AppendWarning(warning) + p.preferJoinType = 0 + } else { + if leftHintInfo != 0 { + p.preferJoinType = leftHintInfo + } else { + p.preferJoinType = rightHintInfo + } + preferJoinType := uint(0) + // Some implementations of physical operators are dependent on the direction, + // and adjustments need to be made based on the direction. + switch p.preferJoinType { + case preferINLJ: + if leftHintInfo != 0 { + preferJoinType |= preferLeftAsINLJInner + } + if rightHintInfo != 0 { + preferJoinType |= preferRightAsINLJInner + } + case preferINLHJ: + if leftHintInfo != 0 { + preferJoinType |= preferLeftAsINLHJInner + } + if rightHintInfo != 0 { + preferJoinType |= preferLeftAsINLHJInner + } + case preferINLMJ: + if leftHintInfo != 0 { + preferJoinType |= preferLeftAsINLMJInner + } + if rightHintInfo != 0 { + preferJoinType |= preferLeftAsINLMJInner + } + case preferHJBuild: + if leftHintInfo != 0 { + preferJoinType |= preferLeftAsHJBuild + } + if rightHintInfo != 0 { + preferJoinType |= preferLeftAsHJBuild + } + case preferHJProbe: + if leftHintInfo != 0 { + preferJoinType |= preferLeftAsHJProbe + } + if rightHintInfo != 0 { + preferJoinType |= preferLeftAsHJProbe + } + default: + preferJoinType = p.preferJoinType + } + p.preferJoinType = preferJoinType + } + // Clear information from left and right child nodes to prevent multiple calls to this function. + p.leftPreferJoinType = 0 + p.rightPreferJoinType = 0 +} + func (ds *DataSource) setPreferredStoreType(hintInfo *tableHintInfo) { if hintInfo == nil { return diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index 8a9cfcf62e41c..6c1832ad9d3a0 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -110,21 +110,31 @@ func (tp JoinType) String() string { } const ( - preferLeftAsINLJInner uint = 1 << iota + // Hint flag for join + preferINLJ uint = 1 << iota + preferINLHJ + preferINLMJ + preferHJBuild + preferHJProbe + preferHashJoin + preferMergeJoin + preferBCJoin + preferShuffleJoin + preferRewriteSemiJoin + + // Hint flag to specify the join with direction + preferLeftAsINLJInner preferRightAsINLJInner preferLeftAsINLHJInner preferRightAsINLHJInner preferLeftAsINLMJInner preferRightAsINLMJInner - preferHashJoin preferLeftAsHJBuild preferRightAsHJBuild preferLeftAsHJProbe preferRightAsHJProbe - preferMergeJoin - preferBCJoin - preferShuffleJoin - preferRewriteSemiJoin + + // Hint flag for Agg preferHashAgg preferStreamAgg preferMPP1PhaseAgg @@ -146,9 +156,11 @@ type LogicalJoin struct { StraightJoin bool // hintInfo stores the join algorithm hint information specified by client. - hintInfo *tableHintInfo - preferJoinType uint - preferJoinOrder bool + hintInfo *tableHintInfo + preferJoinType uint + preferJoinOrder bool + leftPreferJoinType uint + rightPreferJoinType uint EqualConditions []*expression.ScalarFunction NAEQConditions []*expression.ScalarFunction diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 16f630cde85ef..5b305b7b53b32 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -1313,6 +1313,9 @@ type SessionVars struct { // EnableReuseCheck indicates request chunk whether use chunk alloc EnableReuseCheck bool + // EnableAdvancedJoinHint indicates whether the join method hint is compatible with join order hint. + EnableAdvancedJoinHint bool + // preuseChunkAlloc indicates whether pre statement use chunk alloc // like select @@last_sql_use_alloc preUseChunkAlloc bool diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 3049f6f40c9f3..c095d7e8a3d1c 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -2102,6 +2102,10 @@ var defaultSysVars = []*SysVar{ s.RangeMaxSize = TidbOptInt64(val, DefTiDBOptRangeMaxSize) return nil }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptAdvancedJoinHint, Value: BoolToOnOff(DefTiDBOptAdvancedJoinHint), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + s.EnableAdvancedJoinHint = TiDBOptOn(val) + return nil + }}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBAnalyzePartitionConcurrency, Value: strconv.FormatInt(DefTiDBAnalyzePartitionConcurrency, 10), MinValue: 1, MaxValue: uint64(config.GetGlobalConfig().Performance.AnalyzePartitionConcurrencyQuota), SetSession: func(s *SessionVars, val string) error { s.AnalyzePartitionConcurrency = int(TidbOptInt64(val, DefTiDBAnalyzePartitionConcurrency)) diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 7c62b07abe10f..323b735d9f955 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -773,6 +773,9 @@ const ( // limit for ranges. TiDBOptRangeMaxSize = "tidb_opt_range_max_size" + // TiDBOptAdvancedJoinHint indicates whether the join method hint is compatible with join order hint. + TiDBOptAdvancedJoinHint = "tidb_opt_advanced_join_hint" + // TiDBAnalyzePartitionConcurrency indicates concurrency for save/read partitions stats in Analyze TiDBAnalyzePartitionConcurrency = "tidb_analyze_partition_concurrency" // TiDBMergePartitionStatsConcurrency indicates the concurrency when merge partition stats into global stats @@ -1130,7 +1133,12 @@ const ( DefTiDBAutoBuildStatsConcurrency = 1 DefTiDBSysProcScanConcurrency = 1 DefTiDBRcWriteCheckTs = false +<<<<<<< HEAD DefTiDBForeignKeyChecks = false +======= + DefTiDBForeignKeyChecks = true + DefTiDBOptAdvancedJoinHint = false +>>>>>>> fca20d64da1 (planner: distinguish the source of join hint information (#41440)) DefTiDBAnalyzePartitionConcurrency = 1 DefTiDBOptRangeMaxSize = 64 * int64(size.MB) // 64 MB DefTiDBCostModelVer = 2 From c0ac23eaf7064eedecfe31b2bc2d122fc3bcda87 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Fri, 28 Jul 2023 12:40:46 +0800 Subject: [PATCH 2/2] fixup --- sessionctx/variable/tidb_vars.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 323b735d9f955..086f32a685839 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -1133,12 +1133,8 @@ const ( DefTiDBAutoBuildStatsConcurrency = 1 DefTiDBSysProcScanConcurrency = 1 DefTiDBRcWriteCheckTs = false -<<<<<<< HEAD DefTiDBForeignKeyChecks = false -======= - DefTiDBForeignKeyChecks = true DefTiDBOptAdvancedJoinHint = false ->>>>>>> fca20d64da1 (planner: distinguish the source of join hint information (#41440)) DefTiDBAnalyzePartitionConcurrency = 1 DefTiDBOptRangeMaxSize = 64 * int64(size.MB) // 64 MB DefTiDBCostModelVer = 2