From 59ce83d2f2f2e933cdde56c2c7d6be42c451a95e Mon Sep 17 00:00:00 2001 From: tangenta Date: Mon, 23 Dec 2024 13:23:09 +0800 Subject: [PATCH] This is an automated cherry-pick of #58408 Signed-off-by: ti-chi-bot --- pkg/ddl/index_presplit.go | 430 ++++++++++++++++++ pkg/parser/ast/ddl.go | 27 ++ pkg/parser/ast/ddl_test.go | 91 ++++ pkg/parser/tidb/features.go | 12 + .../addindextest3/functional_test.go | 262 +++++++++++ 5 files changed, 822 insertions(+) create mode 100644 pkg/ddl/index_presplit.go create mode 100644 tests/realtikvtest/addindextest3/functional_test.go diff --git a/pkg/ddl/index_presplit.go b/pkg/ddl/index_presplit.go new file mode 100644 index 0000000000000..78d2412d1c32d --- /dev/null +++ b/pkg/ddl/index_presplit.go @@ -0,0 +1,430 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ddl + +import ( + "bytes" + "context" + "fmt" + "math" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/failpoint" + "github.com/pingcap/tidb/pkg/ddl/logutil" + "github.com/pingcap/tidb/pkg/expression" + "github.com/pingcap/tidb/pkg/expression/exprctx" + "github.com/pingcap/tidb/pkg/kv" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/pingcap/tidb/pkg/sessionctx" + "github.com/pingcap/tidb/pkg/table/tables" + "github.com/pingcap/tidb/pkg/tablecodec" + "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util" + "github.com/pingcap/tidb/pkg/util/chunk" + contextutil "github.com/pingcap/tidb/pkg/util/context" + "github.com/pingcap/tidb/pkg/util/dbterror/exeerrors" + "github.com/pingcap/tidb/pkg/util/dbterror/plannererrors" + "go.uber.org/zap" +) + +func preSplitIndexRegions( + ctx context.Context, + sctx sessionctx.Context, + store kv.Storage, + tblInfo *model.TableInfo, + allIndexInfos []*model.IndexInfo, + reorgMeta *model.DDLReorgMeta, + args *model.ModifyIndexArgs, +) error { + warnHandler := contextutil.NewStaticWarnHandler(0) + exprCtx, err := newReorgExprCtxWithReorgMeta(reorgMeta, warnHandler) + if err != nil { + return errors.Trace(err) + } + splitOnTempIdx := reorgMeta.ReorgTp == model.ReorgTypeLitMerge || + reorgMeta.ReorgTp == model.ReorgTypeTxnMerge + for i, idxInfo := range allIndexInfos { + idxArg := args.IndexArgs[i] + splitArgs, err := evalSplitDatumFromArgs(exprCtx, tblInfo, idxInfo, idxArg) + if err != nil { + return errors.Trace(err) + } + if splitArgs == nil { + continue + } + splitKeys, err := getSplitIdxKeys(sctx, tblInfo, idxInfo, splitArgs) + if err != nil { + return errors.Trace(err) + } + if splitOnTempIdx { + for i := range splitKeys { + tablecodec.IndexKey2TempIndexKey(splitKeys[i]) + } + } + failpoint.InjectCall("beforePresplitIndex", splitKeys) + err = splitIndexRegionAndWait(ctx, sctx, store, tblInfo, idxInfo, splitKeys) + if err != nil { + return errors.Trace(err) + } + } + return nil +} + +type splitArgs struct { + byRows [][]types.Datum + + betweenLower []types.Datum + betweenUpper []types.Datum + regionsCnt int +} + +func getSplitIdxKeys( + sctx sessionctx.Context, + tblInfo *model.TableInfo, + idxInfo *model.IndexInfo, + args *splitArgs, +) ([][]byte, error) { + // Split index regions by user specified value lists. + if len(args.byRows) > 0 { + return getSplitIdxKeysFromValueList(sctx, tblInfo, idxInfo, args.byRows) + } + + return getSplitIdxKeysFromBound( + sctx, tblInfo, idxInfo, args.betweenLower, args.betweenUpper, args.regionsCnt) +} + +func getSplitIdxKeysFromValueList( + sctx sessionctx.Context, + tblInfo *model.TableInfo, + idxInfo *model.IndexInfo, + byRows [][]types.Datum, +) (destKeys [][]byte, err error) { + pi := tblInfo.GetPartitionInfo() + if pi == nil { + destKeys = make([][]byte, 0, len(byRows)+1) + return getSplitIdxPhysicalKeysFromValueList(sctx, tblInfo, idxInfo, tblInfo.ID, byRows, destKeys) + } + + if idxInfo.Global { + destKeys = make([][]byte, 0, len(byRows)+1) + return getSplitIdxPhysicalKeysFromValueList(sctx, tblInfo, idxInfo, tblInfo.ID, byRows, destKeys) + } + + destKeys = make([][]byte, 0, (len(byRows)+1)*len(pi.Definitions)) + for _, p := range pi.Definitions { + destKeys, err = getSplitIdxPhysicalKeysFromValueList(sctx, tblInfo, idxInfo, p.ID, byRows, destKeys) + if err != nil { + return nil, err + } + } + return destKeys, nil +} + +func getSplitIdxPhysicalKeysFromValueList( + sctx sessionctx.Context, + tblInfo *model.TableInfo, + idxInfo *model.IndexInfo, + physicalID int64, + splitDatum [][]types.Datum, + destKeys [][]byte, +) ([][]byte, error) { + destKeys = getSplitIdxPhysicalStartAndOtherIdxKeys(tblInfo, idxInfo, physicalID, destKeys) + index := tables.NewIndex(physicalID, tblInfo, idxInfo) + sc := sctx.GetSessionVars().StmtCtx + for _, v := range splitDatum { + idxKey, _, err := index.GenIndexKey(sc.ErrCtx(), sc.TimeZone(), v, kv.IntHandle(math.MinInt64), nil) + if err != nil { + return nil, err + } + destKeys = append(destKeys, idxKey) + } + return destKeys, nil +} + +func getSplitIdxPhysicalStartAndOtherIdxKeys( + tblInfo *model.TableInfo, + idxInfo *model.IndexInfo, + physicalID int64, + keys [][]byte, +) [][]byte { + // 1. Split in the start key for the index if the index is not the first index. + // For the first index, splitting the start key can produce the region [tid, tid_i_1), which is useless. + if len(tblInfo.Indices) > 0 && tblInfo.Indices[0].ID != idxInfo.ID { + startKey := tablecodec.EncodeTableIndexPrefix(physicalID, idxInfo.ID) + keys = append(keys, startKey) + } + + // 2. Split in the end key. + endKey := tablecodec.EncodeTableIndexPrefix(physicalID, idxInfo.ID+1) + keys = append(keys, endKey) + return keys +} + +func getSplitIdxKeysFromBound( + sctx sessionctx.Context, + tblInfo *model.TableInfo, + idxInfo *model.IndexInfo, + lower, upper []types.Datum, + splitNum int, +) (keys [][]byte, err error) { + pi := tblInfo.GetPartitionInfo() + if pi == nil { + keys = make([][]byte, 0, splitNum) + return getSplitIdxPhysicalKeysFromBound( + sctx, tblInfo, idxInfo, tblInfo.ID, lower, upper, splitNum, keys) + } + keys = make([][]byte, 0, splitNum*len(pi.Definitions)) + for _, p := range pi.Definitions { + keys, err = getSplitIdxPhysicalKeysFromBound( + sctx, tblInfo, idxInfo, p.ID, lower, upper, splitNum, keys) + if err != nil { + return nil, err + } + } + return keys, nil +} + +func getSplitIdxPhysicalKeysFromBound( + sctx sessionctx.Context, + tblInfo *model.TableInfo, + idxInfo *model.IndexInfo, + physicalID int64, + lower, upper []types.Datum, + splitNum int, + destKeys [][]byte, +) ([][]byte, error) { + destKeys = getSplitIdxPhysicalStartAndOtherIdxKeys(tblInfo, idxInfo, physicalID, destKeys) + index := tables.NewIndex(physicalID, tblInfo, idxInfo) + // Split index regions by lower, upper value and calculate the step by (upper - lower)/num. + sc := sctx.GetSessionVars().StmtCtx + lowerIdxKey, _, err := index.GenIndexKey(sc.ErrCtx(), sc.TimeZone(), lower, kv.IntHandle(math.MinInt64), nil) + if err != nil { + return nil, err + } + // Use math.MinInt64 as handle_id for the upper index key to avoid affecting calculate split point. + // If use math.MaxInt64 here, test of `TestSplitIndex` will report error. + upperIdxKey, _, err := index.GenIndexKey(sc.ErrCtx(), sc.TimeZone(), upper, kv.IntHandle(math.MinInt64), nil) + if err != nil { + return nil, err + } + + if bytes.Compare(lowerIdxKey, upperIdxKey) >= 0 { + lowerStr := datumSliceToString(lower) + upperStr := datumSliceToString(upper) + errMsg := fmt.Sprintf("Split index `%v` region lower value %v should less than the upper value %v", + idxInfo.Name, lowerStr, upperStr) + return nil, exeerrors.ErrInvalidSplitRegionRanges.GenWithStackByArgs(errMsg) + } + return util.GetValuesList(lowerIdxKey, upperIdxKey, splitNum, destKeys), nil +} + +func datumSliceToString(ds []types.Datum) string { + str := "(" + for i, d := range ds { + s, err := d.ToString() + if err != nil { + return fmt.Sprintf("%v", ds) + } + if i > 0 { + str += "," + } + str += s + } + str += ")" + return str +} + +func splitIndexRegionAndWait( + ctx context.Context, + sctx sessionctx.Context, + store kv.Storage, + tblInfo *model.TableInfo, + idxInfo *model.IndexInfo, + splitIdxKeys [][]byte, +) error { + s, ok := store.(kv.SplittableStore) + if !ok { + return nil + } + start := time.Now() + ctxWithTimeout, cancel := context.WithTimeout(ctx, sctx.GetSessionVars().GetSplitRegionTimeout()) + defer cancel() + regionIDs, err := s.SplitRegions(ctxWithTimeout, splitIdxKeys, true, &tblInfo.ID) + if err != nil { + logutil.DDLLogger().Error("split table index region failed", + zap.String("table", tblInfo.Name.L), + zap.String("index", tblInfo.Name.L), + zap.Error(err)) + return err + } + failpoint.Inject("mockSplitIndexRegionAndWaitErr", func(_ failpoint.Value) { + failpoint.Return(context.DeadlineExceeded) + }) + finishScatterRegions := waitScatterRegionFinish(ctxWithTimeout, sctx, start, s, regionIDs, tblInfo.Name.L, idxInfo.Name.L) + logutil.DDLLogger().Info("split table index region finished", + zap.String("table", tblInfo.Name.L), + zap.String("index", idxInfo.Name.L), + zap.Int("splitRegions", len(regionIDs)), + zap.Int("scatterRegions", finishScatterRegions), + ) + return nil +} + +func evalSplitDatumFromArgs( + buildCtx exprctx.BuildContext, + tblInfo *model.TableInfo, + idxInfo *model.IndexInfo, + idxArg *model.IndexArg, +) (*splitArgs, error) { + opt := idxArg.SplitOpt + if opt == nil { + return nil, nil + } + if len(opt.ValueLists) > 0 { + indexValues := make([][]types.Datum, 0, len(opt.ValueLists)) + for i, valueList := range opt.ValueLists { + if len(valueList) > len(idxInfo.Columns) { + return nil, plannererrors.ErrWrongValueCountOnRow.GenWithStackByArgs(i + 1) + } + values, err := evalConstExprNodes(buildCtx, valueList, tblInfo, idxInfo) + if err != nil { + return nil, err + } + indexValues = append(indexValues, values) + } + return &splitArgs{byRows: indexValues}, nil + } + + if len(opt.Lower) == 0 && len(opt.Upper) == 0 && opt.Num > 0 { + lowerVals := make([]types.Datum, 0, len(idxInfo.Columns)) + upperVals := make([]types.Datum, 0, len(idxInfo.Columns)) + for i := 0; i < len(idxInfo.Columns); i++ { + lowerVals = append(lowerVals, types.MinNotNullDatum()) + upperVals = append(upperVals, types.MaxValueDatum()) + } + return &splitArgs{ + betweenLower: lowerVals, + betweenUpper: upperVals, + regionsCnt: int(opt.Num), + }, nil + } + + // Split index regions by lower, upper value. + checkLowerUpperValue := func(valuesItem []string, name string) ([]types.Datum, error) { + if len(valuesItem) == 0 { + return nil, errors.Errorf("Split index `%v` region %s value count should be greater than 0", idxInfo.Name, name) + } + if len(valuesItem) > len(idxInfo.Columns) { + return nil, errors.Errorf("Split index `%v` region column count doesn't match value count at %v", idxInfo.Name, name) + } + return evalConstExprNodes(buildCtx, valuesItem, tblInfo, idxInfo) + } + lowerValues, err := checkLowerUpperValue(opt.Lower, "lower") + if err != nil { + return nil, err + } + upperValues, err := checkLowerUpperValue(opt.Upper, "upper") + if err != nil { + return nil, err + } + splitArgs := &splitArgs{ + betweenLower: lowerValues, + betweenUpper: upperValues, + } + splitArgs.regionsCnt = int(opt.Num) + return splitArgs, nil +} + +func evalConstExprNodes( + buildCtx exprctx.BuildContext, + valueList []string, + tblInfo *model.TableInfo, + idxInfo *model.IndexInfo, +) ([]types.Datum, error) { + values := make([]types.Datum, 0, len(valueList)) + for j, value := range valueList { + colOffset := idxInfo.Columns[j].Offset + col := tblInfo.Columns[colOffset] + exp, err := expression.ParseSimpleExpr(buildCtx, value) + if err != nil { + return nil, err + } + evalCtx := buildCtx.GetEvalCtx() + evaluatedVal, err := exp.Eval(evalCtx, chunk.Row{}) + if err != nil { + return nil, err + } + + d, err := evaluatedVal.ConvertTo(evalCtx.TypeCtx(), &col.FieldType) + if err != nil { + if !types.ErrTruncated.Equal(err) && + !types.ErrTruncatedWrongVal.Equal(err) && + !types.ErrBadNumber.Equal(err) { + return nil, err + } + valStr, err1 := evaluatedVal.ToString() + if err1 != nil { + return nil, err + } + return nil, types.ErrTruncated.GenWithStack("Incorrect value: '%-.128s' for column '%.192s'", valStr, col.Name.O) + } + values = append(values, d) + } + return values, nil +} + +func waitScatterRegionFinish( + ctxWithTimeout context.Context, + sctx sessionctx.Context, + startTime time.Time, + store kv.SplittableStore, + regionIDs []uint64, + tableName, indexName string, +) int { + remainMillisecond := 0 + finishScatterNum := 0 + for _, regionID := range regionIDs { + select { + case <-ctxWithTimeout.Done(): + // Do not break here for checking remain regions scatter finished with a very short backoff time. + // Consider this situation - Regions 1, 2, and 3 are to be split. + // Region 1 times out before scattering finishes, while Region 2 and Region 3 have finished scattering. + // In this case, we should return 2 Regions, instead of 0, have finished scattering. + remainMillisecond = 50 + default: + remainMillisecond = int((sctx.GetSessionVars().GetSplitRegionTimeout().Seconds() - time.Since(startTime).Seconds()) * 1000) + } + + err := store.WaitScatterRegionFinish(ctxWithTimeout, regionID, remainMillisecond) + if err == nil { + finishScatterNum++ + } else { + if len(indexName) == 0 { + logutil.DDLLogger().Warn("wait scatter region failed", + zap.Uint64("regionID", regionID), + zap.String("table", tableName), + zap.Error(err)) + } else { + logutil.DDLLogger().Warn("wait scatter region failed", + zap.Uint64("regionID", regionID), + zap.String("table", tableName), + zap.String("index", indexName), + zap.Error(err)) + } + } + } + return finishScatterNum +} diff --git a/pkg/parser/ast/ddl.go b/pkg/parser/ast/ddl.go index 00e7e6c9deb0a..46d09c0467ab0 100644 --- a/pkg/parser/ast/ddl.go +++ b/pkg/parser/ast/ddl.go @@ -783,6 +783,33 @@ func (n *IndexOption) Restore(ctx *format.RestoreCtx) error { case IndexVisibilityInvisible: ctx.WriteKeyWord("INVISIBLE") } +<<<<<<< HEAD +======= + hasPrevOption = true + } + + if n.SplitOpt != nil { + if hasPrevOption { + ctx.WritePlain(" ") + } + err := ctx.WriteWithSpecialComments(tidb.FeatureIDPresplit, func() error { + ctx.WriteKeyWord("PRE_SPLIT_REGIONS") + ctx.WritePlain(" = ") + if n.SplitOpt.Num != 0 && len(n.SplitOpt.Lower) == 0 { + ctx.WritePlainf("%d", n.SplitOpt.Num) + } else { + ctx.WritePlain("(") + if err := n.SplitOpt.Restore(ctx); err != nil { + return errors.Annotate(err, "An error occurred while splicing IndexOption SplitOpt") + } + ctx.WritePlain(")") + } + return nil + }) + if err != nil { + return err + } +>>>>>>> 3735ed55a39 (parser: support pre-split global index add special comment support for pre_split index option (#58408)) } return nil } diff --git a/pkg/parser/ast/ddl_test.go b/pkg/parser/ast/ddl_test.go index 9a67c9b796f06..3b5a6bcf7f40c 100644 --- a/pkg/parser/ast/ddl_test.go +++ b/pkg/parser/ast/ddl_test.go @@ -918,3 +918,94 @@ func TestTableOptionTTLRestoreWithTTLEnableOffFlag(t *testing.T) { runNodeRestoreTestWithFlagsStmtChange(t, testCases, "%s", extractNodeFunc, ca.flags) } } +<<<<<<< HEAD +======= + +func TestPresplitIndexSpecialComments(t *testing.T) { + specialCmtFlag := format.DefaultRestoreFlags | format.RestoreTiDBSpecialComment + cases := []struct { + sourceSQL string + flags format.RestoreFlags + expectSQL string + }{ + {"ALTER TABLE t ADD INDEX (a) PRE_SPLIT_REGIONS = 4", specialCmtFlag, "ALTER TABLE `t` ADD INDEX(`a`) /*T![pre_split] PRE_SPLIT_REGIONS = 4 */"}, + {"ALTER TABLE t ADD INDEX (a) PRE_SPLIT_REGIONS 4", specialCmtFlag, "ALTER TABLE `t` ADD INDEX(`a`) /*T![pre_split] PRE_SPLIT_REGIONS = 4 */"}, + {"ALTER TABLE t ADD PRIMARY KEY (a) CLUSTERED PRE_SPLIT_REGIONS = 4", specialCmtFlag, "ALTER TABLE `t` ADD PRIMARY KEY(`a`) /*T![clustered_index] CLUSTERED */ /*T![pre_split] PRE_SPLIT_REGIONS = 4 */"}, + {"ALTER TABLE t ADD PRIMARY KEY (a) PRE_SPLIT_REGIONS = 4 NONCLUSTERED", specialCmtFlag, "ALTER TABLE `t` ADD PRIMARY KEY(`a`) /*T![clustered_index] NONCLUSTERED */ /*T![pre_split] PRE_SPLIT_REGIONS = 4 */"}, + {"ALTER TABLE t ADD INDEX (a) PRE_SPLIT_REGIONS = (between (1, 'a') and (2, 'b') regions 4);", specialCmtFlag, "ALTER TABLE `t` ADD INDEX(`a`) /*T![pre_split] PRE_SPLIT_REGIONS = (BETWEEN (1,_UTF8MB4'a') AND (2,_UTF8MB4'b') REGIONS 4) */"}, + {"ALTER TABLE t ADD INDEX idx(a) pre_split_regions = 100, ADD INDEX idx2(b) pre_split_regions = (by(1),(2),(3))", specialCmtFlag, "ALTER TABLE `t` ADD INDEX `idx`(`a`) /*T![pre_split] PRE_SPLIT_REGIONS = 100 */, ADD INDEX `idx2`(`b`) /*T![pre_split] PRE_SPLIT_REGIONS = (BY (1),(2),(3)) */"}, + {"ALTER TABLE t ADD INDEX (a) comment 'a' PRE_SPLIT_REGIONS = (between (1, 'a') and (2, 'b') regions 4);", specialCmtFlag, "ALTER TABLE `t` ADD INDEX(`a`) COMMENT 'a' /*T![pre_split] PRE_SPLIT_REGIONS = (BETWEEN (1,_UTF8MB4'a') AND (2,_UTF8MB4'b') REGIONS 4) */"}, + } + + extractNodeFunc := func(node Node) Node { + return node + } + + for _, ca := range cases { + testCases := []NodeRestoreTestCase{ + {ca.sourceSQL, ca.expectSQL}, + } + runNodeRestoreTestWithFlags(t, testCases, "%s", extractNodeFunc, ca.flags) + } +} + +func TestResourceGroupDDLStmtRestore(t *testing.T) { + createTestCases := []NodeRestoreTestCase{ + { + "CREATE RESOURCE GROUP IF NOT EXISTS rg1 RU_PER_SEC = 500 BURSTABLE", + "CREATE RESOURCE GROUP IF NOT EXISTS `rg1` RU_PER_SEC = 500, BURSTABLE = TRUE", + }, + { + "CREATE RESOURCE GROUP IF NOT EXISTS rg2 RU_PER_SEC = 600", + "CREATE RESOURCE GROUP IF NOT EXISTS `rg2` RU_PER_SEC = 600", + }, + { + "CREATE RESOURCE GROUP IF NOT EXISTS rg3 RU_PER_SEC = 100 PRIORITY = HIGH", + "CREATE RESOURCE GROUP IF NOT EXISTS `rg3` RU_PER_SEC = 100, PRIORITY = HIGH", + }, + { + "CREATE RESOURCE GROUP IF NOT EXISTS rg1 RU_PER_SEC = 500 QUERY_LIMIT=(EXEC_ELAPSED='60s', ACTION=COOLDOWN)", + "CREATE RESOURCE GROUP IF NOT EXISTS `rg1` RU_PER_SEC = 500, QUERY_LIMIT = (EXEC_ELAPSED = '60s' ACTION = COOLDOWN)", + }, + { + "CREATE RESOURCE GROUP IF NOT EXISTS rg1 RU_PER_SEC = 500 QUERY_LIMIT=(ACTION=SWITCH_GROUP(rg2))", + "CREATE RESOURCE GROUP IF NOT EXISTS `rg1` RU_PER_SEC = 500, QUERY_LIMIT = (ACTION = SWITCH_GROUP(`rg2`))", + }, + } + extractNodeFunc := func(node Node) Node { + return node.(*CreateResourceGroupStmt) + } + runNodeRestoreTest(t, createTestCases, "%s", extractNodeFunc) + + alterTestCase := []NodeRestoreTestCase{ + { + "ALTER RESOURCE GROUP rg1 QUERY_LIMIT=(EXEC_ELAPSED='60s', ACTION=KILL, WATCH=SIMILAR DURATION='10m')", + "ALTER RESOURCE GROUP `rg1` QUERY_LIMIT = (EXEC_ELAPSED = '60s' ACTION = KILL WATCH = SIMILAR DURATION = '10m')", + }, + { + "ALTER RESOURCE GROUP rg1 QUERY_LIMIT=(EXEC_ELAPSED='1m', ACTION=SWITCH_GROUP(rg2), WATCH=SIMILAR DURATION='10m')", + "ALTER RESOURCE GROUP `rg1` QUERY_LIMIT = (EXEC_ELAPSED = '1m' ACTION = SWITCH_GROUP(`rg2`) WATCH = SIMILAR DURATION = '10m')", + }, + { + "ALTER RESOURCE GROUP rg1 QUERY_LIMIT=NULL", + "ALTER RESOURCE GROUP `rg1` QUERY_LIMIT = NULL", + }, + { + "ALTER RESOURCE GROUP `default` BACKGROUND=(TASK_TYPES='br,ddl')", + "ALTER RESOURCE GROUP `default` BACKGROUND = (TASK_TYPES = 'br,ddl')", + }, + { + "ALTER RESOURCE GROUP `default` BACKGROUND=NULL", + "ALTER RESOURCE GROUP `default` BACKGROUND = NULL", + }, + { + "ALTER RESOURCE GROUP `default` BACKGROUND=(TASK_TYPES='')", + "ALTER RESOURCE GROUP `default` BACKGROUND = (TASK_TYPES = '')", + }, + } + extractNodeFunc = func(node Node) Node { + return node.(*AlterResourceGroupStmt) + } + runNodeRestoreTest(t, alterTestCase, "%s", extractNodeFunc) +} +>>>>>>> 3735ed55a39 (parser: support pre-split global index add special comment support for pre_split index option (#58408)) diff --git a/pkg/parser/tidb/features.go b/pkg/parser/tidb/features.go index 1bdb511d20f35..4b195b5da8cd2 100644 --- a/pkg/parser/tidb/features.go +++ b/pkg/parser/tidb/features.go @@ -32,6 +32,13 @@ const ( FeatureIDTTL = "ttl" // FeatureIDResourceGroup is the `resource group` feature. FeatureIDResourceGroup = "resource_group" +<<<<<<< HEAD +======= + // FeatureIDGlobalIndex is the `Global Index` feature. + FeatureIDGlobalIndex = "global_index" + // FeatureIDPresplit is the pre-split feature. + FeatureIDPresplit = "pre_split" +>>>>>>> 3735ed55a39 (parser: support pre-split global index add special comment support for pre_split index option (#58408)) ) var featureIDs = map[string]struct{}{ @@ -42,6 +49,11 @@ var featureIDs = map[string]struct{}{ FeatureIDForceAutoInc: {}, FeatureIDPlacement: {}, FeatureIDTTL: {}, +<<<<<<< HEAD +======= + FeatureIDGlobalIndex: {}, + FeatureIDPresplit: {}, +>>>>>>> 3735ed55a39 (parser: support pre-split global index add special comment support for pre_split index option (#58408)) } // CanParseFeature is used to check if a feature can be parsed. diff --git a/tests/realtikvtest/addindextest3/functional_test.go b/tests/realtikvtest/addindextest3/functional_test.go new file mode 100644 index 0000000000000..f23817713fd7a --- /dev/null +++ b/tests/realtikvtest/addindextest3/functional_test.go @@ -0,0 +1,262 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package addindextest + +import ( + "bytes" + "context" + "fmt" + "strings" + "sync" + "testing" + + "github.com/pingcap/tidb/pkg/ddl" + "github.com/pingcap/tidb/pkg/ddl/ingest" + "github.com/pingcap/tidb/pkg/meta/model" + pmodel "github.com/pingcap/tidb/pkg/parser/model" + "github.com/pingcap/tidb/pkg/table/tables" + "github.com/pingcap/tidb/pkg/tablecodec" + "github.com/pingcap/tidb/pkg/testkit" + "github.com/pingcap/tidb/pkg/testkit/testfailpoint" + "github.com/pingcap/tidb/tests/realtikvtest" + "github.com/stretchr/testify/require" + "github.com/tikv/client-go/v2/tikv" + "github.com/tikv/client-go/v2/util" +) + +func TestDDLTestEstimateTableRowSize(t *testing.T) { + store, dom := realtikvtest.CreateMockStoreAndDomainAndSetup(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test;") + tk.MustExec("create table t (a int, b int);") + tk.MustExec("insert into t values (1, 1);") + + ctx := context.Background() + ctx = util.WithInternalSourceType(ctx, "estimate_row_size") + tkSess := tk.Session() + exec := tkSess.GetRestrictedSQLExecutor() + tbl, err := dom.InfoSchema().TableByName(context.Background(), pmodel.NewCIStr("test"), pmodel.NewCIStr("t")) + require.NoError(t, err) + + size := ddl.EstimateTableRowSizeForTest(ctx, store, exec, tbl) + require.Equal(t, 0, size) // No data in information_schema.columns. + tk.MustExec("analyze table t all columns;") + size = ddl.EstimateTableRowSizeForTest(ctx, store, exec, tbl) + require.Equal(t, 16, size) + + tk.MustExec("alter table t add column c varchar(255);") + tk.MustExec("update t set c = repeat('a', 50) where a = 1;") + tk.MustExec("analyze table t all columns;") + size = ddl.EstimateTableRowSizeForTest(ctx, store, exec, tbl) + require.Equal(t, 67, size) + + tk.MustExec("drop table t;") + tk.MustExec("create table t (id bigint primary key, b text) partition by hash(id) partitions 4;") + for i := 1; i < 10; i++ { + insertSQL := fmt.Sprintf("insert into t values (%d, repeat('a', 10))", i*10000) + tk.MustExec(insertSQL) + } + tk.MustQuery("split table t between (0) and (1000000) regions 2;").Check(testkit.Rows("4 1")) + tk.MustExec("set global tidb_analyze_skip_column_types=`json,blob,mediumblob,longblob`") + tk.MustExec("analyze table t all columns;") + tbl, err = dom.InfoSchema().TableByName(context.Background(), pmodel.NewCIStr("test"), pmodel.NewCIStr("t")) + require.NoError(t, err) + size = ddl.EstimateTableRowSizeForTest(ctx, store, exec, tbl) + require.Equal(t, 19, size) + ptbl := tbl.GetPartitionedTable() + pids := ptbl.GetAllPartitionIDs() + for _, pid := range pids { + partition := ptbl.GetPartition(pid) + size = ddl.EstimateTableRowSizeForTest(ctx, store, exec, partition) + require.Equal(t, 19, size) + } +} + +func TestBackendCtxConcurrentUnregister(t *testing.T) { + store := realtikvtest.CreateMockStoreAndSetup(t) + discovery := store.(tikv.Storage).GetRegionCache().PDClient().GetServiceDiscovery() + bCtx, err := ingest.LitBackCtxMgr.Register(context.Background(), 1, false, nil, discovery, "test", 1, 0, 0) + require.NoError(t, err) + idxIDs := []int64{1, 2, 3, 4, 5, 6, 7} + uniques := make([]bool, 0, len(idxIDs)) + for range idxIDs { + uniques = append(uniques, false) + } + _, err = bCtx.Register([]int64{1, 2, 3, 4, 5, 6, 7}, uniques, tables.MockTableFromMeta(&model.TableInfo{})) + require.NoError(t, err) + + wg := sync.WaitGroup{} + wg.Add(3) + for i := 0; i < 3; i++ { + go func() { + err := bCtx.FinishAndUnregisterEngines(ingest.OptCloseEngines) + require.NoError(t, err) + wg.Done() + }() + } + wg.Wait() + ingest.LitBackCtxMgr.Unregister(1) +} + +func TestMockMemoryUsedUp(t *testing.T) { + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ddl/ingest/setMemTotalInMB", "return(100)") + store := realtikvtest.CreateMockStoreAndSetup(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test;") + tk.MustExec("create table t (c int, c2 int, c3 int, c4 int);") + tk.MustExec("insert into t values (1,1,1,1), (2,2,2,2), (3,3,3,3);") + tk.MustGetErrMsg("alter table t add index i(c), add index i2(c2);", "[ddl:8247]Ingest failed: memory used up") +} + +func TestTiDBEncodeKeyTempIndexKey(t *testing.T) { + store := realtikvtest.CreateMockStoreAndSetup(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table t (a int primary key, b int);") + tk.MustExec("insert into t values (1, 1);") + runDML := false + testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/beforeRunOneJobStep", func(job *model.Job) { + if !runDML && job.Type == model.ActionAddIndex && job.SchemaState == model.StateWriteOnly { + tk2 := testkit.NewTestKit(t, store) + tk2.MustExec("use test") + tk2.MustExec("insert into t values (2, 2);") + runDML = true + } + }) + tk.MustExec("create index idx on t(b);") + require.True(t, runDML) + + rows := tk.MustQuery("select tidb_mvcc_info(tidb_encode_index_key('test', 't', 'idx', 1, 1));").Rows() + rs := rows[0][0].(string) + require.Equal(t, 1, strings.Count(rs, "writes"), rs) + rows = tk.MustQuery("select tidb_mvcc_info(tidb_encode_index_key('test', 't', 'idx', 2, 2));").Rows() + rs = rows[0][0].(string) + require.Equal(t, 2, strings.Count(rs, "writes"), rs) +} + +func TestAddIndexPresplitIndexRegions(t *testing.T) { + store := realtikvtest.CreateMockStoreAndSetup(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + + var splitKeyHex [][]byte + testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/beforePresplitIndex", func(splitKeys [][]byte) { + for _, k := range splitKeys { + splitKeyHex = append(splitKeyHex, bytes.Clone(k)) + } + }) + checkSplitKeys := func(idxID int64, count int, reset bool) { + cnt := 0 + for _, k := range splitKeyHex { + indexID, err := tablecodec.DecodeIndexID(k) + if err == nil && indexID == idxID { + cnt++ + } + } + require.Equal(t, count, cnt, splitKeyHex) + if reset { + splitKeyHex = nil + } + } + var idxID int64 + nextIdxID := func() int64 { + idxID++ + return idxID + } + resetIdxID := func() { + idxID = 0 + } + + tk.MustExec("create table t (a int primary key, b int);") + for i := 0; i < 10; i++ { + insertSQL := fmt.Sprintf("insert into t values (%[1]d, %[1]d);", 10000*i) + tk.MustExec(insertSQL) + } + retRows := tk.MustQuery("show table t regions;").Rows() + require.Len(t, retRows, 1) + tk.MustExec("set @@global.tidb_ddl_enable_fast_reorg = off;") + tk.MustExec("set @@global.tidb_enable_dist_task = off;") + tk.MustExec("alter table t add index idx(b) pre_split_regions = 4;") + checkSplitKeys(nextIdxID(), 3, true) + tk.MustExec("drop index idx on t;") + tk.MustExec("alter table t add index idx(b) pre_split_regions = (by (10000), (20000), (30000));") + checkSplitKeys(nextIdxID(), 3, true) + tk.MustExec("drop index idx on t;") + tk.MustExec("alter table t add index idx(b) /*T![pre_split] pre_split_regions = (by (10000), (20000), (30000)) */;") + checkSplitKeys(nextIdxID(), 3, true) + tk.MustExec("drop index idx on t;") + tk.MustExec("alter table t add index idx(b) pre_split_regions = (between (0) and (10 * 10000) regions 3);") + checkSplitKeys(nextIdxID(), 2, true) + tk.MustExec("drop index idx on t;") + tk.MustExec("set @@global.tidb_ddl_enable_fast_reorg = on;") + + tk.MustExec("alter table t add index idx(b) pre_split_regions = (by (10000), (20000), (30000));") + nextID := nextIdxID() + checkSplitKeys(nextID, 0, false) + checkSplitKeys(tablecodec.TempIndexPrefix|nextID, 3, true) + + tk.MustExec("set @@global.tidb_ddl_enable_fast_reorg = off;") + + // Test partition tables. + resetIdxID() + tk.MustExec("drop table t;") + tk.MustExec("create table t (a int primary key, b int) partition by hash(a) partitions 4;") + for i := 0; i < 10; i++ { + insertSQL := fmt.Sprintf("insert into t values (%[1]d, %[1]d);", 10000*i) + tk.MustExec(insertSQL) + } + tk.MustExec("alter table t add index idx(b) pre_split_regions = (by (10000), (20000), (30000));") + checkSplitKeys(nextIdxID(), 3*4, true) + tk.MustExec("drop index idx on t;") + tk.MustExec("alter table t add index idx(b) pre_split_regions = (between (0) and (10 * 10000) regions 3);") + checkSplitKeys(nextIdxID(), 2*4, true) + tk.MustExec("drop index idx on t;") + tk.MustExec("set @@global.tidb_ddl_enable_fast_reorg = on;") + tk.MustExec("alter table t add index idx(b) pre_split_regions = (by (10000), (20000), (30000));") + checkSplitKeys(nextIdxID(), 0, false) + checkSplitKeys(tablecodec.TempIndexPrefix|3, 12, true) + tk.MustExec("drop index idx on t;") + tk.MustExec("set @@global.tidb_ddl_enable_fast_reorg = off;") + + resetIdxID() + tk.MustExec("drop table t;") + tk.MustExec("set @@global.tidb_ddl_enable_fast_reorg = on;") + tk.MustExec("set @@global.tidb_enable_dist_task = off;") + tk.MustExec("create table t (a int, b int) partition by range (b)" + + " (partition p0 values less than (10), " + + " partition p1 values less than (maxvalue));") + tk.MustExec("alter table t add unique index p_a (a) global pre_split_regions = (by (5), (15));") + checkSplitKeys(tablecodec.TempIndexPrefix|nextIdxID(), 2, true) +} + +func TestAddIndexPresplitFunctional(t *testing.T) { + store := realtikvtest.CreateMockStoreAndSetup(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table t (a int primary key, b int);") + + tk.MustGetErrMsg("alter table t add index idx(b) pre_split_regions = (between (0) and (10 * 10000) regions 0);", + "Split index region num should be greater than 0") + tk.MustGetErrMsg("alter table t add index idx(b) pre_split_regions = (between (0) and (10 * 10000) regions 10000);", + "Split index region num exceeded the limit 1000") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ddl/mockSplitIndexRegionAndWaitErr", "2*return") + tk.MustExec("alter table t add index idx(b) pre_split_regions = (between (0) and (10 * 10000) regions 3);") + + tk.MustExec("drop table t;") + tk.MustExec("create table t (a bigint primary key, b int);") + tk.MustExec("insert into t values (1, 1), (10, 1);") + tk.MustExec("alter table t add index idx(b) pre_split_regions = (between (1) and (2) regions 3);") + tk.MustExec("drop table t;") +}