diff --git a/go/cmd/dolt/dolt.go b/go/cmd/dolt/dolt.go index bd3ea4adaba..cb9a39a2791 100644 --- a/go/cmd/dolt/dolt.go +++ b/go/cmd/dolt/dolt.go @@ -18,6 +18,7 @@ import ( "context" crand "crypto/rand" "encoding/binary" + "errors" "fmt" "math/rand" "net/http" @@ -236,6 +237,7 @@ const stdErrFlag = "--stderr" const stdOutAndErrFlag = "--out-and-err" const ignoreLocksFlag = "--ignore-lock-file" const verboseEngineSetupFlag = "--verbose-engine-setup" +const profilePath = "--prof-path" const cpuProf = "cpu" const memProf = "mem" @@ -266,25 +268,44 @@ func runMain() int { verboseEngineSetup := false if len(args) > 0 { var doneDebugFlags bool + var profileOpts []func(p *profile.Profile) + hasUnstartedProfile := false for !doneDebugFlags && len(args) > 0 { switch args[0] { + case profilePath: + path := args[1] + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + panic(fmt.Sprintf("profile path does not exist: %s", path)) + } + profileOpts = append(profileOpts, profile.ProfilePath(path)) + args = args[2:] case profFlag: + if hasUnstartedProfile { + defer profile.Start(profileOpts...).Stop() + profileOpts = nil + hasUnstartedProfile = false + } + + profileOpts = append(profileOpts, profile.NoShutdownHook) + hasUnstartedProfile = true + switch args[1] { case cpuProf: + profileOpts = append(profileOpts, profile.CPUProfile) cli.Println("cpu profiling enabled.") - defer profile.Start(profile.CPUProfile, profile.NoShutdownHook).Stop() case memProf: + profileOpts = append(profileOpts, profile.MemProfile) cli.Println("mem profiling enabled.") - defer profile.Start(profile.MemProfile, profile.NoShutdownHook).Stop() case blockingProf: + profileOpts = append(profileOpts, profile.BlockProfile) cli.Println("block profiling enabled") - defer profile.Start(profile.BlockProfile, profile.NoShutdownHook).Stop() case traceProf: + profileOpts = append(profileOpts, profile.TraceProfile) cli.Println("trace profiling enabled") - defer profile.Start(profile.TraceProfile, profile.NoShutdownHook).Stop() default: panic("Unexpected prof flag: " + args[1]) } + args = args[2:] case pprofServerFlag: @@ -427,6 +448,9 @@ func runMain() int { doneDebugFlags = true } } + if hasUnstartedProfile { + defer profile.Start(profileOpts...).Stop() + } } seedGlobalRand() diff --git a/go/go.mod b/go/go.mod index eeaebf34980..4b45e22a20d 100644 --- a/go/go.mod +++ b/go/go.mod @@ -57,7 +57,7 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.17.1-0.20240206231401-eee58447437c + github.com/dolthub/go-mysql-server v0.17.1-0.20240207043547-b122de451568 github.com/dolthub/swiss v0.1.0 github.com/goccy/go-json v0.10.2 github.com/google/go-github/v57 v57.0.0 diff --git a/go/go.sum b/go/go.sum index 033b7e04ff1..148bac89145 100644 --- a/go/go.sum +++ b/go/go.sum @@ -183,10 +183,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw1+y/N5SSCkma7FhAPw7KeGmD6c9PBZW9Y= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168= -github.com/dolthub/go-mysql-server v0.17.1-0.20240206230042-840d387981a7 h1:PfMDjBcGzcFTpbTzcWbqb9bxh+eZwz2+4ewlFJyusxw= -github.com/dolthub/go-mysql-server v0.17.1-0.20240206230042-840d387981a7/go.mod h1:ic+L5GOnDgwPaNHqvegDMofJvB7Jg2bVVqPrFXVOUoM= -github.com/dolthub/go-mysql-server v0.17.1-0.20240206231401-eee58447437c h1:+IiUy8OdNteoZQYKtv5KBOrTz+F9izOPBQfZku/1TXo= -github.com/dolthub/go-mysql-server v0.17.1-0.20240206231401-eee58447437c/go.mod h1:Kjfapc/1feJ3/WgTgiY0LoR5x/3ax5PJ26brPnvYUO8= +github.com/dolthub/go-mysql-server v0.17.1-0.20240207043547-b122de451568 h1:4d2g1uAWafAl9150dCjzzhIpNGQt5X3jMyqPTedlBvI= +github.com/dolthub/go-mysql-server v0.17.1-0.20240207043547-b122de451568/go.mod h1:Kjfapc/1feJ3/WgTgiY0LoR5x/3ax5PJ26brPnvYUO8= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto= github.com/dolthub/jsonpath v0.0.2-0.20240201003050-392940944c15 h1:sfTETOpsrNJPDn2KydiCtDgVu6Xopq8k3JP8PjFT22s= @@ -197,8 +195,6 @@ github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9X github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY= github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc= github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ= -github.com/dolthub/vitess v0.0.0-20240205203605-9e6c6d650813 h1:tGwsoLAMFQ+7FDEyIWOIJ1Vc/nptbFi0Fh7SQahB8ro= -github.com/dolthub/vitess v0.0.0-20240205203605-9e6c6d650813/go.mod h1:IwjNXSQPymrja5pVqmfnYdcy7Uv7eNJNBPK/MEh9OOw= github.com/dolthub/vitess v0.0.0-20240206204925-6acf16fa777c h1:Zt23BHsxvPHGfpHV9k/FcsHqWZjfybyQQux2OLpRni8= github.com/dolthub/vitess v0.0.0-20240206204925-6acf16fa777c/go.mod h1:IwjNXSQPymrja5pVqmfnYdcy7Uv7eNJNBPK/MEh9OOw= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go index b59f199c151..512f2234a2f 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go @@ -5204,6 +5204,82 @@ var DoltAutoIncrementTests = []queries.ScriptTest{ }, }, }, + { + // Dropping the primary key constraint from a table implicitly truncates the table, which resets the + // auto_increment value for the table to 0. These tests assert that the correct auto_increment value is + // restored after the drop pk operation. + Name: "drop auto_increment primary key", + SetUpScript: []string{ + "create table t (a int primary key auto_increment, b int, key (a))", + "call dolt_commit('-Am', 'empty table')", + "call dolt_branch('branch1')", + "call dolt_branch('branch2')", + "insert into t (b) values (1), (2)", + "call dolt_commit('-am', 'two values on main')", + "call dolt_checkout('branch1')", + "insert into t (b) values (3), (4)", + "call dolt_commit('-am', 'two values on branch1')", + "call dolt_checkout('branch2')", + "insert into t (b) values (5), (6)", + "call dolt_checkout('main')", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "alter table t drop primary key", + Expected: []sql.Row{{types.NewOkResult(0)}}, + }, + { + // highest value in any branch is 6 + Query: "insert into t (b) values (7), (8)", + Expected: []sql.Row{{types.OkResult{RowsAffected: 2, InsertID: 7}}}, + }, + { + Query: "select * from t order by a", + Expected: []sql.Row{ + {1, 1}, + {2, 2}, + {7, 7}, + {8, 8}, + }, + }, + { + Query: "call dolt_checkout('branch2')", + SkipResultsCheck: true, + }, + { + Query: "insert into t (b) values (9), (10)", + Expected: []sql.Row{{types.OkResult{RowsAffected: 2, InsertID: 9}}}, + }, + { + Query: "select * from t order by a", + Expected: []sql.Row{ + {5, 5}, + {6, 6}, + {9, 9}, + {10, 10}, + }, + }, + { + Query: "alter table t drop primary key", + Expected: []sql.Row{{types.NewOkResult(0)}}, + }, + { + Query: "insert into t (b) values (11), (12)", + Expected: []sql.Row{{types.OkResult{RowsAffected: 2, InsertID: 11}}}, + }, + { + Query: "select * from t order by a", + Expected: []sql.Row{ + {5, 5}, + {6, 6}, + {9, 9}, + {10, 10}, + {11, 11}, + {12, 12}, + }, + }, + }, + }, } var DoltCherryPickTests = []queries.ScriptTest{ diff --git a/go/libraries/doltcore/sqle/tables.go b/go/libraries/doltcore/sqle/tables.go index d775ad5b0e0..fff93b32011 100644 --- a/go/libraries/doltcore/sqle/tables.go +++ b/go/libraries/doltcore/sqle/tables.go @@ -1556,6 +1556,16 @@ func (t *AlterableDoltTable) RewriteInserter( }) } + // Grab the next auto_increment value before we call truncate, since truncate will delete the table + // and clear out the auto_increment tracking for this table. + var nextAutoIncValue uint64 + if t.autoIncCol.AutoIncrement { + nextAutoIncValue, err = t.PeekNextAutoIncrementValue(ctx) + if err != nil { + return nil, err + } + } + // TODO: test for this when the table is auto increment and exists on another branch dt, err = t.truncate(ctx, dt, newSch, sess) if err != nil { @@ -1590,6 +1600,14 @@ func (t *AlterableDoltTable) RewriteInserter( newWs := ws.WithWorkingRoot(newRoot) + // Restore the next auto increment value, since it was cleared when we truncated the table + if t.autoIncCol.AutoIncrement { + err = t.AutoIncrementSetter(ctx).SetAutoIncrementValue(ctx, nextAutoIncValue) + if err != nil { + return nil, err + } + } + opts := dbState.WriteSession().GetOptions() opts.ForeignKeyChecksDisabled = true writeSession := writer.NewWriteSession(dt.Format(), newWs, ait, opts) @@ -1945,8 +1963,9 @@ func validateSchemaChange( } func (t *AlterableDoltTable) adjustForeignKeysForDroppedPk(ctx *sql.Context, tbl string, root *doltdb.RootValue) (*doltdb.RootValue, error) { - if t.autoIncCol.AutoIncrement { - return nil, sql.ErrWrongAutoKey.New() + err := sql.ValidatePrimaryKeyDrop(ctx, t, t.PrimaryKeySchema()) + if err != nil { + return nil, err } fkc, err := root.GetForeignKeyCollection(ctx) diff --git a/go/performance/utils/sysbench_runner/config.go b/go/performance/utils/sysbench_runner/config.go index 5488d9eca8b..cc1962aeaa9 100644 --- a/go/performance/utils/sysbench_runner/config.go +++ b/go/performance/utils/sysbench_runner/config.go @@ -49,6 +49,17 @@ const ( sysbenchUsername = "sysbench" sysbenchUserLocal = "'sysbench'@'localhost'" sysbenchPassLocal = "sysbenchpass" + + userFlag = "--user" + hostFlag = "--host" + portFlag = "--port" + skipBinLogFlag = "--skip-log-bin" + profileFlag = "--prof" + profilePathFlag = "--prof-path" + cpuProfile = "cpu" + doltgresDataDirFlag = "--data-dir" + MysqlDataDirFlag = "--datadir" + MysqlInitializeInsecureFlag = "--initialize-insecure" ) var ( @@ -64,9 +75,6 @@ var defaultSysbenchParams = []string{ } var defaultDoltServerParams = []string{"sql-server"} -var defaultMysqlServerParams = []string{} -var defaultDoltgresServerParams = []string{} -var defaultPostgresServerParams = []string{} var defaultSysbenchTests = []*ConfigTest{ NewConfigTest("oltp_read_only", []string{}, false), @@ -283,6 +291,12 @@ type ServerConfig struct { // Socket is the path to the server socket Socket string + + // ServerProfile specifies the golang profile to take of a Dolt server + ServerProfile string + + // ProfilePath path to directory where server profile will be written + ProfilePath string } func (sc *ServerConfig) GetId() string { @@ -293,36 +307,39 @@ func (sc *ServerConfig) GetId() string { } // GetServerArgs returns the args used to start a server -func (sc *ServerConfig) GetServerArgs() []string { +func (sc *ServerConfig) GetServerArgs() ([]string, error) { params := make([]string, 0) - defaultParams := make([]string, 0) if sc.Server == Dolt { - defaultParams = defaultDoltServerParams + if sc.ServerProfile != "" { + if sc.ServerProfile == cpuProfile { + params = append(params, profileFlag, cpuProfile) + } else { + return nil, fmt.Errorf("unsupported server profile: %s", sc.ServerProfile) + } + if sc.ProfilePath != "" { + params = append(params, profilePathFlag, sc.ProfilePath) + } + } + params = append(params, defaultDoltServerParams...) } else if sc.Server == MySql { - defaultParams = defaultMysqlServerParams if sc.ServerUser != "" { - params = append(params, fmt.Sprintf("--user=%s", sc.ServerUser)) + params = append(params, fmt.Sprintf("%s=%s", userFlag, sc.ServerUser)) } if sc.SkipLogBin { - params = append(params, "--skip-log-bin") + params = append(params, skipBinLogFlag) } - } else if sc.Server == Doltgres { - defaultParams = defaultDoltgresServerParams - } else if sc.Server == Postgres { - defaultParams = defaultPostgresServerParams } - params = append(params, defaultParams...) if sc.Server == Dolt || sc.Server == Doltgres { - params = append(params, fmt.Sprintf("--host=%s", sc.Host)) + params = append(params, fmt.Sprintf("%s=%s", hostFlag, sc.Host)) } if sc.Port != 0 { - params = append(params, fmt.Sprintf("--port=%d", sc.Port)) + params = append(params, fmt.Sprintf("%s=%d", portFlag, sc.Port)) } params = append(params, sc.ServerArgs...) - return params + return params, nil } // Config is the configuration for a benchmarking run diff --git a/go/performance/utils/sysbench_runner/dolt.go b/go/performance/utils/sysbench_runner/dolt.go index dad829ba73e..3bae67be6b8 100644 --- a/go/performance/utils/sysbench_runner/dolt.go +++ b/go/performance/utils/sysbench_runner/dolt.go @@ -41,9 +41,12 @@ var stampFunc = func() string { return time.Now().UTC().Format(stampFormat) } // BenchmarkDolt benchmarks dolt based on the provided configurations func BenchmarkDolt(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { - serverParams := serverConfig.GetServerArgs() + serverParams, err := serverConfig.GetServerArgs() + if err != nil { + return nil, err + } - err := DoltVersion(ctx, serverConfig.ServerExec) + err = DoltVersion(ctx, serverConfig.ServerExec) if err != nil { return nil, err } diff --git a/go/performance/utils/sysbench_runner/doltgres.go b/go/performance/utils/sysbench_runner/doltgres.go index 012dd383194..7364fa4efef 100644 --- a/go/performance/utils/sysbench_runner/doltgres.go +++ b/go/performance/utils/sysbench_runner/doltgres.go @@ -32,9 +32,12 @@ import ( // BenchmarkDoltgres benchmarks doltgres based on the provided configurations func BenchmarkDoltgres(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { - serverParams := serverConfig.GetServerArgs() + serverParams, err := serverConfig.GetServerArgs() + if err != nil { + return nil, err + } - err := DoltVersion(ctx, serverConfig.ServerExec) + err = DoltVersion(ctx, serverConfig.ServerExec) if err != nil { return nil, err } @@ -47,7 +50,7 @@ func BenchmarkDoltgres(ctx context.Context, config *Config, serverConfig *Server cleanupDoltgresServerDir(serverDir) }() - serverParams = append(serverParams, fmt.Sprintf("--data-dir=%s", serverDir)) + serverParams = append(serverParams, fmt.Sprintf("%s=%s", doltgresDataDirFlag, serverDir)) withKeyCtx, cancel := context.WithCancel(ctx) gServer, serverCtx := errgroup.WithContext(withKeyCtx) diff --git a/go/performance/utils/sysbench_runner/mysql.go b/go/performance/utils/sysbench_runner/mysql.go index 3048946718c..7a802a92099 100644 --- a/go/performance/utils/sysbench_runner/mysql.go +++ b/go/performance/utils/sysbench_runner/mysql.go @@ -64,8 +64,13 @@ func BenchmarkMysql(ctx context.Context, config *Config, serverConfig *ServerCon } gServer, serverCtx = errgroup.WithContext(withKeyCtx) - serverParams := serverConfig.GetServerArgs() - serverParams = append(serverParams, fmt.Sprintf("--datadir=%s", serverDir)) + var serverParams []string + serverParams, err = serverConfig.GetServerArgs() + if err != nil { + cancel() + return nil, err + } + serverParams = append(serverParams, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) server = getMysqlServer(serverCtx, serverConfig, serverParams) @@ -153,7 +158,7 @@ func InitMysqlDataDir(ctx context.Context, config *ServerConfig) (string, error) return "", err } - msInit := ExecCommand(ctx, config.ServerExec, "--initialize-insecure", fmt.Sprintf("--datadir=%s", serverDir)) + msInit := ExecCommand(ctx, config.ServerExec, MysqlInitializeInsecureFlag, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir)) err = msInit.Run() if err != nil { return "", err diff --git a/go/performance/utils/sysbench_runner/postgres.go b/go/performance/utils/sysbench_runner/postgres.go index 87148152adb..ce2e26be82e 100644 --- a/go/performance/utils/sysbench_runner/postgres.go +++ b/go/performance/utils/sysbench_runner/postgres.go @@ -64,7 +64,12 @@ func BenchmarkPostgres(ctx context.Context, config *Config, serverConfig *Server return nil, err } gServer, serverCtx = errgroup.WithContext(withKeyCtx) - serverParams := serverConfig.GetServerArgs() + var serverParams []string + serverParams, err = serverConfig.GetServerArgs() + if err != nil { + cancel() + return nil, err + } serverParams = append(serverParams, "-D", serverDir) server = getMysqlServer(serverCtx, serverConfig, serverParams) server.Env = append(server.Env, "LC_ALL=C") diff --git a/go/performance/utils/sysbench_runner/results.go b/go/performance/utils/sysbench_runner/results.go index ffc406cd980..5ad7256196b 100644 --- a/go/performance/utils/sysbench_runner/results.go +++ b/go/performance/utils/sysbench_runner/results.go @@ -154,7 +154,10 @@ func (r *Result) Stamp(stampFunc func() string) { // FromConfigsNewResult returns a new result with some fields set based on the provided configs func FromConfigsNewResult(config *Config, serverConfig *ServerConfig, t *Test, suiteId string, idFunc func() string) (*Result, error) { - serverParams := serverConfig.GetServerArgs() + serverParams, err := serverConfig.GetServerArgs() + if err != nil { + return nil, err + } var getId func() string if idFunc == nil { diff --git a/go/performance/utils/tpcc_runner/dolt.go b/go/performance/utils/tpcc_runner/dolt.go index 2e258a46196..e9ab273cf70 100644 --- a/go/performance/utils/tpcc_runner/dolt.go +++ b/go/performance/utils/tpcc_runner/dolt.go @@ -37,9 +37,12 @@ const ( // BenchmarkDolt executes a set of tpcc tests against a dolt server. func BenchmarkDolt(ctx context.Context, tppcConfig *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig) (sysbench_runner.Results, error) { - serverParams := serverConfig.GetServerArgs() + serverParams, err := serverConfig.GetServerArgs() + if err != nil { + return nil, err + } - err := sysbench_runner.UpdateDoltConfig(ctx, serverConfig.ServerExec) + err = sysbench_runner.UpdateDoltConfig(ctx, serverConfig.ServerExec) if err != nil { return nil, err } diff --git a/go/performance/utils/tpcc_runner/mysql.go b/go/performance/utils/tpcc_runner/mysql.go index 2f3e9628190..19a4e593a40 100644 --- a/go/performance/utils/tpcc_runner/mysql.go +++ b/go/performance/utils/tpcc_runner/mysql.go @@ -54,8 +54,13 @@ func BenchmarkMysql(ctx context.Context, config *TpccBenchmarkConfig, serverConf } gServer, serverCtx = errgroup.WithContext(withKeyCtx) - serverParams := serverConfig.GetServerArgs() - serverParams = append(serverParams, fmt.Sprintf("--datadir=%s", serverDir)) + var serverParams []string + serverParams, err = serverConfig.GetServerArgs() + if err != nil { + cancel() + return nil, err + } + serverParams = append(serverParams, fmt.Sprintf("%s=%s", sysbench_runner.MysqlDataDirFlag, serverDir)) server = getMysqlServer(serverCtx, serverConfig, serverParams) // launch the mysql server diff --git a/go/performance/utils/tpcc_runner/results.go b/go/performance/utils/tpcc_runner/results.go index 03ffb94ea68..33e0a11e262 100644 --- a/go/performance/utils/tpcc_runner/results.go +++ b/go/performance/utils/tpcc_runner/results.go @@ -25,7 +25,10 @@ import ( // FromConfigsNewResult returns a new result with some fields set based on the provided configs func FromConfigsNewResult(config *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig, test *TpccTest, suiteId string, idFunc func() string) (*sysbench_runner.Result, error) { - serverParams := serverConfig.GetServerArgs() + serverParams, err := serverConfig.GetServerArgs() + if err != nil { + return nil, err + } var getId func() string if idFunc == nil {