[no-release-notes] refactor sysbench_runner in preparation for pgo dolt releases

.github/workflows/ci-sysbench-runner-tests.yaml

@@ -1,24 +1,24 @@
-name: Test Sysbench Runner Utility Works
-
-on:
-  pull_request:
-    branches: [ main ]
-    paths:
-      - 'go/**'
-      - 'integration-tests/**'
-
-concurrency:
-  group: ci-sysbench-runner-tests-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  mysql_client_integrations_job:
-    runs-on: ubuntu-22.04
-    name: Test Sysbench Runner
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Copy Dockerfile
-        run: cp -r ./go/performance/continuous_integration/. .
-      - name: Test sysbench runner
-        uses: ./.github/actions/sysbench-runner-tests
+#name: Test Sysbench Runner Utility Works
+#
+#on:
+#  pull_request:
+#    branches: [ main ]
+#    paths:
+#      - 'go/**'
+#      - 'integration-tests/**'
+#
+#concurrency:
+#  group: ci-sysbench-runner-tests-${{ github.event.pull_request.number || github.ref }}
+#  cancel-in-progress: true
+#
+#jobs:
+#  mysql_client_integrations_job:
+#    runs-on: ubuntu-22.04
+#    name: Test Sysbench Runner
+#    steps:
+#      - name: Checkout
+#        uses: actions/checkout@v3
+#      - name: Copy Dockerfile
+#        run: cp -r ./go/performance/continuous_integration/. .
+#      - name: Test sysbench runner
+#        uses: ./.github/actions/sysbench-runner-tests

go/performance/utils/sysbench_runner/README.md → go/performance/utils/benchmark_runner/README.md

@@ -1,11 +1,6 @@
 Sysbench runner is a tool for running sysbench tests against sql servers. Custom sysbench lua scripts used
 for benchmarking Dolt are [here](https://github.com/dolthub/sysbench-lua-scripts).
 
-The tool requires a json config file to run:
-```bash
-$ sysbench_runner --config=config.json
-```
-
 Configuration:
 
 ```json
@@ -62,8 +57,6 @@ oltp_update_non_index
 
 `Port` is the server port. Defaults to **3306** for `dolt` and `mysql` Servers. (**Optional**)
 
-`Server` is the server. Only `dolt` and `mysql` are supported. (**Required**)
-
 `Version` is the server version. (**Required**)
 
 `ResultsFormat` is the format the results should be written in. Only `json` and `csv` are supported. (**Required**)
@@ -99,3 +92,33 @@ oltp_update_non_index
 `sysbench [options]... [testname] [command]`
 
 Note: Be sure that all mysql processes are off when running this locally.
+
+# TPCC
+
+TPCC runner is a tool for running TPCC tests against sql servers. These tests run against the
+Percona Labs repo [here](https://github.com/Percona-Lab/sysbench-tpcc).
+
+Note to this run this locally you need to have the TPCC repo cloned. The `ScriptDir` variable should then be linked
+to the path of the cloned repo.
+
+Configuration:
+
+```json
+{
+  "Servers": "[...]",
+  "ScriptDir":"/Users/vinairachakonda/go/src/dolthub/sysbench-tpcc",
+  "ScaleFactors": [1]
+}
+```
+
+`Servers`: The server defintions to run the benchmark against. Accepts Dolt and MySQL configuratiosn.
+
+`ScriptDir`: The directory of the TPCC testing scripts
+
+`ScaleFactors`: The number of warehouse to be generated in the test case.
+
+`NomsBinFormat`: The NomsBinFormat to use for this benchmark.
+
+Note that this configuration is still incomplete for the amount of the variable TPCC varies. This intentional as we
+want expose small amounts of independent variables until Dolt gets more robust. See `config.go` to get a breakdown of all the
+variables TPCC varies.

go/performance/utils/benchmark_runner/benchmark.go

@@ -0,0 +1,21 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import "context"
+
+type Benchmarker interface {
+	Benchmark(ctx context.Context) (Results, error)
+}

go/performance/utils/benchmark_runner/config.go

@@ -0,0 +1,39 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import "context"
+
+type Config interface {
+	GetRuns() int
+	GetScriptDir() string
+	GetNomsBinFormat() string
+	GetRuntimeOs() string
+	GetRuntimeGoArch() string
+	GetServerConfigs() []ServerConfig
+	Validate(ctx context.Context) error
+	ContainsServerOfType(server ServerType) bool
+}
+
+type SysbenchConfig interface {
+	Config
+	GetTestOptions() []string
+	GetTestConfigs() []TestConfig
+}
+
+type TpccConfig interface {
+	Config
+	GetScaleFactors() []int
+}

go/performance/utils/benchmark_runner/constants.go

@@ -0,0 +1,209 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import "time"
+
+const (
+	Dolt     ServerType = "dolt"
+	Doltgres ServerType = "doltgres"
+	Postgres ServerType = "postgres"
+	MySql    ServerType = "mysql"
+
+	CsvFormat  = "csv"
+	JsonFormat = "json"
+
+	CsvExt  = ".csv"
+	JsonExt = ".json"
+
+	CpuServerProfile ServerProfile = "cpu"
+
+	defaultHost         = "127.0.0.1"
+	defaultDoltPort     = 3306
+	defaultMysqlPort    = defaultDoltPort
+	defaultDoltgresPort = 5432
+	defaultPostgresPort = defaultDoltgresPort
+
+	defaultMysqlSocket = "/var/run/mysqld/mysqld.sock"
+
+	tcpProtocol  = "tcp"
+	unixProtocol = "unix"
+
+	sysbenchUsername          = "sysbench"
+	sysbenchUserLocal         = "'sysbench'@'localhost'"
+	sysbenchPassLocal         = "sysbenchpass"
+	sysbenchDbPsModeFlag      = "--db-ps-mode"
+	sysbenchDbPsModeDisable   = "disable"
+	sysbenchRandTypeFlag      = "--rand-type"
+	sysbenchRandTypeUniform   = "uniform"
+	sysbenchMysqlDbFlag       = "--mysql-db"
+	sysbenchDbDriverFlag      = "--db-driver"
+	sysbenchMysqlHostFlag     = "--mysql-host"
+	sysbenchMysqlPortFlag     = "--mysql-port"
+	sysbenchMysqlUserFlag     = "--mysql-user"
+	sysbenchMysqlPasswordFlag = "--mysql-password"
+	sysbenchPostgresDbDriver  = "pgsql"
+	sysbenchPostgresDbFlag    = "--pgsql-db"
+	sysbenchPostgresHostFlag  = "--pgsql-host"
+	sysbenchPostgresPortFlag  = "--pgsql-port"
+	sysbenchPostgresUserFlag  = "--pgsql-user"
+
+	doltSqlServerCommand = "sql-server"
+
+	userFlag                    = "--user"
+	hostFlag                    = "--host"
+	portFlag                    = "--port"
+	skipBinLogFlag              = "--skip-log-bin"
+	profileFlag                 = "--prof"
+	profilePathFlag             = "--prof-path"
+	cpuProfile                  = "cpu"
+	doltgresDataDirFlag         = "--data-dir"
+	MysqlDataDirFlag            = "--datadir"
+	MysqlInitializeInsecureFlag = "--initialize-insecure"
+	cpuProfileFilename          = "cpu.pprof"
+
+	sysbenchOltpReadOnlyTestName       = "oltp_read_only"
+	sysbenchOltpInsertTestName         = "oltp_insert"
+	sysbenchBulkInsertTestName         = "bulk_insert"
+	sysbenchOltpPointSelectTestName    = "oltp_point_select"
+	sysbenchSelectRandomPointsTestName = "select_random_points"
+	sysbenchSelectRandomRangesTestName = "select_random_ranges"
+	sysbenchOltpWriteOnlyTestName      = "oltp_write_only"
+	sysbenchOltpReadWriteTestName      = "oltp_read_write"
+	sysbenchOltpUpdateIndexTestName    = "oltp_update_index"
+	sysbenchOltpUpdateNonIndexTestName = "oltp_update_non_index"
+
+	sysbenchCoveringIndexScanLuaTestName = "covering_index_scan.lua"
+	sysbenchGroupByScanLuaTestName       = "groupby_scan.lua"
+	sysbenchIndexJoinLuaTestName         = "index_join.lua"
+	sysbenchIndexJoinScanLuaTestName     = "index_join_scan.lua"
+	sysbenchIndexScanLuaTestName         = "index_scan.lua"
+	sysbenchOltpDeleteInsertLuaTestName  = "oltp_delete_insert.lua"
+	sysbenchTableScanLuaTestName         = "table_scan.lua"
+	sysbenchTypesDeleteInsertLuaTestName = "types_delete_insert.lua"
+	sysbenchTypesTableScanLuaTestName    = "types_table_scan.lua"
+
+	sysbenchCoveringIndexScanPostgresLuaTestName = "covering_index_scan_postgres.lua"
+	sysbenchGroupByScanPostgresLuaTestName       = "groupby_scan_postgres.lua"
+	sysbenchIndexJoinPostgresLuaTestName         = "index_join_postgres.lua"
+	sysbenchIndexJoinScanPostgresLuaTestName     = "index_join_scan_postgres.lua"
+	sysbenchIndexScanPostgresLuaTestName         = "index_scan_postgres.lua"
+	sysbenchOltpDeleteInsertPostgresLuaTestName  = "oltp_delete_insert_postgres.lua"
+	sysbenchTableScanPostgresLuaTestName         = "table_scan_postgres.lua"
+	sysbenchTypesDeleteInsertPostgresLuaTestName = "types_delete_insert_postgres.lua"
+	sysbenchTypesTableScanPostgresLuaTestName    = "types_table_scan_postgres.lua"
+
+	doltConfigUsernameKey = "user.name"
+	doltConfigEmailKey    = "user.email"
+	doltBenchmarkUser     = "benchmark"
+	doltBenchmarkEmail    = "[email protected]"
+	doltConfigCommand     = "config"
+	doltConfigGlobalFlag  = "--global"
+	doltConfigGetFlag     = "--get"
+	doltConfigAddFlag     = "--add"
+	doltCloneCommand      = "clone"
+	doltVersionCommand    = "version"
+	doltInitCommand       = "init"
+	dbName                = "test"
+	bigEmptyRepo          = "max-hoffman/big-empty"
+	nbfEnvVar             = "DOLT_DEFAULT_BIN_FORMAT"
+
+	postgresDriver         = "postgres"
+	doltgresUser           = "doltgres"
+	doltDataDir            = ".dolt"
+	createDatabaseTemplate = "create database %s;"
+	psqlDsnTemplate        = "host=%s port=%d user=%s password=%s dbname=%s sslmode=disable"
+
+	expectedServerKilledErrorMessage     = "signal: killed"
+	expectedServerTerminatedErrorMessage = "signal: terminated"
+
+	sysbenchCommand        = "sysbench"
+	sysbenchVersionFlag    = "--version"
+	sysbenchPrepareCommand = "prepare"
+	sysbenchRunCommand     = "run"
+	sysbenchCleanupCommand = "cleanup"
+	luaPathEnvVarTemplate  = "LUA_PATH=%s"
+	luaPath                = "?.lua"
+
+	defaultMysqlUser = "root"
+
+	// Note this is built for the SysbenchDocker file. If you want to run locally you'll need to override these variables
+	// for your local MySQL setup.
+	tpccUserLocal = "'sysbench'@'localhost'"
+	tpccPassLocal = "sysbenchpass"
+
+	tpccDbName              = "sbt"
+	tpccScaleFactorTemplate = "tpcc-scale-factor-%d"
+
+	tpccDbDriverFlag         = "--db-driver"
+	tpccMysqlUsername        = "sysbench"
+	tpccMysqlDbFlag          = "--mysql-db"
+	tpccMysqlHostFlag        = "--mysql-host"
+	tpccMysqlUserFlag        = "--mysql-user"
+	tpccMysqlPasswordFlag    = "--mysql-password"
+	tpccMysqlPortFlag        = "--mysql-port"
+	tpccTimeFlag             = "--time"
+	tpccThreadsFlag          = "--threads"
+	tpccReportIntervalFlag   = "--report_interval"
+	tpccTablesFlag           = "--tables"
+	tpccScaleFlag            = "--scale"
+	tpccTransactionLevelFlag = "--trx_level"
+	tpccReportCsv            = "reportCsv"
+	tpccTransactionLevelRr   = "RR"
+	tpccLuaFilename          = "tpcc.lua"
+
+	mysqlDriverName                  = "mysql"
+	mysqlRootTCPDsnTemplate          = "root@tcp(%s:%d)/"
+	mysqlRootUnixDsnTemplate         = "root@unix(%s)/"
+	mysqlDropDatabaseSqlTemplate     = "DROP DATABASE IF EXISTS %s;"
+	mysqlCreateDatabaseSqlTemplate   = "CREATE DATABASE %s;"
+	mysqlDropUserSqlTemplate         = "DROP USER IF EXISTS %s;"
+	mysqlCreateUserSqlTemplate       = "CREATE USER %s IDENTIFIED WITH mysql_native_password BY '%s';"
+	mysqlGrantPermissionsSqlTemplate = "GRANT ALL ON %s.* to %s;"
+	mysqlSetGlobalLocalInfileSql     = "SET GLOBAL local_infile = 'ON';"
+	mysqlSetGlobalSqlModeSql         = "SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));"
+
+	postgresInitDbDataDirFlag         = "--pgdata"
+	postgresUsernameFlag              = "--username"
+	postgresUsername                  = "postgres"
+	postgresDataDirFlag               = "-D"
+	postgresDropDatabaseSqlTemplate   = "DROP DATABASE IF EXISTS %s;"
+	postgresDropUserSqlTemplate       = "DROP USER IF EXISTS %s;"
+	postgresCreateUserSqlTemplate     = "CREATE USER %s WITH PASSWORD '%s';"
+	postgresCreateDatabaseSqlTemplate = "CREATE DATABASE %s WITH OWNER %s;"
+	postgresLcAllEnvVarKey            = "LC_ALL"
+	postgresLcAllEnvVarValue          = "C"
+
+	resultsDirname = "results"
+	stampFormat    = time.RFC3339
+	SqlStatsPrefix = "SQL statistics:"
+	read           = "read"
+	write          = "write"
+	other          = "other"
+	totalQueries   = "total"
+	totalEvents    = "total number of events"
+	min            = "min"
+	avg            = "avg"
+	max            = "max"
+	percentile     = "percentile"
+	sum            = "sum"
+	transactions   = "transactions"
+	queriesPerSec  = "queries"
+	ignoredErrors  = "ignored errors"
+	reconnects     = "reconnects"
+	totalTimeSecs  = "total time"
+
+	ResultFileTemplate = "%s_%s_%s_sysbench_performance%s"
+)

go/performance/utils/sysbench_runner/csv.go → go/performance/utils/benchmark_runner/csv.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package sysbench_runner
+package benchmark_runner
 
 import (
 	"encoding/csv"
@@ -26,7 +26,7 @@ import (
 // FromResultCsvHeaders returns supported csv headers for a Result
 func FromResultCsvHeaders() []string {
 	return []string{
-		"id",
+		"id", // todo: replace with constants
 		"suite_id",
 		"test_id",
 		"runtime_os",