build: cmd & test are code (pingcap#28903)

Makefile

@@ -44,7 +44,7 @@ goword:tools/bin/goword
 	tools/bin/goword $(FILES) 2>&1 | $(FAIL_ON_STDOUT)
 
 check-static: tools/bin/golangci-lint
-	tools/bin/golangci-lint run -v $$($(PACKAGE_DIRECTORIES_WITHOUT_BR))
+	tools/bin/golangci-lint run -v $$($(PACKAGE_DIRECTORIES_TIDB_TESTS))
 
 unconvert:tools/bin/unconvert
 	@echo "unconvert check(skip check the genenrated or copied code in lightning)"
@@ -60,11 +60,11 @@ errdoc:tools/bin/errdoc-gen
 
 lint:tools/bin/revive
 	@echo "linting"
-	@tools/bin/revive -formatter friendly -config tools/check/revive.toml $(FILES_WITHOUT_BR)
+	@tools/bin/revive -formatter friendly -config tools/check/revive.toml $(FILES_TIDB_TESTS)
 
 vet:
 	@echo "vet"
-	$(GO) vet -all $(PACKAGES_WITHOUT_BR) 2>&1 | $(FAIL_ON_STDOUT)
+	$(GO) vet -all $(PACKAGES_TIDB_TESTS) 2>&1 | $(FAIL_ON_STDOUT)
 
 tidy:
 	@echo "go mod tidy"
@@ -114,44 +114,20 @@ ifeq ("$(TRAVIS_COVERAGE)", "1")
 endif
 
 devgotest: failpoint-enable
-ifeq ("$(TRAVIS_COVERAGE)", "1")
-	@echo "Running in TRAVIS_COVERAGE mode."
-	$(GO) get github.com/go-playground/overalls
-	@export log_level=info; \
-	$(OVERALLS) -project=github.com/pingcap/tidb \
-			-covermode=count \
-			-ignore='.git,br,vendor,cmd,docs,tests,LICENSES' \
-			-concurrency=4 \
-			-- -coverpkg=./... \
-			|| { $(FAILPOINT_DISABLE); exit 1; }
-else
 # grep regex: Filter out all tidb logs starting with:
 # - '[20' (like [2021/09/15 ...] [INFO]..)
 # - 'PASS:' to ignore passed tests
 # - 'ok ' to ignore passed directories
 	@echo "Running in native mode."
 	@export log_level=info; export TZ='Asia/Shanghai'; \
-	$(GOTEST) -ldflags '$(TEST_LDFLAGS)' $(EXTRA_TEST_ARGS) -cover $(PACKAGES_WITHOUT_BR) -check.p true > gotest.log || { $(FAILPOINT_DISABLE); grep -v '^\([[]20\|PASS:\|ok \)' 'gotest.log'; exit 1; }
-endif
+	$(GOTEST) -ldflags '$(TEST_LDFLAGS)' $(EXTRA_TEST_ARGS) -cover $(PACKAGES_TIDB_TESTS) -check.p true > gotest.log || { $(FAILPOINT_DISABLE); grep -v '^\([[]20\|PASS:\|ok \)' 'gotest.log'; exit 1; }
 	@$(FAILPOINT_DISABLE)
 
 
 gotest: failpoint-enable
-ifeq ("$(TRAVIS_COVERAGE)", "1")
-	@echo "Running in TRAVIS_COVERAGE mode."
-	$(GO) get github.com/go-playground/overalls
-	@export log_level=info; \
-	$(OVERALLS) -project=github.com/pingcap/tidb \
-			-covermode=count \
-			-ignore='.git,br,vendor,cmd,docs,tests,LICENSES' \
-			-concurrency=4 \
-			-- -coverpkg=./... \
-			|| { $(FAILPOINT_DISABLE); exit 1; }
-else
 	@echo "Running in native mode."
 	@export log_level=info; export TZ='Asia/Shanghai'; \
-	$(GOTEST) -ldflags '$(TEST_LDFLAGS)' $(EXTRA_TEST_ARGS) -cover $(PACKAGES_WITHOUT_BR) -check.p true > gotest.log || { $(FAILPOINT_DISABLE); cat 'gotest.log'; exit 1; }
-endif
+	$(GOTEST) -ldflags '$(TEST_LDFLAGS)' $(EXTRA_TEST_ARGS) -cover $(PACKAGES_TIDB_TESTS) -check.p true > gotest.log || { $(FAILPOINT_DISABLE); cat 'gotest.log'; exit 1; }
 	@$(FAILPOINT_DISABLE)
 
 race: failpoint-enable

Makefile.common

@@ -44,14 +44,14 @@ endif
 ARCH      := "`uname -s`"
 LINUX     := "Linux"
 MAC       := "Darwin"
-PACKAGE_LIST  := go list ./...| grep -vE "cmd|github.com\/pingcap\/tidb\/tests"
-PACKAGE_LIST_WITHOUT_BR  := go list ./...| grep -vE "cmd|github.com\/pingcap\/tidb\/tests|github.com\/pingcap\/tidb\/br"
+PACKAGE_LIST  := go list ./...
+PACKAGE_LIST_TIDB_TESTS  := go list ./... | grep -vE "github.com\/pingcap\/tidb\/br|github.com\/pingcap\/tidb\/cmd"
 PACKAGES  ?= $$($(PACKAGE_LIST))
-PACKAGES_WITHOUT_BR ?= $$($(PACKAGE_LIST_WITHOUT_BR))
+PACKAGES_TIDB_TESTS ?= $$($(PACKAGE_LIST_TIDB_TESTS))
 PACKAGE_DIRECTORIES := $(PACKAGE_LIST) | sed 's|github.com/pingcap/$(PROJECT)/||'
-PACKAGE_DIRECTORIES_WITHOUT_BR := $(PACKAGE_LIST_WITHOUT_BR) | sed 's|github.com/pingcap/$(PROJECT)/||'
+PACKAGE_DIRECTORIES_TIDB_TESTS := $(PACKAGE_LIST_TIDB_TESTS) | sed 's|github.com/pingcap/$(PROJECT)/||'
 FILES     := $$(find $$($(PACKAGE_DIRECTORIES)) -name "*.go")
-FILES_WITHOUT_BR := $$(find $$($(PACKAGE_DIRECTORIES_WITHOUT_BR)) -name "*.go")
+FILES_TIDB_TESTS := $$(find $$($(PACKAGE_DIRECTORIES_TIDB_TESTS)) -name "*.go")
 UNCONVERT_PACKAGES_LIST := go list ./...| grep -vE "lightning\/checkpoints|lightning\/manual|lightning\/common"
 UNCONVERT_PACKAGES := $$($(UNCONVERT_PACKAGES_LIST))
 

br/cmd/tidb-lightning-ctl/main.go

@@ -160,7 +160,6 @@ func fetchMode(ctx context.Context, cfg *config.Config, tls *common.TLS) error {
 	)
 }
 
-
 func checkpointErrorIgnore(ctx context.Context, cfg *config.Config, tableName string) error {
 	cpdb, err := checkpoints.OpenCheckpointsDB(ctx, cfg)
 	if err != nil {
@@ -350,4 +349,3 @@ func importEngine(ctx context.Context, cfg *config.Config, tls *common.TLS, engi
 	}
 	return errors.Trace(ce.Import(ctx, regionSplitSize))
 }
-

cmd/benchdb/main.go

@@ -224,6 +224,7 @@ func (ut *benchDB) runCountTimes(name string, count int, f func()) {
 		name, sum/time.Duration(count), count, sum, first, last, max, min)
 }
 
+// #nosec G404
 func (ut *benchDB) insertRows(spec string) {
 	start, end, _ := ut.mustParseSpec(spec)
 	loopCount := (end - start + *batchSize - 1) / *batchSize
@@ -244,6 +245,7 @@ func (ut *benchDB) insertRows(spec string) {
 	})
 }
 
+// #nosec G404
 func (ut *benchDB) updateRandomRows(spec string) {
 	start, end, totalCount := ut.mustParseSpec(spec)
 	loopCount := (totalCount + *batchSize - 1) / *batchSize

cmd/benchfilesort/main.go

@@ -56,6 +56,7 @@ var (
 	outputRatio int
 )
 
+// #nosec G404
 func nextRow(r *rand.Rand, keySize int, valSize int) *comparableRow {
 	key := make([]types.Datum, keySize)
 	for i := range key {
@@ -188,6 +189,7 @@ func decodeMeta(fd *os.File) error {
  *		One 64-bit integer represent the row size in bytes, followed by the
  *      the actual row bytes.
  */
+// #nosec G404
 func export() error {
 	var outputBytes []byte
 

cmd/benchkv/main.go

@@ -14,6 +14,7 @@
 
 package main
 
+// #nosec G108
 import (
 	"context"
 	"flag"
@@ -129,7 +130,12 @@ func main() {
 	resp, err := http.Get("http://localhost:9191/metrics")
 	terror.MustNil(err)
 
-	defer terror.Call(resp.Body.Close)
+	defer func() {
+		if err := resp.Body.Close(); err != nil {
+			log.Error("function call errored", zap.Error(err), zap.Stack("stack"))
+		}
+	}()
+
 	text, err1 := io.ReadAll(resp.Body)
 	terror.Log(errors.Trace(err1))
 

cmd/benchraw/main.go

@@ -14,6 +14,7 @@
 
 package main
 
+// #nosec G108
 import (
 	"context"
 	"flag"

cmd/ddltest/random.go

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// #nosec G404
+
 package ddltest
 
 import (

cmd/explaintest/main.go

@@ -445,7 +445,12 @@ func (t *tester) create(tableName string, qText string) error {
 		return err
 	}
 
-	return os.WriteFile(t.statsFileName(tableName), js, 0644)
+	err = resp.Body.Close()
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(t.statsFileName(tableName), js, 0600)
 }
 
 func (t *tester) commit() error {
@@ -544,7 +549,7 @@ func (t *tester) flushResult() error {
 	if !record {
 		return nil
 	}
-	return os.WriteFile(t.resultFileName(), t.buf.Bytes(), 0644)
+	return os.WriteFile(t.resultFileName(), t.buf.Bytes(), 0600)
 }
 
 func (t *tester) statsFileName(tableName string) string {

cmd/importer/data.go

@@ -61,6 +61,7 @@ func (d *datum) setInitInt64Value(min int64, max int64) {
 	d.init = true
 }
 
+// #nosec G404
 func (d *datum) updateRemains() {
 	if uint32(rand.Int31n(100))+1 <= 100-d.probability {
 		d.remains -= uint64(rand.Int63n(int64(d.remains))) + 1

cmd/importer/db.go

@@ -132,6 +132,7 @@ func genRowData(table *table) (string, error) {
 	return sql, nil
 }
 
+// #nosec G404
 func genColumnData(table *table, column *column) (string, error) {
 	tp := column.tp
 	incremental := column.incremental

cmd/importer/parser.go

@@ -174,17 +174,17 @@ func (t *table) String() string {
 	}
 
 	ret := fmt.Sprintf("[table]name: %s\n", t.name)
-	ret += fmt.Sprintf("[table]columns:\n")
+	ret += "[table]columns:\n"
 	ret += t.printColumns()
 
 	ret += fmt.Sprintf("[table]column list: %s\n", t.columnList)
 
-	ret += fmt.Sprintf("[table]indices:\n")
+	ret += "[table]indices:\n"
 	for k, v := range t.indices {
 		ret += fmt.Sprintf("key->%s, value->%v", k, v)
 	}
 
-	ret += fmt.Sprintf("[table]unique indices:\n")
+	ret += "[table]unique indices:\n"
 	for k, v := range t.uniqIndices {
 		ret += fmt.Sprintf("key->%s, value->%v", k, v)
 	}

cmd/importer/rand.go

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// #nosec G404
+
 package main
 
 import (

cmd/importer/stats.go

@@ -75,6 +75,7 @@ func (h *histogram) randInt() int64 {
 	return h.Bounds.GetRow(idx).GetInt64(0)
 }
 
+// #nosec G404
 func getValidPrefix(lower, upper string) string {
 	for i := range lower {
 		if i >= len(upper) {