From ede8cf46ef8e82e5937fe8261d978540b840af45 Mon Sep 17 00:00:00 2001 From: Teppei Fukuda Date: Wed, 15 Jan 2020 14:08:55 +0200 Subject: [PATCH] BREAKING CHANGE: make it possible to pass a command directly (#9) * fix: make it possible to pass a command directly * fix: lint issue * chore(README): update --- README.md | 77 +++++++++++++----------------------------------- config.go | 20 ++++++------- main.go | 83 ++++++++++++++++------------------------------------ main_test.go | 62 ++------------------------------------- 4 files changed, 58 insertions(+), 184 deletions(-) diff --git a/README.md b/README.md index fd1033b..abae2c6 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ jobs: run: curl -sfL https://raw.githubusercontent.com/knqyf263/cob/master/install.sh | sudo sh -s -- -b /usr/local/bin - name: Run Benchmark - run: cob -benchmem ./... + run: cob ``` ## Travis CI @@ -76,7 +76,7 @@ before_script: - curl -sfL https://raw.githubusercontent.com/knqyf263/cob/master/install.sh | sudo sh -s -- -b /usr/local/bin script: - - cob -benchmem ./... + - cob ``` ## CircleCI @@ -94,7 +94,7 @@ jobs: command: curl -sfL https://raw.githubusercontent.com/knqyf263/cob/master/install.sh | sudo sh -s -- -b /usr/local/bin - run: name: Run cob - command: cob -benchmem ./... + command: cob workflows: version: 2 build-workflow: @@ -104,58 +104,25 @@ workflows: # Example -## Print memory allocation statistics for benchmarks -``` -$ cob -benchmem ./... -``` +## Override a command to measure benchmarks -
-Result +To measure benchmarks by `make bench`, you can use `-bench-cmd` and `-bench-args` options. ``` -2020/01/12 17:31:16 Run Benchmark: 4363944cbed3da7a8245cbcdc8d8240b8976eb24 HEAD{@1} -2020/01/12 17:31:19 Run Benchmark: 599a5523729d4d99a331b9d3f71dde9e1e6daef0 HEAD - -Result -====== - -+-----------------------------+----------+---------------+-------------------+ -| Name | Commit | NsPerOp | AllocedBytesPerOp | -+-----------------------------+----------+---------------+-------------------+ -| BenchmarkAppend_Allocate-16 | HEAD | 175.00 ns/op | 111 B/op | -+ +----------+---------------+-------------------+ -| | HEAD@{1} | 108.00 ns/op | 23 B/op | -+-----------------------------+----------+---------------+-------------------+ -| BenchmarkCall-16 | HEAD | 0.27 ns/op | 0 B/op | -+ +----------+---------------+ + -| | HEAD@{1} | 0.29 ns/op | | -+-----------------------------+----------+---------------+-------------------+ - -Comparison -========== - -+-----------------------------+---------+-------------------+ -| Name | NsPerOp | AllocedBytesPerOp | -+-----------------------------+---------+-------------------+ -| BenchmarkAppend_Allocate-16 | 62.04% | 382.61% | -+-----------------------------+---------+-------------------+ -| BenchmarkCall-16 | 7.53% | 0.00% | -+-----------------------------+---------+-------------------+ - -2020/01/12 17:31:21 This commit makes benchmarks worse +$ cob -bench-cmd make -bench-args bench ``` -
- - ## Run only those benchmarks matching a regular expression +``` +$ cob -bench-args "test -bench Append -benchmem ./..." +``` +
Result ``` -$ cob -bench Append ./... 2020/01/12 17:32:30 Run Benchmark: 4363944cbed3da7a8245cbcdc8d8240b8976eb24 HEAD{@1} 2020/01/12 17:32:32 Run Benchmark: 599a5523729d4d99a331b9d3f71dde9e1e6daef0 HEAD @@ -185,7 +152,7 @@ Comparison ## Show only benchmarks with worse score ``` -$ cob -benchmem -only-degression +$ cob -only-degression ```
@@ -228,7 +195,6 @@ $ cob --base origin/master ./... # Usage ``` -$ cob -h NAME: cob - Continuous Benchmark for Go project @@ -239,13 +205,12 @@ COMMANDS: help, h Shows a list of commands or help for one command GLOBAL OPTIONS: - --only-degression Show only benchmarks with worse score (default: false) - --threshold value The program fails if the benchmark gets worse than the threshold (default: 0.1) - --bench value Run only those benchmarks matching a regular expression. (default: ".") - --benchmem Print memory allocation statistics for benchmarks. (default: false) - --benchtime value Run enough iterations of each benchmark to take t, specified as a time.Duration (for example, -benchtime 1h30s). (default: "1s") - --help, -h show help (default: false) - + --only-degression Show only benchmarks with worse score (default: false) + --threshold value The program fails if the benchmark gets worse than the threshold (default: 0.2) + --base value Specify a base commit compared with HEAD (default: "HEAD~1") + --bench-cmd value Specify a command to measure benchmarks (default: "go") + --bench-args value Specify arguments passed to -cmd (default: "test -run '^$' -bench . -benchmem ./...") + --help, -h show help (default: false) ``` # Q&A @@ -255,16 +220,16 @@ GLOBAL OPTIONS: Specify a package name. ``` -$ cob -benchmem ./foo -$ cob -benchmem ./bar +$ cob -bench-args "test -bench . -benchmem ./foo" +$ cob -bench-args "test -bench . -benchmem ./bar" ``` ## A result of benchmarks is unstable -You can specify `--benchtime`. +You can specify `-benchtime`. ``` -$ cob -benchtime 10s ./... +$ cob -bench-args "test -bench . -benchmem -benchtime 10s ./..." ``` # License diff --git a/config.go b/config.go index 677b44f..5dccb78 100644 --- a/config.go +++ b/config.go @@ -1,27 +1,25 @@ package main -import "github.com/urfave/cli/v2" +import ( + "strings" + + "github.com/urfave/cli/v2" +) type config struct { - args []string onlyDegression bool threshold float64 base string - bench string - benchmem bool - benchtime string - tags string + benchCmd string + benchArgs []string } func newConfig(c *cli.Context) config { return config{ - args: c.Args().Slice(), onlyDegression: c.Bool("only-degression"), threshold: c.Float64("threshold"), base: c.String("base"), - bench: c.String("bench"), - benchmem: c.Bool("benchmem"), - benchtime: c.String("benchtime"), - tags: c.String("tags"), + benchCmd: c.String("bench-cmd"), + benchArgs: strings.Fields(c.String("bench-args")), } } diff --git a/main.go b/main.go index e8812f9..f9a1223 100644 --- a/main.go +++ b/main.go @@ -48,22 +48,14 @@ func main() { Value: "HEAD~1", }, &cli.StringFlag{ - Name: "bench", - Usage: "Run only those benchmarks matching a regular expression.", - Value: ".", - }, - &cli.BoolFlag{ - Name: "benchmem", - Usage: "Print memory allocation statistics for benchmarks.", - }, - &cli.StringFlag{ - Name: "benchtime", - Usage: "Run enough iterations of each benchmark to take t, specified as a time.Duration (for example, -benchtime 1h30s).", - Value: "1s", + Name: "bench-cmd", + Usage: "Specify a command to measure benchmarks", + Value: "go", }, &cli.StringFlag{ - Name: "tags", - Usage: "Run only those benchmarks with the specified build tags.", + Name: "bench-args", + Usage: "Specify arguments passed to -cmd", + Value: "test -run '^$' -bench . -benchmem ./...", }, }, } @@ -109,10 +101,12 @@ func run(c config) error { return xerrors.Errorf("failed to reset the worktree to a previous commit: %w", err) } - args := prepareBenchArgs(c) + defer func() { + _ = w.Reset(&git.ResetOptions{Commit: head.Hash(), Mode: git.HardReset}) + }() log.Printf("Run Benchmark: %s %s", prev, c.base) - prevSet, err := runBenchmark(args) + prevSet, err := runBenchmark(c.benchCmd, c.benchArgs) if err != nil { return xerrors.Errorf("failed to run a benchmark: %w", err) } @@ -123,7 +117,7 @@ func run(c config) error { } log.Printf("Run Benchmark: %s %s", head.Hash(), "HEAD") - headSet, err := runBenchmark(args) + headSet, err := runBenchmark(c.benchCmd, c.benchArgs) if err != nil { return xerrors.Errorf("failed to run a benchmark: %w", err) } @@ -151,8 +145,8 @@ func run(c config) error { ratioAllocedBytesPerOp = (float64(headBench.AllocedBytesPerOp) - float64(prevBench.AllocedBytesPerOp)) / float64(prevBench.AllocedBytesPerOp) } - rows = append(rows, generateRow("HEAD", headBench, c.benchmem)) - rows = append(rows, generateRow("HEAD@{1}", prevBench, c.benchmem)) + rows = append(rows, generateRow("HEAD", headBench)) + rows = append(rows, generateRow("HEAD@{1}", prevBench)) ratios = append(ratios, result{ Name: benchName, @@ -162,10 +156,10 @@ func run(c config) error { } if !c.onlyDegression { - showResult(os.Stdout, rows, c.benchmem) + showResult(os.Stdout, rows) } - degression := showRatio(os.Stdout, ratios, c.benchmem, c.threshold, c.onlyDegression) + degression := showRatio(os.Stdout, ratios, c.threshold, c.onlyDegression) if degression { return xerrors.New("This commit makes benchmarks worse") } @@ -173,22 +167,10 @@ func run(c config) error { return nil } -func prepareBenchArgs(c config) []string { - args := []string{"test", "-run='^$'", "-benchtime", c.benchtime, "-bench", c.bench} - if c.benchmem { - args = append(args, "-benchmem") - } - if c.tags != "" { - args = append(args, "-tags", c.tags) - } - args = append(args, c.args...) - return args -} - -func runBenchmark(args []string) (parse.Set, error) { - out, err := exec.Command("go", args...).Output() +func runBenchmark(cmd string, args []string) (parse.Set, error) { + out, err := exec.Command(cmd, args...).Output() if err != nil { - return nil, xerrors.Errorf("failed to run 'go test' command: %w", err) + return nil, xerrors.Errorf("failed to run '%s %s' command: %w", cmd, strings.Join(args, " "), err) } b := bytes.NewBuffer(out) @@ -199,25 +181,19 @@ func runBenchmark(args []string) (parse.Set, error) { return s, nil } -func generateRow(ref string, b *parse.Benchmark, benchmem bool) []string { - row := []string{b.Name, ref, fmt.Sprintf(" %.2f ns/op", b.NsPerOp)} - if benchmem { - row = append(row, fmt.Sprintf(" %d B/op", b.AllocedBytesPerOp)) - } - return row +func generateRow(ref string, b *parse.Benchmark) []string { + return []string{b.Name, ref, fmt.Sprintf(" %.2f ns/op", b.NsPerOp), + fmt.Sprintf(" %d B/op", b.AllocedBytesPerOp)} } -func showResult(w io.Writer, rows [][]string, benchmem bool) { +func showResult(w io.Writer, rows [][]string) { fmt.Fprintln(w, "\nResult") fmt.Fprintf(w, "%s\n\n", strings.Repeat("=", 6)) table := tablewriter.NewWriter(w) table.SetAutoFormatHeaders(false) table.SetAlignment(tablewriter.ALIGN_CENTER) - headers := []string{"Name", "Commit", "NsPerOp"} - if benchmem { - headers = append(headers, "AllocedBytesPerOp") - } + headers := []string{"Name", "Commit", "NsPerOp", "AllocedBytesPerOp"} table.SetHeader(headers) table.SetAutoMergeCells(true) table.SetRowLine(true) @@ -225,15 +201,12 @@ func showResult(w io.Writer, rows [][]string, benchmem bool) { table.Render() } -func showRatio(w io.Writer, results []result, benchmem bool, threshold float64, onlyDegression bool) bool { +func showRatio(w io.Writer, results []result, threshold float64, onlyDegression bool) bool { table := tablewriter.NewWriter(w) table.SetAutoFormatHeaders(false) table.SetAlignment(tablewriter.ALIGN_CENTER) table.SetRowLine(true) - headers := []string{"Name", "NsPerOp"} - if benchmem { - headers = append(headers, "AllocedBytesPerOp") - } + headers := []string{"Name", "NsPerOp", "AllocedBytesPerOp"} table.SetHeader(headers) var degression bool @@ -245,11 +218,7 @@ func showRatio(w io.Writer, results []result, benchmem bool, threshold float64, continue } } - row := []string{result.Name, generateRatioItem(result.RatioNsPerOp)} - if benchmem { - row = append(row, generateRatioItem(result.RatioAllocedBytesPerOp)) - } - + row := []string{result.Name, generateRatioItem(result.RatioNsPerOp), generateRatioItem(result.RatioAllocedBytesPerOp)} colors := []tablewriter.Colors{{}} colors = append(colors, generateColor(result.RatioNsPerOp)) colors = append(colors, generateColor(result.RatioAllocedBytesPerOp)) diff --git a/main_test.go b/main_test.go index 51d547a..b34662c 100644 --- a/main_test.go +++ b/main_test.go @@ -20,33 +20,6 @@ func Test_showResult(t *testing.T) { }{ { name: "happy path", - args: args{ - rows: [][]string{ - {"BenchmarkA", "HEAD", "123 ns/op"}, - {"BenchmarkA", "HEAD{@1}", "133 ns/op"}, - {"BenchmarkB", "HEAD", "5 ns/op"}, - {"BenchmarkB", "HEAD{@1}", "9 ns/op"}, - }, - }, - wantTable: ` -Result -====== - -+------------+----------+-----------+ -| Name | Commit | NsPerOp | -+------------+----------+-----------+ -| BenchmarkA | HEAD | 123 ns/op | -+ +----------+-----------+ -| | HEAD{@1} | 133 ns/op | -+------------+----------+-----------+ -| BenchmarkB | HEAD | 5 ns/op | -+ +----------+-----------+ -| | HEAD{@1} | 9 ns/op | -+------------+----------+-----------+ -`, - }, - { - name: "with benchmem", args: args{ rows: [][]string{ {"BenchmarkA", "HEAD", "123 ns/op", "234 B/op"}, @@ -77,7 +50,7 @@ Result for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { w := &bytes.Buffer{} - showResult(w, tt.args.rows, tt.args.benchmem) + showResult(w, tt.args.rows) gotTable := w.String() assert.Equal(t, tt.wantTable, gotTable, tt.name) }) @@ -87,7 +60,6 @@ Result func Test_showRatio(t *testing.T) { type args struct { results []result - benchmem bool threshold float64 onlyDegression bool } @@ -99,32 +71,6 @@ func Test_showRatio(t *testing.T) { }{ { name: "happy path", - args: args{ - results: []result{ - { - Name: "BenchmarkA", - RatioNsPerOp: 0.01, - }, - }, - benchmem: false, - threshold: 0.2, - onlyDegression: false, - }, - want: false, - wantTable: fmt.Sprintf(` -Comparison -========== - -+------------+---------+ -| Name | NsPerOp | -+------------+---------+ -| BenchmarkA | %s | -+------------+---------+ - -`, "\x1b[1;91m1.00%\x1b[0m"), - }, - { - name: "with benchmem", args: args{ results: []result{ { @@ -133,7 +79,6 @@ Comparison RatioAllocedBytesPerOp: 0.5, }, }, - benchmem: true, threshold: 0.2, onlyDegression: false, }, @@ -165,7 +110,6 @@ Comparison RatioAllocedBytesPerOp: 0.5, }, }, - benchmem: true, threshold: 0.95, onlyDegression: false, }, @@ -199,7 +143,6 @@ Comparison RatioAllocedBytesPerOp: 0.5, }, }, - benchmem: true, threshold: 0.4, onlyDegression: true, }, @@ -231,7 +174,6 @@ Comparison RatioAllocedBytesPerOp: 0.5, }, }, - benchmem: false, threshold: 0.6, onlyDegression: true, }, @@ -242,7 +184,7 @@ Comparison for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { w := &bytes.Buffer{} - got := showRatio(w, tt.args.results, tt.args.benchmem, tt.args.threshold, tt.args.onlyDegression) + got := showRatio(w, tt.args.results, tt.args.threshold, tt.args.onlyDegression) gotTable := w.String() assert.Equal(t, tt.wantTable, gotTable, tt.name) assert.Equal(t, tt.want, got, tt.name)