From 34fbfa92f22dac4c4478cf2dfc529bf47b0395ce Mon Sep 17 00:00:00 2001 From: Davide Petilli Date: Sun, 24 Jul 2022 12:09:26 +0200 Subject: [PATCH] Add TIMED OUT and NOT VIABLE (#40) (#49) --- .deepsource.toml | 4 +- .github/workflows/release.yml | 1 + .goreleaser.yaml | 3 ++ README.md | 15 ++++++- mutant/mutant.go | 6 +++ mutant/mutant_test.go | 10 +++++ mutator/mutator.go | 60 +++++++++++++++++++++------- mutator/mutator_test.go | 73 +++++++++++++++++++++++++++-------- report/report.go | 27 +++++++++---- report/report_test.go | 11 +++++- 10 files changed, 169 insertions(+), 41 deletions(-) diff --git a/.deepsource.toml b/.deepsource.toml index b4036fae..e9a83d64 100644 --- a/.deepsource.toml +++ b/.deepsource.toml @@ -5,4 +5,6 @@ name = "go" enabled = true [analyzers.meta] -import_root = "github.com/k3rn31/gremlins" \ No newline at end of file +import_root = "github.com/k3rn31/gremlins" +cgo_enabled = false +dependencies_vendored = false \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0a54cf9e..7a001259 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,3 +29,4 @@ jobs: args: release --rm-dist env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WORKFLOW_PAT: ${{ secrets.WORKFLOW_PAT }} diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 9fbaa346..7ab85f67 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -33,6 +33,9 @@ brews: - tap: owner: k3rn31 name: gremlins-tap + branch: main + token: "{{ .Env.WORKFLOW_PAT }}" + folder: Formula homepage: https://github.com/k3rn31/gremlins description: A mutation testing tool for Go. license: Apache 2.0 License diff --git a/README.md b/README.md index ebec3057..91a9b0e0 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ [![codecov](https://codecov.io/gh/k3rn31/gremlins/branch/main/graph/badge.svg?token=MICF9A6U3J)](https://codecov.io/gh/k3rn31/gremlins) [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fk3rn31%2Fgremlins.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fk3rn31%2Fgremlins?ref=badge_shield) -**WARNING: Gremlins is in its early stages of development, and it can be unstable, poorly performant and not really -polished.** +**WARNING1: Gremlins is in its early stages of development, and it can be unstable and/or poorly performant.** +**WARNING2: Gremlins isn't currently supported on Windows.** Gremlins is a mutation testing tool for Go. @@ -27,6 +27,7 @@ Gremlins is a mutation testing tool for Go. - [What Inspired Gremlins](#what-inspired-gremlins) - [Other Mutation Testing Tools for Go](#other-mutation-testing-tools-for-go) - [Contributing](#contributing) +- [License](#license) ## What is Mutation Testing @@ -102,6 +103,16 @@ To perform the analysis without actually running the tests: $ gremlins unleash --dry-run ``` +Gremlins will report each mutation as: + +- `RUNNABLE`: In _dry-run_ mode, a mutation that can be tested. +- `NOT COVERED`: A mutation not covered by tests; it will not be tested. +- `KILLED`: The mutation has been caught by the test suite. +- `LIVED`: The mutation hasn't been caught by the test suite. +- `TIMED OUT`: The tests timed out while testing the mutation: the mutation actually made the tests fail, but not + explicitly. +- `NOT VIABLE`: The mutation makes the build fail. + ### Supported mutations #### Conditionals Boundaries diff --git a/mutant/mutant.go b/mutant/mutant.go index b6e732f2..f756275c 100644 --- a/mutant/mutant.go +++ b/mutant/mutant.go @@ -36,6 +36,8 @@ const ( Runnable Lived Killed + NotViable + TimedOut ) func (ms Status) String() string { @@ -48,6 +50,10 @@ func (ms Status) String() string { return "LIVED" case Killed: return "KILLED" + case NotViable: + return "NOT VIABLE" + case TimedOut: + return "TIMED OUT" default: panic("this should not happen") } diff --git a/mutant/mutant_test.go b/mutant/mutant_test.go index 9f9026fa..98c03133 100644 --- a/mutant/mutant_test.go +++ b/mutant/mutant_test.go @@ -48,6 +48,16 @@ func TestStatusString(t *testing.T) { mutant.Killed, "KILLED", }, + { + "NotViable", + mutant.NotViable, + "NOT VIABLE", + }, + { + "TimedOut", + mutant.TimedOut, + "TIMED OUT", + }, } for _, tc := range testCases { tc := tc diff --git a/mutator/mutator.go b/mutator/mutator.go index 9c7c8f27..f3ef88ec 100644 --- a/mutator/mutator.go +++ b/mutator/mutator.go @@ -17,6 +17,7 @@ package mutator import ( + "context" "github.com/k3rn31/gremlins/coverage" "github.com/k3rn31/gremlins/log" "github.com/k3rn31/gremlins/mutant" @@ -55,7 +56,7 @@ type Mutator struct { const timeoutCoefficient = 2 -type execContext = func(name string, args ...string) *exec.Cmd +type execContext = func(ctx context.Context, name string, args ...string) *exec.Cmd // Option for the Mutator initialization. type Option func(m Mutator) Mutator @@ -77,7 +78,7 @@ func New(fs fs.FS, r coverage.Result, manager workdir.Dealer, opts ...Option) Mu covProfile: r.Profile, testExecutionTime: r.Elapsed * timeoutCoefficient, fs: fs, - execContext: exec.Command, + execContext: exec.CommandContext, apply: func(m mutant.Mutant) error { return m.Apply() }, @@ -212,24 +213,19 @@ func (mu *Mutator) executeTests() report.Results { report.Mutant(m) continue } + if err := mu.apply(m); err != nil { log.Errorf("failed to apply mutation at %s - %s\n\t%v", m.Position(), m.Status(), err) continue } - m.SetStatus(mutant.Lived) - args := []string{"test", "-timeout", mu.testExecutionTime.String()} - if mu.buildTags != "" { - args = append(args, "-tags", mu.buildTags) - } - args = append(args, "./...") - cmd := mu.execContext("go", args...) - if err := cmd.Run(); err != nil { - m.SetStatus(mutant.Killed) - } + + m.SetStatus(mu.runTests()) + if err := mu.rollback(m); err != nil { - log.Errorf("failed to restore mutation at %s - %s\n\t%v", m.Position(), m.Status(), err) // What should we do now? + log.Errorf("failed to restore mutation at %s - %s\n\t%v", m.Position(), m.Status(), err) } + report.Mutant(m) mutants = append(mutants, m) } @@ -239,3 +235,41 @@ func (mu *Mutator) executeTests() report.Results { } return results } + +func (mu *Mutator) runTests() mutant.Status { + ctx, cancel := context.WithTimeout(context.Background(), mu.testExecutionTime) + defer cancel() + cmd := mu.execContext(ctx, "go", mu.getTestArgs()...) + + err := cmd.Run() + if ctx.Err() == context.DeadlineExceeded { + return mutant.TimedOut + } + if err != nil { + err, ok := err.(*exec.ExitError) + if ok { + return getTestFailedStatus(err.ExitCode()) + } + } + return mutant.Lived +} + +func (mu *Mutator) getTestArgs() []string { + args := []string{"test"} + if mu.buildTags != "" { + args = append(args, "-tags", mu.buildTags) + } + args = append(args, "./...") + return args +} + +func getTestFailedStatus(exitCode int) mutant.Status { + switch exitCode { + case 1: + return mutant.Killed + case 2: + return mutant.NotViable + default: + return mutant.Lived + } +} diff --git a/mutator/mutator_test.go b/mutator/mutator_test.go index fc99b758..9fe174a7 100644 --- a/mutator/mutator_test.go +++ b/mutator/mutator_test.go @@ -17,6 +17,7 @@ package mutator_test import ( + "context" "fmt" "github.com/google/go-cmp/cmp" "github.com/k3rn31/gremlins/coverage" @@ -32,16 +33,18 @@ import ( "time" ) +const expectedTimeout = 10 * time.Second + func coveredPosition(fixture string) coverage.Result { fn := filenameFromFixture(fixture) p := coverage.Profile{fn: {{StartLine: 6, EndLine: 7, StartCol: 8, EndCol: 9}}} - return coverage.Result{Profile: p, Elapsed: 1 * time.Second} + return coverage.Result{Profile: p, Elapsed: expectedTimeout} } func notCoveredPosition(fixture string) coverage.Result { fn := filenameFromFixture(fixture) p := coverage.Profile{fn: {{StartLine: 9, EndLine: 9, StartCol: 8, EndCol: 9}}} - return coverage.Result{Profile: p, Elapsed: 1 * time.Second} + return coverage.Result{Profile: p, Elapsed: expectedTimeout} } type dealerStub struct{} @@ -280,9 +283,10 @@ func TestSkipTestAndNonGoFiles(t *testing.T) { type commandHolder struct { command string args []string + timeout time.Duration } -type execContext = func(name string, args ...string) *exec.Cmd +type execContext = func(ctx context.Context, name string, args ...string) *exec.Cmd func TestMutatorRun(t *testing.T) { t.Parallel() @@ -308,12 +312,25 @@ func TestMutatorRun(t *testing.T) { _ = mut.Run() - want := "go test -timeout 2s -tags tag1 tag2 ./..." + want := "go test -tags tag1 tag2 ./..." got := fmt.Sprintf("go %v", strings.Join(holder.args, " ")) if !cmp.Equal(got, want) { t.Errorf(cmp.Diff(got, want)) } + + timeoutDifference := absTimeDiff(holder.timeout, expectedTimeout*2) + diffThreshold := 50 * time.Microsecond + if timeoutDifference > diffThreshold { + t.Errorf("expected timeout to be within %s from the set timeout, got %s", diffThreshold, timeoutDifference) + } +} + +func absTimeDiff(a, b time.Duration) time.Duration { + if a > b { + return a - b + } + return b - a } func TestMutatorTestExecution(t *testing.T) { @@ -341,10 +358,17 @@ func TestMutatorTestExecution(t *testing.T) { { name: "if tests fails then mutation is KILLED", fixture: "testdata/fixtures/gtr_go", - testResult: fakeExecCommandFailure, + testResult: fakeExecCommandTestsFailure, covResult: coveredPosition("testdata/fixtures/gtr_go"), wantMutStatus: mutant.Killed, }, + { + name: "if build fails then mutation is BUILD FAILED", + fixture: "testdata/fixtures/gtr_go", + testResult: fakeExecCommandBuildFailure, + covResult: coveredPosition("testdata/fixtures/gtr_go"), + wantMutStatus: mutant.NotViable, + }, } for _, tc := range testCases { tc := tc @@ -390,43 +414,58 @@ func TestCoverageProcessSuccess(_ *testing.T) { os.Exit(0) } -func TestCoverageProcessFailure(_ *testing.T) { +func TestProcessTestsFailure(_ *testing.T) { if os.Getenv("GO_TEST_PROCESS") != "1" { return } os.Exit(1) } -func fakeExecCommandSuccess(command string, args ...string) *exec.Cmd { +func TestProcessBuildFailure(_ *testing.T) { + if os.Getenv("GO_TEST_PROCESS") != "1" { + return + } + os.Exit(2) +} + +func fakeExecCommandSuccess(ctx context.Context, command string, args ...string) *exec.Cmd { cs := []string{"-test.run=TestCoverageProcessSuccess", "--", command} cs = append(cs, args...) // #nosec G204 - We are in tests, we don't care - cmd := exec.Command(os.Args[0], cs...) + cmd := exec.CommandContext(ctx, os.Args[0], cs...) cmd.Env = []string{"GO_TEST_PROCESS=1"} return cmd } func fakeExecCommandSuccessWithHolder(got *commandHolder) execContext { - return func(command string, args ...string) *exec.Cmd { + return func(ctx context.Context, command string, args ...string) *exec.Cmd { + dl, _ := ctx.Deadline() if got != nil { got.command = command got.args = args + got.timeout = time.Until(dl) } cs := []string{"-test.run=TestCoverageProcessSuccess", "--", command} cs = append(cs, args...) - // #nosec G204 - We are in tests, we don't care - cmd := exec.Command(os.Args[0], cs...) - cmd.Env = []string{"GO_TEST_PROCESS=1"} - - return cmd + return getCmd(ctx, cs) } } -func fakeExecCommandFailure(command string, args ...string) *exec.Cmd { - cs := []string{"-test.run=TestCoverageProcessFailure", "--", command} +func fakeExecCommandTestsFailure(ctx context.Context, command string, args ...string) *exec.Cmd { + cs := []string{"-test.run=TestProcessTestsFailure", "--", command} cs = append(cs, args...) + return getCmd(ctx, cs) +} + +func fakeExecCommandBuildFailure(ctx context.Context, command string, args ...string) *exec.Cmd { + cs := []string{"-test.run=TestProcessBuildFailure", "--", command} + cs = append(cs, args...) + return getCmd(ctx, cs) +} + +func getCmd(ctx context.Context, cs []string) *exec.Cmd { // #nosec G204 - We are in tests, we don't care - cmd := exec.Command(os.Args[0], cs...) + cmd := exec.CommandContext(ctx, os.Args[0], cs...) cmd.Env = []string{"GO_TEST_PROCESS=1"} return cmd } diff --git a/report/report.go b/report/report.go index 2d9c1a7e..68348cba 100644 --- a/report/report.go +++ b/report/report.go @@ -27,6 +27,8 @@ import ( var ( fgRed = color.New(color.FgRed).SprintFunc() fgGreen = color.New(color.FgGreen).SprintFunc() + fgHiGreen = color.New(color.FgHiGreen).SprintFunc() + fgHiBlack = color.New(color.FgHiBlack).SprintFunc() fgHiYellow = color.New(color.FgYellow).SprintFunc() ) @@ -46,7 +48,7 @@ func Do(results Results) { log.Infoln("\nNo results to report.") return } - var k, l, n, r int + var k, l, t, nc, nv, r int for _, m := range results.Mutants { switch m.Status() { case mutant.Killed: @@ -54,16 +56,20 @@ func Do(results Results) { case mutant.Lived: l++ case mutant.NotCovered: - n++ + nc++ + case mutant.TimedOut: + t++ + case mutant.NotViable: + nv++ case mutant.Runnable: r++ } } elapsed := durafmt.Parse(results.Elapsed).LimitFirstN(2) - notCovered := fgHiYellow(n) + notCovered := fgHiYellow(nc) if r > 0 { runnable := fgGreen(r) - rCoverage := float64(r) / float64(r+n) * 100 + rCoverage := float64(r) / float64(r+nc) * 100 log.Infoln("") log.Infof("Dry run completed in %s\n", elapsed.String()) log.Infof("Runnable: %s, Not covered: %s\n", runnable, notCovered) @@ -71,12 +77,15 @@ func Do(results Results) { return } tEfficacy := float64(k) / float64(k+l) * 100 - rCoverage := float64(k+l) / float64(k+l+n) * 100 - killed := fgGreen(k) + rCoverage := float64(k+l) / float64(k+l+nc) * 100 + killed := fgHiGreen(k) lived := fgRed(l) + timedOut := fgGreen(t) + notViable := fgHiBlack(nv) log.Infoln("") log.Infof("Mutation testing completed in %s\n", elapsed.String()) log.Infof("Killed: %s, Lived: %s, Not covered: %s\n", killed, lived, notCovered) + log.Infof("Timed out: %s, Not viable: %s\n", timedOut, notViable) log.Infof("Test efficacy: %.2f%%\n", tEfficacy) log.Infof("Mutant coverage: %.2f%%\n", rCoverage) } @@ -90,11 +99,15 @@ func Mutant(m mutant.Mutant) { status := m.Status().String() switch m.Status() { case mutant.Killed, mutant.Runnable: - status = fgGreen(m.Status()) + status = fgHiGreen(m.Status()) case mutant.Lived: status = fgRed(m.Status()) case mutant.NotCovered: status = fgHiYellow(m.Status()) + case mutant.TimedOut: + status = fgGreen(m.Status()) + case mutant.NotViable: + status = fgHiBlack(m.Status()) } log.Infof("%s%s %s at %s\n", padding(m.Status()), status, m.Type(), m.Position()) } diff --git a/report/report_test.go b/report/report_test.go index 33b7375e..1bdac2ed 100644 --- a/report/report_test.go +++ b/report/report_test.go @@ -37,6 +37,8 @@ func TestReport(t *testing.T) { stubMutant{mutant.Lived, mutant.ConditionalsNegation}, stubMutant{mutant.Killed, mutant.ConditionalsNegation}, stubMutant{mutant.NotCovered, mutant.ConditionalsNegation}, + stubMutant{mutant.NotViable, mutant.ConditionalsBoundary}, + stubMutant{mutant.TimedOut, mutant.ConditionalsBoundary}, } data := report.Results{ Mutants: mutants, @@ -51,6 +53,7 @@ func TestReport(t *testing.T) { // Limit the time reporting to the first two units (millis are excluded) "Mutation testing completed in 2 minutes 22 seconds\n" + "Killed: 1, Lived: 1, Not covered: 1\n" + + "Timed out: 1, Not viable: 1\n" + "Test efficacy: 50.00%\n" + "Mutant coverage: 66.67%\n" @@ -126,6 +129,10 @@ func TestMutantLog(t *testing.T) { report.Mutant(m) m = stubMutant{mutant.Runnable, mutant.ConditionalsBoundary} report.Mutant(m) + m = stubMutant{mutant.NotViable, mutant.ConditionalsBoundary} + report.Mutant(m) + m = stubMutant{mutant.TimedOut, mutant.ConditionalsBoundary} + report.Mutant(m) got := out.String() @@ -133,7 +140,9 @@ func TestMutantLog(t *testing.T) { " LIVED CONDITIONALS_BOUNDARY at aFolder/aFile.go:12:3\n" + " KILLED CONDITIONALS_BOUNDARY at aFolder/aFile.go:12:3\n" + " NOT COVERED CONDITIONALS_BOUNDARY at aFolder/aFile.go:12:3\n" + - " RUNNABLE CONDITIONALS_BOUNDARY at aFolder/aFile.go:12:3\n" + " RUNNABLE CONDITIONALS_BOUNDARY at aFolder/aFile.go:12:3\n" + + " NOT VIABLE CONDITIONALS_BOUNDARY at aFolder/aFile.go:12:3\n" + + " TIMED OUT CONDITIONALS_BOUNDARY at aFolder/aFile.go:12:3\n" if !cmp.Equal(got, want) { t.Errorf(cmp.Diff(got, want))