diff --git a/README.md b/README.md index c03ed75..a81992f 100644 --- a/README.md +++ b/README.md @@ -510,6 +510,17 @@ the base `Spec` fields listed above): present in `stderr`. * `assert.err.contains_one_of`: (optional) a list of one or more strings of which *at least one* must be present in `stderr`. +* `on`: (optional) an object describing actions to take upon certain + conditions. +* `on.fail`: (optional) an object describing an action to take when any + assertion fails for the test action. +* `on.fail.exec`: a string with the exact command to execute upon test + assertion failure. You may execute more than one command but must include the + `on.fail.shell` field to indicate that the command should be run in a shell. +* `on.fail.shell`: (optional) a string with the specific shell to use in executing the + command to run upon test assertion failure. If empty (the default), no shell + is used to execute the command and instead the operating system's `exec` family + of calls is used. [execspec]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/exec/spec.go#L11-L34 [pipeexpect]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/exec/assertions.go#L15-L26 diff --git a/plugin/exec/action.go b/plugin/exec/action.go new file mode 100644 index 0000000..9884972 --- /dev/null +++ b/plugin/exec/action.go @@ -0,0 +1,97 @@ +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. + +package exec + +import ( + "bytes" + "context" + "os/exec" + "testing" + + gdtcontext "github.com/gdt-dev/gdt/context" + "github.com/gdt-dev/gdt/debug" + gdterrors "github.com/gdt-dev/gdt/errors" + "github.com/google/shlex" +) + +// Action describes a single execution of one or more commands via the +// operating system's `exec` family of functions. +type Action struct { + // Exec is the exact command to execute. + // + // You may execute more than one command but must include the `shell` field + // to indicate that the command should be run in a shell. It is best + // practice, however, to simply use multiple `exec` specs instead of + // executing multiple commands in a single shell call. + Exec string `yaml:"exec"` + // Shell is the specific shell to use in executing the command. If empty + // (the default), no shell is used to execute the command and instead the + // operating system's `exec` family of calls is used. + Shell string `yaml:"shell,omitempty"` +} + +// Do performs a single command or shell execution returning the corresponding +// exit code and any runtime error. The `outbuf` and `errbuf` buffers will be +// filled with the contents of the command's stdout and stderr pipes +// respectively. +func (a *Action) Do( + ctx context.Context, + t *testing.T, + outbuf *bytes.Buffer, + errbuf *bytes.Buffer, + exitcode *int, +) error { + var target string + var args []string + if a.Shell == "" { + // Parse time already validated exec string parses into valid shell + // args + args, _ = shlex.Split(a.Exec) + target = args[0] + args = args[1:] + } else { + target = a.Shell + args = []string{"-c", a.Exec} + } + + debug.Println(ctx, t, "exec: %s %s", target, args) + + var cmd *exec.Cmd + cmd = exec.CommandContext(ctx, target, args...) + + outpipe, err := cmd.StdoutPipe() + if err != nil { + return err + } + errpipe, err := cmd.StderrPipe() + if err != nil { + return err + } + + err = cmd.Start() + if gdtcontext.TimedOut(ctx, err) { + return gdterrors.ErrTimeoutExceeded + } + if err != nil { + return err + } + if outbuf != nil { + outbuf.ReadFrom(outpipe) + } + if errbuf != nil { + errbuf.ReadFrom(errpipe) + } + + err = cmd.Wait() + if gdtcontext.TimedOut(ctx, err) { + return gdterrors.ErrTimeoutExceeded + } + if err != nil && exitcode != nil { + eerr, _ := err.(*exec.ExitError) + ec := eerr.ExitCode() + *exitcode = ec + } + return nil +} diff --git a/plugin/exec/eval.go b/plugin/exec/eval.go index 68fef54..243680f 100644 --- a/plugin/exec/eval.go +++ b/plugin/exec/eval.go @@ -7,12 +7,8 @@ package exec import ( "bytes" "context" - "os/exec" "testing" - "github.com/google/shlex" - - gdtcontext "github.com/gdt-dev/gdt/context" "github.com/gdt-dev/gdt/debug" gdterrors "github.com/gdt-dev/gdt/errors" "github.com/gdt-dev/gdt/result" @@ -25,57 +21,39 @@ func (s *Spec) Eval(ctx context.Context, t *testing.T) *result.Result { outbuf := &bytes.Buffer{} errbuf := &bytes.Buffer{} - var err error - var cmd *exec.Cmd - var target string - var args []string - if s.Shell == "" { - // Parse time already validated exec string parses into valid shell - // args - args, _ = shlex.Split(s.Exec) - target = args[0] - args = args[1:] - } else { - target = s.Shell - args = []string{"-c", s.Exec} - } - - debug.Println(ctx, t, "exec: %s %s", target, args) - cmd = exec.CommandContext(ctx, target, args...) - - outpipe, err := cmd.StdoutPipe() - if err != nil { - return result.New(result.WithRuntimeError(ExecRuntimeError(err))) - } - errpipe, err := cmd.StderrPipe() - if err != nil { - return result.New(result.WithRuntimeError(ExecRuntimeError(err))) - } + var ec int - err = cmd.Start() - if gdtcontext.TimedOut(ctx, err) { - return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded)) - } - if err != nil { + if err := s.Do(ctx, t, outbuf, errbuf, &ec); err != nil { + if err == gdterrors.ErrTimeoutExceeded { + return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded)) + } return result.New(result.WithRuntimeError(ExecRuntimeError(err))) } - outbuf.ReadFrom(outpipe) - errbuf.ReadFrom(errpipe) - - err = cmd.Wait() - if gdtcontext.TimedOut(ctx, err) { - return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded)) - } - ec := 0 - if err != nil { - eerr, _ := err.(*exec.ExitError) - ec = eerr.ExitCode() - } a := newAssertions(s.Assert, ec, outbuf, errbuf) if !a.OK() { for _, fail := range a.Failures() { t.Error(fail) } + if s.On != nil { + if s.On.Fail != nil { + outbuf.Reset() + errbuf.Reset() + err := s.On.Fail.Do(ctx, t, outbuf, errbuf, nil) + if err != nil { + debug.Println(ctx, t, "error in on.fail.exec: %s", err) + } + if outbuf.Len() > 0 { + debug.Println( + ctx, t, "on.fail.exec: stdout: %s", outbuf.String(), + ) + } + if errbuf.Len() > 0 { + debug.Println( + ctx, t, "on.fail.exec: stderr: %s", errbuf.String(), + ) + } + } + } } return result.New(result.WithFailures(a.Failures()...)) } diff --git a/plugin/exec/eval_test.go b/plugin/exec/eval_test.go index 3518af3..96ed19c 100644 --- a/plugin/exec/eval_test.go +++ b/plugin/exec/eval_test.go @@ -235,3 +235,50 @@ func TestTimeoutCascade(t *testing.T) { require.Contains(debugout, "using timeout of 500ms (expected: false) [scenario default]") require.Contains(debugout, "using timeout of 20ms (expected: true)") } + +// Unfortunately there's not really any good way of testing things like this +// except by manually causing an assertion to fail in the test case and +// checking to see if the `on.fail` action was taken and debug output emitted +// to the console. +// +// When I change the `testdata/on-fail-exec.yaml` file to have a failed +// assertion by changing `assert.out.is` to "dat" instead of "cat", I get the +// correct behaviour: +// +// === RUN TestOnFail +// === RUN TestOnFail/on-fail-exec +// +// action.go:59: exec: echo [cat] +// eval.go:35: assertion failed: not equal: expected dat but got cat +// action.go:59: exec: echo [bad kitty] +// eval.go:46: on.fail.exec: stdout: bad kitty +// +// === NAME TestOnFail +// +// eval_test.go:256: +// Error Trace: /home/jaypipes/src/github.com/gdt-dev/gdt/plugin/exec/eval_test.go:256 +// Error: Should be false +// Test: TestOnFail +// +// --- FAIL: TestOnFail (0.00s) +// +// --- FAIL: TestOnFail/on-fail-exec (0.00s) +func TestOnFail(t *testing.T) { + require := require.New(t) + + fp := filepath.Join("testdata", "on-fail-exec.yaml") + f, err := os.Open(fp) + require.Nil(err) + + s, err := scenario.FromReader( + f, + scenario.WithPath(fp), + ) + require.Nil(err) + require.NotNil(s) + + ctx := gdtcontext.New(gdtcontext.WithDebug()) + err = s.Run(ctx, t) + require.Nil(err) + require.False(t.Failed()) +} diff --git a/plugin/exec/on.go b/plugin/exec/on.go new file mode 100644 index 0000000..5f56e09 --- /dev/null +++ b/plugin/exec/on.go @@ -0,0 +1,29 @@ +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. + +package exec + +// On describes actions that can be taken upon certain conditions. +type On struct { + // Fail contains one or more actions to take if any of a Spec's assertions + // fail. + // + // For example, if you wanted to grep a log file in the event that no + // connectivity on a particular IP:PORT combination could be made you might + // do this: + // + // ```yaml + // tests: + // - exec: nc -z $HOST $PORT + // on: + // fail: + // exec: grep ERROR /var/log/myapp.log + // ``` + // + // The `grep ERROR /var/log/myapp.log` command will only be executed if + // there is no connectivity to $HOST:$PORT and the results of that grep + // will be directed to the test's output. You can use the `gdt.WithDebug()` + // function to configure additional `io.Writer`s to direct this output to. + Fail *Action `yaml:"fail,omitempty"` +} diff --git a/plugin/exec/parse.go b/plugin/exec/parse.go index b7a458b..a08e063 100644 --- a/plugin/exec/parse.go +++ b/plugin/exec/parse.go @@ -72,6 +72,15 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error { return err } s.Assert = e + case "on": + if valNode.Kind != yaml.MappingNode { + return errors.ExpectedMapAt(valNode) + } + var o *On + if err := valNode.Decode(&o); err != nil { + return err + } + s.On = o default: if lo.Contains(gdttypes.BaseSpecFields, key) { continue diff --git a/plugin/exec/parse_test.go b/plugin/exec/parse_test.go index 32bdfd1..bc0e697 100644 --- a/plugin/exec/parse_test.go +++ b/plugin/exec/parse_test.go @@ -57,7 +57,9 @@ func TestSimpleCommand(t *testing.T) { Index: 0, Defaults: &gdttypes.Defaults{}, }, - Exec: "ls", + Action: gdtexec.Action{ + Exec: "ls", + }, }, } assert.Equal(expTests, s.Tests) diff --git a/plugin/exec/spec.go b/plugin/exec/spec.go index 90f87f8..2589369 100644 --- a/plugin/exec/spec.go +++ b/plugin/exec/spec.go @@ -12,19 +12,11 @@ import ( // operating system's `exec` family of functions. type Spec struct { gdttypes.Spec - // Exec is the exact command to execute. - // - // You may execute more than one command but must include the `shell` field - // to indicate that the command should be run in a shell. It is best - // practice, however, to simply use multiple `exec` specs instead of - // executing multiple commands in a single shell call. - Exec string `yaml:"exec"` - // Shell is the specific shell to use in executing the command. If empty - // (the default), no shell is used to execute the command and instead the - // operating system's `exec` family of calls is used. - Shell string `yaml:"shell,omitempty"` + Action // Assert is an object containing the conditions that the Spec will assert. Assert *Expect `yaml:"assert,omitempty"` + // On is an object containing actions to take upon certain conditions. + On *On `yaml:"on,omitempty"` } func (s *Spec) SetBase(b gdttypes.Spec) { diff --git a/plugin/exec/testdata/on-fail-exec.yaml b/plugin/exec/testdata/on-fail-exec.yaml new file mode 100644 index 0000000..5cfc00f --- /dev/null +++ b/plugin/exec/testdata/on-fail-exec.yaml @@ -0,0 +1,31 @@ +name: on-fail-exec +description: a scenario that has an on.fail.exec clause +tests: + - exec: echo "cat" + assert: + out: + is: cat + # Unfortunately there's not really any good way of testing things like this + # except by manually causing an assertion to fail in the test case and checking + # to see if the `on.fail` action was taken and debug output emitted to the + # console. + # + # When I change `assert.out.is` above to "dat" instead of "cat", I get the + # correct behaviour: + # + # === RUN TestOnFail + # === RUN TestOnFail/on-fail-exec + # action.go:59: exec: echo [cat] + # eval.go:35: assertion failed: not equal: expected dat but got cat + # action.go:59: exec: echo [bad kitty] + # eval.go:46: on.fail.exec: stdout: bad kitty + # === NAME TestOnFail + # eval_test.go:256: + # Error Trace: /home/jaypipes/src/github.com/gdt-dev/gdt/plugin/exec/eval_test.go:256 + # Error: Should be false + # Test: TestOnFail + # --- FAIL: TestOnFail (0.00s) + # --- FAIL: TestOnFail/on-fail-exec (0.00s) + on: + fail: + exec: echo "bad kitty"