diff --git a/api/v1/common.go b/api/v1/common.go index 7afcf05ed..fee740d93 100644 --- a/api/v1/common.go +++ b/api/v1/common.go @@ -203,6 +203,10 @@ type Template struct { JSONPath string `yaml:"jsonPath,omitempty" json:"jsonPath,omitempty"` } +func (t Template) IsEmpty() bool { + return t.Template == "" && t.JSONPath == "" +} + // +k8s:deepcopy-gen=false type DisplayTemplate interface { GetDisplayTemplate() Template diff --git a/checks/junit.go b/checks/junit.go index 8820104cb..886716add 100644 --- a/checks/junit.go +++ b/checks/junit.go @@ -63,7 +63,21 @@ func newPod(ctx *context.Context, check v1.JunitCheck) *corev1.Pod { pod.Namespace = ctx.Namespace pod.Name = ctx.Canary.Name + "-" + strings.ToLower(rand.String(5)) pod.Spec = check.Spec - pod.Spec.InitContainers = pod.Spec.Containers + for _, container := range pod.Spec.Containers { + if len(container.Command) > 0 { + // attemp to wrap the command so that it always completes, allowing for access to junit results + container.Args = []string{fmt.Sprintf(` + set -e + EXIT_CODE=0 + %s %s || EXIT_CODE=$? + echo "Completed with exit code of $EXIT_CODE" + echo $EXIT_CODE > %s/exit-code + exit 0 + `, strings.Join(container.Command, " "), strings.Join(container.Args, " "), filepath.Dir(check.TestResults))} + container.Command = []string{"bash", "-c"} + } + pod.Spec.InitContainers = append(pod.Spec.InitContainers, container) + } pod.Spec.Containers = []corev1.Container{ { Name: containerName, @@ -101,6 +115,9 @@ func newPod(ctx *context.Context, check v1.JunitCheck) *corev1.Pod { } func deletePod(ctx *context.Context, pod *corev1.Pod) { + if ctx.Canary.Annotations["skipDelete"] == "true" { + return + } if err := ctx.Kommons.DeleteByKind(podKind, pod.Namespace, pod.Name); err != nil { logger.Warnf("failed to delete pod %s/%s: %v", pod.Namespace, pod.Name, err) } @@ -113,15 +130,15 @@ func podExecf(ctx *context.Context, pod corev1.Pod, result *pkg.CheckResult, cmd podFail(ctx, pod, result.Failf("error running %s: %v %v %v", _cmd, stdout, stderr, err)) return "", false } - return stdout, true + return strings.TrimSpace(stdout), true } func podFail(ctx *context.Context, pod corev1.Pod, result *pkg.CheckResult) *pkg.CheckResult { message, _ := ctx.Kommons.GetPodLogs(pod.Namespace, pod.Name, pod.Spec.InitContainers[0].Name) - if len(message) > 3000 { + if !ctx.IsTrace() && !ctx.IsDebug() && len(message) > 3000 { message = message[len(message)-3000:] } - return result.ErrorMessage(fmt.Errorf("pod is not healthy: \n %v", message)) + return result.ErrorMessage(fmt.Errorf("%s is %s\n %v", pod.Name, pod.Status.Phase, message)) } func cleanupExistingPods(ctx *context.Context, k8s kubernetes.Interface, selector string) (bool, error) { @@ -161,7 +178,7 @@ func (c *JunitChecker) Check(ctx *context.Context, extConfig external.Check) *pk return result.ErrorMessage(err) } - timeout := junitCheck.GetTimeout() + timeout := time.Duration(junitCheck.GetTimeout()) * time.Minute pod := newPod(ctx, junitCheck) pods := k8s.CoreV1().Pods(ctx.Namespace) @@ -180,11 +197,13 @@ func (c *JunitChecker) Check(ctx *context.Context, extConfig external.Check) *pk logger.Tracef("[%s/%s] waiting for tests to complete", ctx.Namespace, ctx.Canary.Name) if ctx.IsTrace() { go func() { - _ = ctx.Kommons.StreamLogs(ctx.Namespace, pod.Name) + if err := ctx.Kommons.StreamLogsV2(ctx.Namespace, pod.Name, timeout, pod.Spec.InitContainers[0].Name); err != nil { + logger.Errorf("error streaming: %s", err) + } }() } - if err := ctx.Kommons.WaitForPod(ctx.Namespace, pod.Name, time.Duration(timeout)*time.Minute, corev1.PodRunning, corev1.PodSucceeded, corev1.PodFailed); err != nil { + if err := ctx.Kommons.WaitForPod(ctx.Namespace, pod.Name, timeout, corev1.PodRunning, corev1.PodSucceeded, corev1.PodFailed); err != nil { result.ErrorMessage(err) } @@ -197,9 +216,12 @@ func (c *JunitChecker) Check(ctx *context.Context, extConfig external.Check) *pk return podFail(ctx, *pod, result) } - logger.Tracef("[%s/%s] pod is %s", ctx, &podObj.Status.Phase) - var suites JunitTestSuites + exitCode, _ := podExecf(ctx, *pod, result, "cat %v/exit-code", mountPath) + + if exitCode != "" && exitCode != "0" { + result.ErrorMessage(fmt.Errorf("process exited with: '%s'", exitCode)) + } files, ok := podExecf(ctx, *pod, result, fmt.Sprintf("find %v -name \\*.xml -type f", mountPath)) if !ok { return result @@ -217,13 +239,16 @@ func (c *JunitChecker) Check(ctx *context.Context, extConfig external.Check) *pk return result.ErrorMessage(err) } } + // signal container to exit _, _ = podExecf(ctx, *pod, result, "touch %s/done", mountPath) result.AddDetails(suites) - totals := suites.Aggregate() - result.Duration = int64(totals.Duration * 1000) - if totals.Failed > 0 { - return result.Failf(totals.String()) + result.Duration = int64(suites.Duration * 1000) + if junitCheck.Test.IsEmpty() && suites.Failed > 0 { + if junitCheck.Display.IsEmpty() { + return result.Failf(suites.Totals.String()) + } + return result.Failf("") } return result } diff --git a/checks/junit_xml.go b/checks/junit_xml.go index 410627514..90a0cb32c 100644 --- a/checks/junit_xml.go +++ b/checks/junit_xml.go @@ -2,6 +2,7 @@ package checks import ( "fmt" + "time" "github.com/joshdk/go-junit" ) @@ -56,7 +57,7 @@ type JunitTestSuite struct { func (suites JunitTestSuites) GetMessages() string { var message string count := 0 - for _, suite := range suites { + for _, suite := range suites.Suites { for _, test := range suite.Tests { if test.Status == junit.StatusFailed { message = message + "\n" + test.Name @@ -70,7 +71,10 @@ func (suites JunitTestSuites) GetMessages() string { return message } -type JunitTestSuites []JunitTestSuite +type JunitTestSuites struct { + Suites []JunitTestSuite `json:"suites,omitempty"` + Totals `json:",inline"` +} type Totals struct { Passed int `json:"passed"` @@ -106,6 +110,13 @@ func (t Totals) String() string { s += fmt.Sprintf("%d skipped", t.Skipped) } + if t.Duration > 0 { + if s != "" { + s += " " + } + s += fmt.Sprintf(" in %s", time.Duration(t.Duration)*time.Second) + } + return s } @@ -143,12 +154,13 @@ func FromTest(test junit.Test) JunitTest { } } -func (t *Totals) Add(other Totals) { +func (t Totals) Add(other Totals) Totals { t.Duration += other.Duration t.Passed += other.Passed t.Error += other.Error t.Failed += other.Failed t.Skipped += other.Skipped + return t } func (suites JunitTestSuites) Append(suite junit.Suite) JunitTestSuites { @@ -160,7 +172,9 @@ func (suites JunitTestSuites) Append(suite junit.Suite) JunitTestSuites { for _, test := range suite.Tests { _suite.Tests = append(_suite.Tests, FromTest(test)) } - return append(suites, _suite) + suites.Suites = append(suites.Suites, _suite) + suites.Totals = suites.Totals.Add(_suite.Totals) + return suites } func (suites JunitTestSuites) Ingest(xml string) (JunitTestSuites, error) { @@ -173,11 +187,3 @@ func (suites JunitTestSuites) Ingest(xml string) (JunitTestSuites, error) { } return suites, nil } - -func (suites *JunitTestSuites) Aggregate() Totals { - totals := &Totals{} - for _, suite := range *suites { - totals.Add(suite.Totals) - } - return *totals -} diff --git a/checks/runchecks.go b/checks/runchecks.go index 41550a1c6..7ca891dda 100644 --- a/checks/runchecks.go +++ b/checks/runchecks.go @@ -86,7 +86,7 @@ func template(ctx *context.Context, template v1.Template) (string, error) { if err := tpl.Execute(&buf, unstructured); err != nil { return "", fmt.Errorf("error executing template %s: %v", strings.Split(template.Template, "\n")[0], err) } - return buf.String(), nil + return strings.TrimSpace(buf.String()), nil } return "", nil } diff --git a/cmd/run.go b/cmd/run.go index 777f715f0..66ccf6b7e 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -12,6 +12,7 @@ import ( "time" "github.com/flanksource/commons/console" + "github.com/flanksource/commons/timer" "github.com/spf13/cobra" @@ -27,6 +28,7 @@ var Run = &cobra.Command{ Run: func(cmd *cobra.Command, configFiles []string) { namespace, _ := cmd.Flags().GetString("namespace") junitFile, _ := cmd.Flags().GetString("junit-file") + timer := timer.NewTimer() if len(configFiles) == 0 { log.Fatalln("Must specify at least one canary") } @@ -89,7 +91,7 @@ var Run = &cobra.Command{ } } - logger.Infof("%d passed, %d failed", passed, failed) + logger.Infof("%d passed, %d failed in %s", passed, failed, timer) if failed > 0 { os.Exit(1) diff --git a/fixtures/k8s/junit_fail.yaml b/fixtures/k8s/junit_fail.yaml index c64db16d1..793e409ca 100644 --- a/fixtures/k8s/junit_fail.yaml +++ b/fixtures/k8s/junit_fail.yaml @@ -8,6 +8,28 @@ spec: severity: high junit: - testResults: "/tmp/junit-results/" + display: + template: | + ✅ {{.results.passed}} ❌ {{.results.failed}} in 🕑 {{.results.duration}} + {{- range $r := .results.suites}} + {{- if gt (conv.ToInt $r.failed) 0 }} + {{$r.name}} ✅ {{$r.passed}} ❌ {{$r.failed}} in 🕑 {{$r.duration}} + {{- range $t := $r.tests }} + {{- if not (eq $t.status "passed")}} + ❌ {{$t.classname}}/{{$t.name}} in 🕑 {{$t.duration}} + {{- if $t.message}} + {{ $t.message }} + {{- end }} + {{- if $t.stdout}} + {{$t.stdout}} + {{- end }} + {{- if $t.sterr}} + {{$t.stderr}} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} spec: containers: - name: jes diff --git a/fixtures/k8s/junit_pass.yaml b/fixtures/k8s/junit_pass.yaml index a5cf096ce..b19c72ff6 100644 --- a/fixtures/k8s/junit_pass.yaml +++ b/fixtures/k8s/junit_pass.yaml @@ -10,6 +10,14 @@ spec: severity: high junit: - testResults: "/tmp/junit-results/" + display: + template: | + ✅ {{.results.passed}} ❌ {{.results.failed}} in 🕑 {{.results.duration}} + {{ range $r := .results.suites}} + {{- if gt (conv.ToInt $r.failed) 0 }} + {{$r.name}} ✅ {{$r.passed}} ❌ {{$r.failed}} in 🕑 {{$r.duration}} + {{- end }} + {{- end }} spec: containers: - name: jes diff --git a/fixtures/k8s/pod_pass.yaml b/fixtures/k8s/pod_pass.yaml index 026154c70..066619f93 100644 --- a/fixtures/k8s/pod_pass.yaml +++ b/fixtures/k8s/pod_pass.yaml @@ -23,12 +23,12 @@ spec: path: /foo/bar ingressName: hello-world-golang ingressHost: "hello-world-golang.127.0.0.1.nip.io" - scheduleTimeout: 10000 - readyTimeout: 5000 + scheduleTimeout: 20000 + readyTimeout: 10000 httpTimeout: 7000 deleteTimeout: 12000 ingressTimeout: 10000 - deadline: 29000 + deadline: 60000 httpRetryInterval: 200 expectedContent: bar expectedHttpStatuses: [200, 201, 202] diff --git a/fixtures/quarantine/icmp_pass.yaml b/fixtures/quarantine/icmp_pass.yaml index d686d81f9..ff19aaf79 100644 --- a/fixtures/quarantine/icmp_pass.yaml +++ b/fixtures/quarantine/icmp_pass.yaml @@ -5,7 +5,7 @@ metadata: spec: interval: 30 icmp: - - endpoint: https://google.com + - endpoint: https://api.github.com thresholdMillis: 600 packetLossThreshold: 10 packetCount: 2 diff --git a/go.mod b/go.mod index 6df3a40af..d8a9793b6 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/dustin/go-humanize v1.0.0 github.com/eko/gocache v1.2.0 github.com/flanksource/commons v1.5.8 - github.com/flanksource/kommons v0.24.0 + github.com/flanksource/kommons v0.25.0 github.com/go-ldap/ldap/v3 v3.1.7 github.com/go-logr/logr v0.3.0 github.com/go-logr/zapr v0.2.0 diff --git a/go.sum b/go.sum index 791a856e8..8e2acdc02 100644 --- a/go.sum +++ b/go.sum @@ -392,8 +392,8 @@ github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga github.com/flanksource/commons v1.5.6/go.mod h1:boFAup9GmLcnz0fWQnPV5WVFcHYA7U0aWEA14ww/T/4= github.com/flanksource/commons v1.5.8 h1:dIQ4E/SbbLRCInc96nB7UsJHCybfhKQV3qFHrYMeOms= github.com/flanksource/commons v1.5.8/go.mod h1:boFAup9GmLcnz0fWQnPV5WVFcHYA7U0aWEA14ww/T/4= -github.com/flanksource/kommons v0.24.0 h1:IoOITJOf8GV+VIH3ajV7U76Knc4xS3TbKsrifVPQ+Io= -github.com/flanksource/kommons v0.24.0/go.mod h1:Id4QLKaHWf0VKHdwWIhALP98lhcEl0+rzGNPmfFTfVs= +github.com/flanksource/kommons v0.25.0 h1:zUHE8a97kJwy9xNIbSTHEPrUFfzbwVuTHu5HK/ggj4k= +github.com/flanksource/kommons v0.25.0/go.mod h1:Id4QLKaHWf0VKHdwWIhALP98lhcEl0+rzGNPmfFTfVs= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= diff --git a/test/e2e.sh b/test/e2e.sh index a4a1cf15d..3b27b3351 100755 --- a/test/e2e.sh +++ b/test/e2e.sh @@ -111,6 +111,8 @@ echo $cmd DOCKER_API_VERSION=1.39 set +e -o pipefail sudo --preserve-env=KUBECONFIG,TEST_FOLDER,DOCKER_API_VERSION $cmd 2>&1 | tee test.out +code=$? +echo "return=$code" sudo chown $USER test.out cat test.out | go-junit-report > test-results.xml echo "::endgroup::"