Merge pull request #318 from flanksource/junit

Junit

api/v1/common.go

@@ -203,6 +203,10 @@ type Template struct {
 	JSONPath string `yaml:"jsonPath,omitempty" json:"jsonPath,omitempty"`
 }
 
+func (t Template) IsEmpty() bool {
+	return t.Template == "" && t.JSONPath == ""
+}
+
 // +k8s:deepcopy-gen=false
 type DisplayTemplate interface {
 	GetDisplayTemplate() Template

checks/junit.go

@@ -63,7 +63,21 @@ func newPod(ctx *context.Context, check v1.JunitCheck) *corev1.Pod {
 	pod.Namespace = ctx.Namespace
 	pod.Name = ctx.Canary.Name + "-" + strings.ToLower(rand.String(5))
 	pod.Spec = check.Spec
-	pod.Spec.InitContainers = pod.Spec.Containers
+	for _, container := range pod.Spec.Containers {
+		if len(container.Command) > 0 {
+			// attemp to wrap the command so that it always completes, allowing for access to junit results
+			container.Args = []string{fmt.Sprintf(`
+			set -e
+			EXIT_CODE=0
+			%s %s || EXIT_CODE=$?
+			echo "Completed with exit code of $EXIT_CODE"
+			echo $EXIT_CODE > %s/exit-code
+			exit 0
+			`, strings.Join(container.Command, " "), strings.Join(container.Args, " "), filepath.Dir(check.TestResults))}
+			container.Command = []string{"bash", "-c"}
+		}
+		pod.Spec.InitContainers = append(pod.Spec.InitContainers, container)
+	}
 	pod.Spec.Containers = []corev1.Container{
 		{
 			Name:  containerName,
@@ -101,6 +115,9 @@ func newPod(ctx *context.Context, check v1.JunitCheck) *corev1.Pod {
 }
 
 func deletePod(ctx *context.Context, pod *corev1.Pod) {
+	if ctx.Canary.Annotations["skipDelete"] == "true" {
+		return
+	}
 	if err := ctx.Kommons.DeleteByKind(podKind, pod.Namespace, pod.Name); err != nil {
 		logger.Warnf("failed to delete pod %s/%s: %v", pod.Namespace, pod.Name, err)
 	}
@@ -113,15 +130,15 @@ func podExecf(ctx *context.Context, pod corev1.Pod, result *pkg.CheckResult, cmd
 		podFail(ctx, pod, result.Failf("error running %s: %v %v %v", _cmd, stdout, stderr, err))
 		return "", false
 	}
-	return stdout, true
+	return strings.TrimSpace(stdout), true
 }
 
 func podFail(ctx *context.Context, pod corev1.Pod, result *pkg.CheckResult) *pkg.CheckResult {
 	message, _ := ctx.Kommons.GetPodLogs(pod.Namespace, pod.Name, pod.Spec.InitContainers[0].Name)
-	if len(message) > 3000 {
+	if !ctx.IsTrace() && !ctx.IsDebug() && len(message) > 3000 {
 		message = message[len(message)-3000:]
 	}
-	return result.ErrorMessage(fmt.Errorf("pod is not healthy: \n %v", message))
+	return result.ErrorMessage(fmt.Errorf("%s is %s\n %v", pod.Name, pod.Status.Phase, message))
 }
 
 func cleanupExistingPods(ctx *context.Context, k8s kubernetes.Interface, selector string) (bool, error) {
@@ -161,7 +178,7 @@ func (c *JunitChecker) Check(ctx *context.Context, extConfig external.Check) *pk
 		return result.ErrorMessage(err)
 	}
 
-	timeout := junitCheck.GetTimeout()
+	timeout := time.Duration(junitCheck.GetTimeout()) * time.Minute
 	pod := newPod(ctx, junitCheck)
 	pods := k8s.CoreV1().Pods(ctx.Namespace)
 
@@ -180,11 +197,13 @@ func (c *JunitChecker) Check(ctx *context.Context, extConfig external.Check) *pk
 	logger.Tracef("[%s/%s] waiting for tests to complete", ctx.Namespace, ctx.Canary.Name)
 	if ctx.IsTrace() {
 		go func() {
-			_ = ctx.Kommons.StreamLogs(ctx.Namespace, pod.Name)
+			if err := ctx.Kommons.StreamLogsV2(ctx.Namespace, pod.Name, timeout, pod.Spec.InitContainers[0].Name); err != nil {
+				logger.Errorf("error streaming: %s", err)
+			}
 		}()
 	}
 
-	if err := ctx.Kommons.WaitForPod(ctx.Namespace, pod.Name, time.Duration(timeout)*time.Minute, corev1.PodRunning, corev1.PodSucceeded, corev1.PodFailed); err != nil {
+	if err := ctx.Kommons.WaitForPod(ctx.Namespace, pod.Name, timeout, corev1.PodRunning, corev1.PodSucceeded, corev1.PodFailed); err != nil {
 		result.ErrorMessage(err)
 	}
 
@@ -197,9 +216,12 @@ func (c *JunitChecker) Check(ctx *context.Context, extConfig external.Check) *pk
 		return podFail(ctx, *pod, result)
 	}
 
-	logger.Tracef("[%s/%s] pod is %s", ctx, &podObj.Status.Phase)
-
 	var suites JunitTestSuites
+	exitCode, _ := podExecf(ctx, *pod, result, "cat %v/exit-code", mountPath)
+
+	if exitCode != "" && exitCode != "0" {
+		result.ErrorMessage(fmt.Errorf("process exited with: '%s'", exitCode))
+	}
 	files, ok := podExecf(ctx, *pod, result, fmt.Sprintf("find %v -name \\*.xml -type f", mountPath))
 	if !ok {
 		return result
@@ -217,13 +239,16 @@ func (c *JunitChecker) Check(ctx *context.Context, extConfig external.Check) *pk
 			return result.ErrorMessage(err)
 		}
 	}
+
 	// signal container to exit
 	_, _ = podExecf(ctx, *pod, result, "touch %s/done", mountPath)
 	result.AddDetails(suites)
-	totals := suites.Aggregate()
-	result.Duration = int64(totals.Duration * 1000)
-	if totals.Failed > 0 {
-		return result.Failf(totals.String())
+	result.Duration = int64(suites.Duration * 1000)
+	if junitCheck.Test.IsEmpty() && suites.Failed > 0 {
+		if junitCheck.Display.IsEmpty() {
+			return result.Failf(suites.Totals.String())
+		}
+		return result.Failf("")
 	}
 	return result
 }

checks/junit_xml.go

@@ -2,6 +2,7 @@ package checks
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/joshdk/go-junit"
 )
@@ -56,7 +57,7 @@ type JunitTestSuite struct {
 func (suites JunitTestSuites) GetMessages() string {
 	var message string
 	count := 0
-	for _, suite := range suites {
+	for _, suite := range suites.Suites {
 		for _, test := range suite.Tests {
 			if test.Status == junit.StatusFailed {
 				message = message + "\n" + test.Name
@@ -70,7 +71,10 @@ func (suites JunitTestSuites) GetMessages() string {
 	return message
 }
 
-type JunitTestSuites []JunitTestSuite
+type JunitTestSuites struct {
+	Suites []JunitTestSuite `json:"suites,omitempty"`
+	Totals `json:",inline"`
+}
 
 type Totals struct {
 	Passed   int     `json:"passed"`
@@ -106,6 +110,13 @@ func (t Totals) String() string {
 		s += fmt.Sprintf("%d skipped", t.Skipped)
 	}
 
+	if t.Duration > 0 {
+		if s != "" {
+			s += " "
+		}
+		s += fmt.Sprintf(" in %s", time.Duration(t.Duration)*time.Second)
+	}
+
 	return s
 }
 
@@ -143,12 +154,13 @@ func FromTest(test junit.Test) JunitTest {
 	}
 }
 
-func (t *Totals) Add(other Totals) {
+func (t Totals) Add(other Totals) Totals {
 	t.Duration += other.Duration
 	t.Passed += other.Passed
 	t.Error += other.Error
 	t.Failed += other.Failed
 	t.Skipped += other.Skipped
+	return t
 }
 
 func (suites JunitTestSuites) Append(suite junit.Suite) JunitTestSuites {
@@ -160,7 +172,9 @@ func (suites JunitTestSuites) Append(suite junit.Suite) JunitTestSuites {
 	for _, test := range suite.Tests {
 		_suite.Tests = append(_suite.Tests, FromTest(test))
 	}
-	return append(suites, _suite)
+	suites.Suites = append(suites.Suites, _suite)
+	suites.Totals = suites.Totals.Add(_suite.Totals)
+	return suites
 }
 
 func (suites JunitTestSuites) Ingest(xml string) (JunitTestSuites, error) {
@@ -173,11 +187,3 @@ func (suites JunitTestSuites) Ingest(xml string) (JunitTestSuites, error) {
 	}
 	return suites, nil
 }
-
-func (suites *JunitTestSuites) Aggregate() Totals {
-	totals := &Totals{}
-	for _, suite := range *suites {
-		totals.Add(suite.Totals)
-	}
-	return *totals
-}

checks/runchecks.go

@@ -86,7 +86,7 @@ func template(ctx *context.Context, template v1.Template) (string, error) {
 		if err := tpl.Execute(&buf, unstructured); err != nil {
 			return "", fmt.Errorf("error executing template %s: %v", strings.Split(template.Template, "\n")[0], err)
 		}
-		return buf.String(), nil
+		return strings.TrimSpace(buf.String()), nil
 	}
 	return "", nil
 }

cmd/run.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/flanksource/commons/console"
+	"github.com/flanksource/commons/timer"
 
 	"github.com/spf13/cobra"
 
@@ -27,6 +28,7 @@ var Run = &cobra.Command{
 	Run: func(cmd *cobra.Command, configFiles []string) {
 		namespace, _ := cmd.Flags().GetString("namespace")
 		junitFile, _ := cmd.Flags().GetString("junit-file")
+		timer := timer.NewTimer()
 		if len(configFiles) == 0 {
 			log.Fatalln("Must specify at least one canary")
 		}
@@ -89,7 +91,7 @@ var Run = &cobra.Command{
 			}
 		}
 
-		logger.Infof("%d passed, %d failed", passed, failed)
+		logger.Infof("%d passed, %d failed in %s", passed, failed, timer)
 
 		if failed > 0 {
 			os.Exit(1)

fixtures/k8s/junit_fail.yaml

@@ -8,6 +8,28 @@ spec:
   severity: high
   junit:
     - testResults: "/tmp/junit-results/"
+      display:
+        template: |
+          ✅ {{.results.passed}} ❌ {{.results.failed}} in 🕑 {{.results.duration}}
+          {{-  range $r := .results.suites}}
+          {{- if gt (conv.ToInt $r.failed)  0 }}
+          {{$r.name}} ✅ {{$r.passed}} ❌ {{$r.failed}} in 🕑 {{$r.duration}}
+          {{- range $t := $r.tests }}
+          {{- if not (eq $t.status "passed")}}
+          ❌ {{$t.classname}}/{{$t.name}} in 🕑 {{$t.duration}}
+          {{- if $t.message}}
+            {{ $t.message }}
+          {{- end }}
+          {{- if $t.stdout}}
+            {{$t.stdout}}
+          {{- end }}
+          {{- if $t.sterr}}
+            {{$t.stderr}}
+          {{- end }}
+          {{- end }}
+          {{- end }}
+          {{- end }}
+          {{- end }}
       spec:
         containers:
           - name: jes

fixtures/k8s/junit_pass.yaml

@@ -10,6 +10,14 @@ spec:
   severity: high
   junit:
     - testResults: "/tmp/junit-results/"
+      display:
+        template: |
+          ✅ {{.results.passed}} ❌ {{.results.failed}} in 🕑 {{.results.duration}}
+          {{  range $r := .results.suites}}
+          {{- if gt (conv.ToInt $r.failed)  0 }}
+            {{$r.name}} ✅ {{$r.passed}} ❌ {{$r.failed}} in 🕑 {{$r.duration}}
+          {{- end }}
+          {{- end }}
       spec:
         containers:
           - name: jes

fixtures/k8s/pod_pass.yaml

@@ -23,12 +23,12 @@ spec:
       path: /foo/bar
       ingressName: hello-world-golang
       ingressHost: "hello-world-golang.127.0.0.1.nip.io"
-      scheduleTimeout: 10000
-      readyTimeout: 5000
+      scheduleTimeout: 20000
+      readyTimeout: 10000
       httpTimeout: 7000
       deleteTimeout: 12000
       ingressTimeout: 10000
-      deadline: 29000
+      deadline: 60000
       httpRetryInterval: 200
       expectedContent: bar
       expectedHttpStatuses: [200, 201, 202]

fixtures/quarantine/icmp_pass.yaml

@@ -5,7 +5,7 @@ metadata:
 spec:
   interval: 30
   icmp:
-    - endpoint: https://google.com
+    - endpoint: https://api.github.com
       thresholdMillis: 600
       packetLossThreshold: 10
       packetCount: 2

go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/dustin/go-humanize v1.0.0
 	github.com/eko/gocache v1.2.0
 	github.com/flanksource/commons v1.5.8
-	github.com/flanksource/kommons v0.24.0
+	github.com/flanksource/kommons v0.25.0
 	github.com/go-ldap/ldap/v3 v3.1.7
 	github.com/go-logr/logr v0.3.0
 	github.com/go-logr/zapr v0.2.0

go.sum

@@ -392,8 +392,8 @@ github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga
 github.com/flanksource/commons v1.5.6/go.mod h1:boFAup9GmLcnz0fWQnPV5WVFcHYA7U0aWEA14ww/T/4=
 github.com/flanksource/commons v1.5.8 h1:dIQ4E/SbbLRCInc96nB7UsJHCybfhKQV3qFHrYMeOms=
 github.com/flanksource/commons v1.5.8/go.mod h1:boFAup9GmLcnz0fWQnPV5WVFcHYA7U0aWEA14ww/T/4=
-github.com/flanksource/kommons v0.24.0 h1:IoOITJOf8GV+VIH3ajV7U76Knc4xS3TbKsrifVPQ+Io=
-github.com/flanksource/kommons v0.24.0/go.mod h1:Id4QLKaHWf0VKHdwWIhALP98lhcEl0+rzGNPmfFTfVs=
+github.com/flanksource/kommons v0.25.0 h1:zUHE8a97kJwy9xNIbSTHEPrUFfzbwVuTHu5HK/ggj4k=
+github.com/flanksource/kommons v0.25.0/go.mod h1:Id4QLKaHWf0VKHdwWIhALP98lhcEl0+rzGNPmfFTfVs=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=

test/e2e.sh

@@ -111,6 +111,8 @@ echo $cmd
 DOCKER_API_VERSION=1.39
 set +e -o pipefail
 sudo --preserve-env=KUBECONFIG,TEST_FOLDER,DOCKER_API_VERSION $cmd  2>&1 | tee test.out
+code=$?
+echo "return=$code"
 sudo chown $USER test.out
 cat test.out | go-junit-report > test-results.xml
 echo "::endgroup::"