From f8e1d9a09484242a72622baa1c1bcffeb3711494 Mon Sep 17 00:00:00 2001 From: Marcin Owsiany Date: Thu, 21 Nov 2024 09:35:20 +0100 Subject: [PATCH] reporting abstraction Signed-off-by: Marcin Owsiany --- pkg/report/report.go | 62 +++++++++++++++++++++++++++++++ pkg/test/case.go | 29 ++++++--------- pkg/test/case_integration_test.go | 11 +++++- pkg/test/harness.go | 4 +- 4 files changed, 85 insertions(+), 21 deletions(-) diff --git a/pkg/report/report.go b/pkg/report/report.go index 727dad0a..6bf3defc 100644 --- a/pkg/report/report.go +++ b/pkg/report/report.go @@ -114,6 +114,20 @@ type Testsuites struct { lock sync.Mutex } +// StepReporter is an interface for reporting status of a test step. +type StepReporter interface { + Failure(message string, errors ...error) + AddAssertions(i int) +} + +// TestReporter is an interface for reporting status of a test. +// For each step, call Step and use the returned step reporter. +// Make sure to call Done when a test ends (preferably using defer). +type TestReporter interface { + Step(stepName string) StepReporter + Done() +} + // NewSuiteCollection returns the address of a newly created TestSuites func NewSuiteCollection(name string) *Testsuites { start := time.Now() @@ -209,6 +223,54 @@ func (ts *Testsuite) summarize() time.Time { return end } +func (ts *Testsuite) NewTest(name string) TestReporter { + subSuite := ts.NewSubSuite(name) + return &testReporter{suite: subSuite} +} + +type stepReport struct { + name string + failed bool + failureMsg string + errors []error + assertions int +} + +func (s *stepReport) Failure(message string, errors ...error) { + s.failed = true + s.failureMsg = message + s.errors = append(s.errors, errors...) +} + +func (s *stepReport) AddAssertions(i int) { + s.assertions += i +} + +type testReporter struct { + suite *Testsuite + stepReports []*stepReport +} + +func (r *testReporter) Step(stepName string) StepReporter { + step := &stepReport{name: stepName} + r.stepReports = append(r.stepReports, step) + return step +} + +func (r *testReporter) Done() { + for _, report := range r.stepReports { + testCase := NewCase(report.name) + if report.failed { + testCase.Failure = NewFailure(report.failureMsg, report.errors) + } + testCase.Assertions += report.assertions + r.suite.AddTestcase(testCase) + } +} + +var _ TestReporter = (*testReporter)(nil) +var _ StepReporter = (*stepReport)(nil) + // AddTestSuite is a convenience method to add a testsuite to the collection in testsuites func (ts *Testsuites) AddTestSuite(testsuite *Testsuite) { // testsuite is added prior to stat availability, stat management in the close of the testsuites diff --git a/pkg/test/case.go b/pkg/test/case.go index bf424299..68871723 100644 --- a/pkg/test/case.go +++ b/pkg/test/case.go @@ -316,19 +316,18 @@ func shortString(obj *corev1.ObjectReference) string { } // Run runs a test case including all of its steps. -func (t *Case) Run(test *testing.T, ts *report.Testsuite) { - setupReport := report.NewCase("setup") +func (t *Case) Run(test *testing.T, rep report.TestReporter) { + defer rep.Done() + setupReport := rep.Step("setup") ns, err := t.determineNamespace() if err != nil { - setupReport.Failure = report.NewFailure(err.Error(), nil) - ts.AddTestcase(setupReport) + setupReport.Failure(err.Error()) test.Fatal(err) } cl, err := t.Client(false) if err != nil { - setupReport.Failure = report.NewFailure(err.Error(), nil) - ts.AddTestcase(setupReport) + setupReport.Failure(err.Error()) test.Fatal(err) } @@ -341,8 +340,7 @@ func (t *Case) Run(test *testing.T, ts *report.Testsuite) { cl, err = newClient(testStep.Kubeconfig, testStep.Context)(false) if err != nil { - setupReport.Failure = report.NewFailure(err.Error(), nil) - ts.AddTestcase(setupReport) + setupReport.Failure(err.Error()) test.Fatal(err) } @@ -353,15 +351,13 @@ func (t *Case) Run(test *testing.T, ts *report.Testsuite) { if err = t.CreateNamespace(test, c, ns); k8serrors.IsAlreadyExists(err) { t.Logger.Logf("namespace %q already exists, using kubeconfig %q", ns.Name, kc) } else if err != nil { - setupReport.Failure = report.NewFailure("failed to create test namespace", []error{err}) - ts.AddTestcase(setupReport) + setupReport.Failure("failed to create test namespace", err) test.Fatal(err) } } - ts.AddTestcase(setupReport) for _, testStep := range t.Steps { - tc := report.NewCase("step " + testStep.String()) + stepReport := rep.Step("step " + testStep.String()) testStep.Client = t.Client if testStep.Kubeconfig != "" { testStep.Client = newClient(testStep.Kubeconfig, testStep.Context) @@ -371,8 +367,8 @@ func (t *Case) Run(test *testing.T, ts *report.Testsuite) { testStep.DiscoveryClient = newDiscoveryClient(testStep.Kubeconfig, testStep.Context) } testStep.Logger = t.Logger.WithPrefix(testStep.String()) - tc.Assertions += len(testStep.Asserts) - tc.Assertions += len(testStep.Errors) + stepReport.AddAssertions(len(testStep.Asserts)) + stepReport.AddAssertions(len(testStep.Errors)) errs := []error{} @@ -395,15 +391,12 @@ func (t *Case) Run(test *testing.T, ts *report.Testsuite) { if len(errs) > 0 { caseErr := fmt.Errorf("failed in step %s", testStep.String()) - tc.Failure = report.NewFailure(caseErr.Error(), errs) + stepReport.Failure(caseErr.Error(), errs...) test.Error(caseErr) for _, err := range errs { test.Error(err) } - } - ts.AddTestcase(tc) - if len(errs) > 0 { break } } diff --git a/pkg/test/case_integration_test.go b/pkg/test/case_integration_test.go index 83be4c89..cad01611 100644 --- a/pkg/test/case_integration_test.go +++ b/pkg/test/case_integration_test.go @@ -111,5 +111,14 @@ func TestMultiClusterCase(t *testing.T) { }, } - c.Run(t, &report.Testsuite{}) + c.Run(t, &noOpReporter{}) } + +type noOpReporter struct{} + +func (r *noOpReporter) Done() {} +func (r *noOpReporter) Step(string) report.StepReporter { + return r +} +func (r *noOpReporter) AddAssertions(int) {} +func (r *noOpReporter) Failure(string, ...error) {} diff --git a/pkg/test/harness.go b/pkg/test/harness.go index 86722578..51a23b44 100644 --- a/pkg/test/harness.go +++ b/pkg/test/harness.go @@ -397,8 +397,8 @@ func (h *Harness) RunTests() { t.Fatal(err) } - testReport := suiteReport.NewSubSuite(test.Name) - test.Run(t, testReport) + testReporter := suiteReport.NewTest(test.Name) + test.Run(t, testReporter) }) } }