diff --git a/dev/testsreporter/_static/description.tmpl b/dev/testsreporter/_static/description.tmpl index 8cf8a67adb7..c3fe2ba9c57 100644 --- a/dev/testsreporter/_static/description.tmpl +++ b/dev/testsreporter/_static/description.tmpl @@ -1,11 +1,11 @@ {{ .summary }} -{{ if ne .failure "" -}} +{{ if and (ne .failure nil) (ne .failure "") -}} Failure: ``` {{ .failure }} ``` {{- end }} -{{- if ne .error "" -}} +{{- if and (ne .error nil) (ne .error "") -}} Error: ``` {{ .error }} diff --git a/dev/testsreporter/_static/summary.tmpl b/dev/testsreporter/_static/summary.tmpl index 8737ce7c76d..9e5bbe073d4 100644 --- a/dev/testsreporter/_static/summary.tmpl +++ b/dev/testsreporter/_static/summary.tmpl @@ -7,11 +7,19 @@ {{ if .logsDB -}} - LogsDB: enabled {{ end -}} +{{ if and (ne .packageName "") (ne .packageName nil) -}} - Package: {{ .packageName }} - Failing test: {{ .testName }} -{{ if ne .dataStream "" -}} +{{ end -}} +{{ if and (ne .dataStream "") (ne .dataStream nil) -}} - DataStream: {{ .dataStream }} {{ end -}} +{{ if and (ne .packages nil) (ne (len .packages) 0) -}} +- Packages: +{{- range .packages }} + - {{ . }} +{{- end }} +{{ end -}} {{ if ne (len .owners) 0 -}} - Owners: {{- range .owners }} diff --git a/dev/testsreporter/builderror.go b/dev/testsreporter/builderror.go new file mode 100644 index 00000000000..44865cf742b --- /dev/null +++ b/dev/testsreporter/builderror.go @@ -0,0 +1,117 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package testsreporter + +import ( + "fmt" + "strings" +) + +const ( + buildReportingTeam = "@elastic/ecosystem" + buildReportingTeamLabel = "Team:Ecosystem" +) + +type dataError struct { + errorLinks + serverless bool + serverlessProject string + logsDB bool + stackVersion string +} + +type buildError struct { + dataError + teams []string + packages []string +} + +type buildErrorOptions struct { + Serverless bool + ServerlessProject string + LogsDB bool + StackVersion string + Packages []string + BuildURL string + PreviousBuilds []string + ClosedIssueURL string +} + +// Ensures that buildError implements failureObserver interface +var _ failureObserver = new(buildError) + +func newBuildError(options buildErrorOptions) (*buildError, error) { + b := buildError{ + dataError: dataError{ + serverless: options.Serverless, + serverlessProject: options.ServerlessProject, + logsDB: options.LogsDB, + stackVersion: options.StackVersion, + errorLinks: errorLinks{ + firstBuild: options.BuildURL, + closedIssueURL: options.ClosedIssueURL, + previousBuilds: options.PreviousBuilds, + }, + }, + packages: options.Packages, + teams: []string{buildReportingTeam}, + } + + return &b, nil +} + +func (b *buildError) String() string { + var sb strings.Builder + + if b.logsDB { + sb.WriteString("[LogsDB] ") + } + if b.serverless { + sb.WriteString(fmt.Sprintf("[Serverless %s] ", b.serverlessProject)) + } + if b.stackVersion != "" { + sb.WriteString("[Stack ") + sb.WriteString(b.stackVersion) + sb.WriteString("] ") + } + sb.WriteString("Too many packages failing in daily job") + + return sb.String() +} + +func (p *buildError) FirstBuild() string { + return p.errorLinks.firstBuild +} + +func (p *buildError) UpdateLinks(links errorLinks) { + p.errorLinks = links +} + +func (p *buildError) Teams() []string { + return p.teams +} + +func (p *buildError) SummaryData() map[string]any { + return map[string]any{ + "stackVersion": p.stackVersion, + "serverless": p.serverless, + "serverlessProject": p.serverlessProject, + "logsDB": p.logsDB, + "packages": p.packages, + "owners": p.teams, + } +} + +func (p *buildError) DescriptionData() map[string]any { + return map[string]any{ + "firstBuild": p.errorLinks.firstBuild, + "closedIssueURL": p.errorLinks.closedIssueURL, + "previousBuilds": p.errorLinks.previousBuilds, + } +} + +func (p *buildError) Labels() []string { + return []string{buildReportingTeamLabel} +} diff --git a/dev/testsreporter/builderror_test.go b/dev/testsreporter/builderror_test.go new file mode 100644 index 00000000000..c39e7c4341d --- /dev/null +++ b/dev/testsreporter/builderror_test.go @@ -0,0 +1,76 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package testsreporter + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewBuildError(t *testing.T) { + cases := []struct { + title string + options buildErrorOptions + expectedError bool + expected buildError + }{ + { + title: "Sample build error", + options: buildErrorOptions{ + Serverless: true, + ServerlessProject: "observability", + LogsDB: false, + StackVersion: "8.16.0-SNAPSHOT", + Packages: []string{ + "elastic_package_registry", + "nginx", + }, + BuildURL: "https://buildkite.com/elastic/integrations/build/10", + ClosedIssueURL: "https://github.com/elastic/integrations/issues/2", + PreviousBuilds: []string{ + "https://buildkite.com/elastic/integrations/builds/1", + "https://buildkite.com/elastic/integrations/builds/3", + }, + }, + expectedError: false, + expected: buildError{ + dataError: dataError{ + serverless: true, + serverlessProject: "observability", + logsDB: false, + stackVersion: "8.16.0-SNAPSHOT", + errorLinks: errorLinks{ + firstBuild: "https://buildkite.com/elastic/integrations/build/10", + closedIssueURL: "https://github.com/elastic/integrations/issues/2", + previousBuilds: []string{ + "https://buildkite.com/elastic/integrations/builds/1", + "https://buildkite.com/elastic/integrations/builds/3", + }, + }, + }, + packages: []string{ + "elastic_package_registry", + "nginx", + }, + teams: []string{"@elastic/ecosystem"}, + }, + }, + } + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + buildError, err := newBuildError(c.options) + if c.expectedError { + require.Error(t, err) + return + } + require.NoError(t, err) + + assert.Equal(t, c.expected, *buildError) + }) + } + +} diff --git a/dev/testsreporter/failureobserver.go b/dev/testsreporter/failureobserver.go index cca86a77178..d78f7080308 100644 --- a/dev/testsreporter/failureobserver.go +++ b/dev/testsreporter/failureobserver.go @@ -11,4 +11,5 @@ type failureObserver interface { SummaryData() map[string]interface{} DescriptionData() map[string]interface{} String() string + Labels() []string } diff --git a/dev/testsreporter/format.go b/dev/testsreporter/format.go index ae6fa3935e5..06f4fade7c7 100644 --- a/dev/testsreporter/format.go +++ b/dev/testsreporter/format.go @@ -7,6 +7,7 @@ package testsreporter import ( "bytes" _ "embed" + "fmt" "strings" "text/template" ) @@ -32,25 +33,36 @@ func (r resultsFormatter) Owners() []string { return r.result.Teams() } -func (r resultsFormatter) Summary() string { +func (r resultsFormatter) Summary() (string, error) { var rendered bytes.Buffer templ := template.Must(template.New("summary").Parse(summaryTmpl)) - templ.Execute(&rendered, r.result.SummaryData()) - - return rendered.String() + data := r.result.SummaryData() + err := templ.Execute(&rendered, data) + if err != nil { + return "", fmt.Errorf("failed to render summary: %w", err) + } + return rendered.String(), nil } -func (r resultsFormatter) Description() string { +func (r resultsFormatter) Description() (string, error) { var rendered bytes.Buffer templ := template.Must(template.New("description").Parse(descriptionTmpl)) + summary, err := r.Summary() + if err != nil { + return "", err + } + data := r.result.DescriptionData() - data["summary"] = r.Summary() + data["summary"] = summary data["maxPreviousLinks"] = r.maxPreviousLinks - templ.Execute(&rendered, data) + err = templ.Execute(&rendered, data) + if err != nil { + return "", fmt.Errorf("failed to render description: %w", err) + } - return rendered.String() + return rendered.String(), nil } func truncateText(message string, maxLength int) string { diff --git a/dev/testsreporter/format_test.go b/dev/testsreporter/format_test.go index 4b572f20564..b4301d06ce6 100644 --- a/dev/testsreporter/format_test.go +++ b/dev/testsreporter/format_test.go @@ -8,20 +8,23 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestSummary(t *testing.T) { cases := []struct { - title string - packageError packageError - expected string + title string + resultError failureObserver + expected string }{ { title: "summary stack version with data stream", - packageError: packageError{ - stackVersion: "8.14", - packageName: "foo", - dataStream: "data", + resultError: &packageError{ + dataError: dataError{ + stackVersion: "8.14", + }, + packageName: "foo", + dataStream: "data", testCase: testCase{ Name: "mytest", }, @@ -34,9 +37,11 @@ func TestSummary(t *testing.T) { }, { title: "summary stack version with owners wihtout data stream", - packageError: packageError{ - stackVersion: "8.14", - packageName: "foo", + resultError: &packageError{ + dataError: dataError{ + stackVersion: "8.14", + }, + packageName: "foo", testCase: testCase{ Name: "mytest", }, @@ -52,10 +57,12 @@ func TestSummary(t *testing.T) { }, { title: "summary stack version with data stream and owners", - packageError: packageError{ - stackVersion: "8.14", - packageName: "foo", - dataStream: "data", + resultError: &packageError{ + dataError: dataError{ + stackVersion: "8.14", + }, + packageName: "foo", + dataStream: "data", testCase: testCase{ Name: "mytest", }, @@ -72,11 +79,13 @@ func TestSummary(t *testing.T) { }, { title: "summary serverless with data stream and owners", - packageError: packageError{ - serverless: true, - serverlessProject: "observability", - packageName: "foo", - dataStream: "data", + resultError: &packageError{ + dataError: dataError{ + serverless: true, + serverlessProject: "observability", + }, + packageName: "foo", + dataStream: "data", testCase: testCase{ Name: "mytest", }, @@ -93,10 +102,12 @@ func TestSummary(t *testing.T) { }, { title: "summary serverless with owners without data stream", - packageError: packageError{ - serverless: true, - serverlessProject: "observability", - packageName: "foo", + resultError: &packageError{ + dataError: dataError{ + serverless: true, + serverlessProject: "observability", + }, + packageName: "foo", testCase: testCase{ Name: "mytest", }, @@ -112,8 +123,10 @@ func TestSummary(t *testing.T) { }, { title: "summary logsdb", - packageError: packageError{ - logsDB: true, + resultError: &packageError{ + dataError: dataError{ + logsDB: true, + }, packageName: "foo", testCase: testCase{ Name: "mytest", @@ -126,6 +139,28 @@ func TestSummary(t *testing.T) { - Owners: - team1 - team2 +`, + }, + { + title: "summary broad failure", + resultError: &buildError{ + dataError: dataError{ + logsDB: false, + serverless: false, + stackVersion: "8.16", + }, + packages: []string{ + "foo", + "bar", + }, + teams: []string{"team1"}, + }, + expected: `- Stack version: 8.16 +- Packages: + - foo + - bar +- Owners: + - team1 `, }, } @@ -133,9 +168,10 @@ func TestSummary(t *testing.T) { for _, c := range cases { t.Run(c.title, func(t *testing.T) { formatter := resultsFormatter{ - result: &c.packageError, + result: c.resultError, } - summary := formatter.Summary() + summary, err := formatter.Summary() + require.NoError(t, err) assert.Equal(t, c.expected, summary) }) @@ -144,30 +180,32 @@ func TestSummary(t *testing.T) { func TestDescription(t *testing.T) { cases := []struct { - title string - summary string - packageError packageError - maxLinks int - expected string + title string + summary string + resultError failureObserver + maxLinks int + expected string }{ { title: "description error all fields", summary: "summary", - packageError: packageError{ - stackVersion: "8.14", - packageName: "foo", + resultError: &packageError{ + dataError: dataError{ + stackVersion: "8.14", + errorLinks: errorLinks{ + firstBuild: "http://link/1", + closedIssueURL: "http://link/old", + previousBuilds: []string{ + "http://link/2", + "http://link/3", + }, + }, + }, + packageName: "foo", testCase: testCase{ Name: "mytest", Error: "myerror", }, - errorLinks: errorLinks{ - firstBuild: "http://link/1", - closedIssueURL: "http://link/old", - previousBuilds: []string{ - "http://link/2", - "http://link/3", - }, - }, }, expected: `- Stack version: 8.14 - Package: foo @@ -190,21 +228,23 @@ Latest failed builds: { title: "description failure all fields", summary: "summary", - packageError: packageError{ - stackVersion: "8.14", - packageName: "foo", + resultError: &packageError{ + dataError: dataError{ + stackVersion: "8.14", + errorLinks: errorLinks{ + firstBuild: "http://link/1", + closedIssueURL: "http://link/old", + previousBuilds: []string{ + "http://link/2", + "http://link/3", + }, + }, + }, + packageName: "foo", testCase: testCase{ Name: "mytest", Failure: "myfailure", }, - errorLinks: errorLinks{ - firstBuild: "http://link/1", - closedIssueURL: "http://link/old", - previousBuilds: []string{ - "http://link/2", - "http://link/3", - }, - }, }, expected: `- Stack version: 8.14 - Package: foo @@ -227,20 +267,22 @@ Latest failed builds: { title: "description no closed issue", summary: "summary", - packageError: packageError{ - stackVersion: "8.14", - packageName: "foo", + resultError: &packageError{ + dataError: dataError{ + stackVersion: "8.14", + errorLinks: errorLinks{ + firstBuild: "http://link/1", + previousBuilds: []string{ + "http://link/2", + "http://link/3", + }, + }, + }, + packageName: "foo", testCase: testCase{ Name: "mytest", Error: "myerror", }, - errorLinks: errorLinks{ - firstBuild: "http://link/1", - previousBuilds: []string{ - "http://link/2", - "http://link/3", - }, - }, }, expected: `- Stack version: 8.14 - Package: foo @@ -262,20 +304,22 @@ Latest failed builds: title: "description max links", summary: "summary", maxLinks: 2, - packageError: packageError{ - stackVersion: "8.14", - packageName: "foo", + resultError: &packageError{ + dataError: dataError{ + stackVersion: "8.14", + errorLinks: errorLinks{ + firstBuild: "http://link/1", + previousBuilds: []string{ + "http://link/2", + "http://link/3", + }, + }, + }, + packageName: "foo", testCase: testCase{ Name: "mytest", Error: "myerror", }, - errorLinks: errorLinks{ - firstBuild: "http://link/1", - previousBuilds: []string{ - "http://link/2", - "http://link/3", - }, - }, }, expected: `- Stack version: 8.14 - Package: foo @@ -291,6 +335,46 @@ First build failed: http://link/1 Latest 2 failed builds: - http://link/2 - http://link/3 +`, + }, + { + title: "description broad failure", + resultError: &buildError{ + dataError: dataError{ + logsDB: false, + serverless: false, + stackVersion: "8.16", + errorLinks: errorLinks{ + firstBuild: "http://link/1", + previousBuilds: []string{ + "http://link/2", + "http://link/3", + }, + closedIssueURL: "http://issue.link/1", + }, + }, + packages: []string{ + "foo", + "bar", + }, + teams: []string{"team1"}, + }, + expected: `- Stack version: 8.16 +- Packages: + - foo + - bar +- Owners: + - team1 + + + +Latest issue closed for the same test: http://issue.link/1 + +First build failed: http://link/1 + +Latest failed builds: +- http://link/2 +- http://link/3 `, }, } @@ -298,10 +382,11 @@ Latest 2 failed builds: for _, c := range cases { t.Run(c.title, func(t *testing.T) { formatter := resultsFormatter{ - result: &c.packageError, + result: c.resultError, maxPreviousLinks: c.maxLinks, } - description := formatter.Description() + description, err := formatter.Description() + require.NoError(t, err) assert.Equal(t, c.expected, description) }) diff --git a/dev/testsreporter/githubissue.go b/dev/testsreporter/githubissue.go index f1dcdf0b7bb..92f00ecd0e7 100644 --- a/dev/testsreporter/githubissue.go +++ b/dev/testsreporter/githubissue.go @@ -26,6 +26,10 @@ func (i *githubIssue) URL() string { return i.url } +func (i *githubIssue) Labels() []string { + return i.labels +} + type githubIssueOptions struct { Repository string Title string @@ -53,3 +57,7 @@ func newGithubIssue(options githubIssueOptions) *githubIssue { func (i *githubIssue) SetDescription(description string) { i.description = description } + +func (i *githubIssue) AddLabels(labels []string) { + i.labels = append(i.labels, labels...) +} diff --git a/dev/testsreporter/packageerror.go b/dev/testsreporter/packageerror.go index f1dfcec9f82..aa775ed381a 100644 --- a/dev/testsreporter/packageerror.go +++ b/dev/testsreporter/packageerror.go @@ -20,14 +20,10 @@ type errorLinks struct { type packageError struct { testCase - errorLinks - serverless bool - serverlessProject string - logsDB bool - stackVersion string - teams []string - packageName string - dataStream string + dataError + teams []string + packageName string + dataStream string } type packageErrorOptions struct { @@ -43,20 +39,24 @@ type packageErrorOptions struct { Teams []string } +// Ensures that packageError implements failureObserver interface +var _ failureObserver = new(packageError) + func newPackageError(options packageErrorOptions) (*packageError, error) { p := packageError{ - serverless: options.Serverless, - serverlessProject: options.ServerlessProject, - logsDB: options.LogsDB, - stackVersion: options.StackVersion, - testCase: options.TestCase, - teams: options.Teams, - - errorLinks: errorLinks{ - firstBuild: options.BuildURL, - closedIssueURL: options.ClosedIssueURL, - previousBuilds: options.PreviousBuilds, + dataError: dataError{ + serverless: options.Serverless, + serverlessProject: options.ServerlessProject, + logsDB: options.LogsDB, + stackVersion: options.StackVersion, + errorLinks: errorLinks{ + firstBuild: options.BuildURL, + closedIssueURL: options.ClosedIssueURL, + previousBuilds: options.PreviousBuilds, + }, }, + testCase: options.TestCase, + teams: options.Teams, } p.packageName = p.testCase.PackageName() @@ -130,3 +130,7 @@ func (p *packageError) DescriptionData() map[string]any { "previousBuilds": p.errorLinks.previousBuilds, } } + +func (p *packageError) Labels() []string { + return nil +} diff --git a/dev/testsreporter/packageerror_test.go b/dev/testsreporter/packageerror_test.go index 4cdb7bf12a3..55c0abdd3a9 100644 --- a/dev/testsreporter/packageerror_test.go +++ b/dev/testsreporter/packageerror_test.go @@ -35,21 +35,23 @@ func TestNewPackageError(t *testing.T) { }, expectedError: false, expected: packageError{ + dataError: dataError{ + serverless: true, + serverlessProject: "observability", + logsDB: false, + stackVersion: "8.16.0-SNAPSHOT", + errorLinks: errorLinks{ + firstBuild: "https://buildkite.com/elastic/integrations/build/1", + }, + }, testCase: testCase{ Name: "failing test", ClassName: "elastic_package_registry.datastream", Error: "could not find hits", }, - serverless: true, - serverlessProject: "observability", - logsDB: false, - stackVersion: "8.16.0-SNAPSHOT", - packageName: "elastic_package_registry", - dataStream: "datastream", - teams: []string{"@elastic/ecosystem"}, - errorLinks: errorLinks{ - firstBuild: "https://buildkite.com/elastic/integrations/build/1", - }, + packageName: "elastic_package_registry", + dataStream: "datastream", + teams: []string{"@elastic/ecosystem"}, }, }, { @@ -69,21 +71,23 @@ func TestNewPackageError(t *testing.T) { }, expectedError: false, expected: packageError{ + dataError: dataError{ + serverless: true, + serverlessProject: "observability", + logsDB: false, + stackVersion: "8.16.0-SNAPSHOT", + errorLinks: errorLinks{ + firstBuild: "https://buildkite.com/elastic/integrations/build/1", + }, + }, testCase: testCase{ Name: "failing test", ClassName: "elastic_package_registry", Error: "could not find hits", }, - serverless: true, - serverlessProject: "observability", - logsDB: false, - stackVersion: "8.16.0-SNAPSHOT", - packageName: "elastic_package_registry", - dataStream: "", - teams: []string{"@elastic/ecosystem"}, - errorLinks: errorLinks{ - firstBuild: "https://buildkite.com/elastic/integrations/build/1", - }, + packageName: "elastic_package_registry", + dataStream: "", + teams: []string{"@elastic/ecosystem"}, }, }, { diff --git a/dev/testsreporter/reporter.go b/dev/testsreporter/reporter.go index 306127013f5..dcec3614923 100644 --- a/dev/testsreporter/reporter.go +++ b/dev/testsreporter/reporter.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "regexp" + "strings" ) type reporter struct { @@ -22,23 +23,40 @@ func newReporter(ghCli *ghCli, maxPreviousLinks int) reporter { } } -func (r reporter) Report(ctx context.Context, issue *githubIssue, packageError failureObserver) error { - links, nextIssue, err := r.updateLinks(ctx, issue, packageError.FirstBuild()) +func (r reporter) Report(ctx context.Context, issue *githubIssue, resultError failureObserver) error { + links, nextIssue, err := r.updateLinks(ctx, issue, resultError.FirstBuild()) if err != nil { return fmt.Errorf("failed to update links from the error: %w", err) } - packageError.UpdateLinks(*links) + resultError.UpdateLinks(*links) formatter := resultsFormatter{ - result: packageError, + result: resultError, maxPreviousLinks: r.maxPreviousLinks, } - nextIssue.SetDescription(formatter.Description()) + description, err := formatter.Description() + if err != nil { + return err + } + + summary, err := formatter.Summary() + if err != nil { + return fmt.Errorf("failed to render issue summary: %w", err) + } + + nextIssue.SetDescription(description) + nextIssue.AddLabels(resultError.Labels()) - // TEST - TO REMOVE - fmt.Printf("Description of the issue:\n%s", formatter.Description()) + fmt.Println() + fmt.Println("---- Issue ----") + fmt.Printf("Title: %q\n", formatter.Title()) + fmt.Printf("Teams: %q\n", strings.Join(formatter.Owners(), ", ")) + fmt.Printf("Labels: %s\n", strings.Join(nextIssue.Labels(), ", ")) + fmt.Printf("Summary:\n%s", summary) + fmt.Println("----") + fmt.Println() return r.createOrUpdateIssue(ctx, nextIssue) } diff --git a/dev/testsreporter/testsreporter.go b/dev/testsreporter/testsreporter.go index 985930ed162..fd33819982d 100644 --- a/dev/testsreporter/testsreporter.go +++ b/dev/testsreporter/testsreporter.go @@ -10,7 +10,7 @@ import ( "fmt" "os" "path/filepath" - "strings" + "sort" "github.com/elastic/integrations/dev/codeowners" ) @@ -29,7 +29,7 @@ type CheckOptions struct { DryRun bool } -func Check(resultsPath string, options CheckOptions) error { +func Check(ctx context.Context, resultsPath string, options CheckOptions) error { if options.CodeownersPath == "" { // set default value for the GitHub CODEOWNERS file options.CodeownersPath = codeowners.DefaultCodeownersPath @@ -45,48 +45,57 @@ func Check(resultsPath string, options CheckOptions) error { return err } - if len(packageErrors) > options.MaxTestsReported { - fmt.Printf("Skip creating GitHub issues, hit the maximum number (%d) of tests to be reported. Total failing tests: %d.\n", options.MaxTestsReported, len(packageErrors)) - return nil - } - ghCli := newGhCli(githubOptions{ DryRun: options.DryRun, }) aReporter := newReporter(ghCli, options.MaxPreviousLinks) + if len(packageErrors) > options.MaxTestsReported { + fmt.Printf("Skip creating GitHub issues, hit the maximum number (%d) of tests to be reported. Total failing tests: %d.\n", options.MaxTestsReported, len(packageErrors)) + packages, err := packagesFromTests(resultsPath, options) + if err != nil { + return fmt.Errorf("failed to get packages from results files: %w", err) + } + bError, err := newBuildError(buildErrorOptions{ + Serverless: options.Serverless, + ServerlessProject: options.ServerlessProject, + LogsDB: options.LogsDB, + StackVersion: options.StackVersion, + BuildURL: options.BuildURL, + Packages: packages, + }) + if err != nil { + return fmt.Errorf("failed to create the build information error: %w", err) + } + + ghIssue, err := createInitialIssue(bError, options.MaxPreviousLinks) + if err != nil { + return fmt.Errorf("failed to create initial issue: %w", err) + } + + if err := aReporter.Report(ctx, ghIssue, bError); err != nil { + return err + } + return nil + } + var multiErr error for _, pError := range packageErrors { - ctx := context.TODO() - r := resultsFormatter{ - result: &pError, - maxPreviousLinks: options.MaxPreviousLinks, - } - fmt.Println() - fmt.Println("---- Issue ----") - fmt.Printf("Title: %q\n", r.Title()) - fmt.Printf("Teams: %q\n", strings.Join(r.Owners(), ", ")) - fmt.Printf("Summary:\n%s\n", r.Summary()) - fmt.Println("----") - fmt.Println() - - ghIssue := newGithubIssue(githubIssueOptions{ - Title: r.Title(), - Description: r.Description(), - Labels: []string{"flaky-test", "automation"}, - Repository: "elastic/integrations", - }) + ghIssue, err := createInitialIssue(pError, options.MaxPreviousLinks) + if err != nil { + return fmt.Errorf("failed to create initial issue: %w", err) + } - if err := aReporter.Report(ctx, ghIssue, &pError); err != nil { + if err := aReporter.Report(ctx, ghIssue, pError); err != nil { multiErr = errors.Join(multiErr, err) } } return multiErr } -func errorsFromTests(resultsPath string, options CheckOptions) ([]packageError, error) { - var packageErrors []packageError +func errorsFromTests(resultsPath string, options CheckOptions) ([]*packageError, error) { + var packageErrors []*packageError err := filepath.Walk(resultsPath, func(path string, info os.FileInfo, err error) error { if err != nil { return err @@ -115,7 +124,7 @@ func errorsFromTests(resultsPath string, options CheckOptions) ([]packageError, if err != nil { return fmt.Errorf("failed to create package error: %w", err) } - packageErrors = append(packageErrors, *packageError) + packageErrors = append(packageErrors, packageError) } return nil @@ -126,3 +135,55 @@ func errorsFromTests(resultsPath string, options CheckOptions) ([]packageError, return packageErrors, nil } + +// packagesFromTests returns the sorted packages failing given the results file +func packagesFromTests(resultsPath string, options CheckOptions) ([]string, error) { + packages := []string{} + err := filepath.Walk(resultsPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if filepath.Ext(path) != ".xml" { + return nil + } + if info.IsDir() { + return nil + } + cases, err := testFailures(path) + if err != nil { + return err + } + if len(cases) > 0 { + name := cases[0].PackageName() + packages = append(packages, name) + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to look for errors: %w", err) + } + + sort.Strings(packages) + + return packages, nil +} + +func createInitialIssue(resultError failureObserver, maxPreviousLinks int) (*githubIssue, error) { + r := resultsFormatter{ + result: resultError, + maxPreviousLinks: maxPreviousLinks, + } + + description, err := r.Description() + if err != nil { + return nil, fmt.Errorf("failed to render initial description: %w", err) + } + issue := newGithubIssue(githubIssueOptions{ + Title: r.Title(), + Description: description, + Labels: []string{"flaky-test", "automation"}, + Repository: "elastic/integrations", + }) + return issue, nil +} diff --git a/dev/testsreporter/testsreporter_test.go b/dev/testsreporter/testsreporter_test.go index 27c38226e2b..cfc6b944707 100644 --- a/dev/testsreporter/testsreporter_test.go +++ b/dev/testsreporter/testsreporter_test.go @@ -15,13 +15,16 @@ func TestErrorsFromTest(t *testing.T) { cases := []struct { title string xmlFolder string - expected []packageError + expected []*packageError }{ { title: "read XML files", xmlFolder: "testdata", - expected: []packageError{ + expected: []*packageError{ { + dataError: dataError{ + serverless: false, + }, testCase: testCase{ Name: "system test: default", ClassName: "cisco_umbrella.log", @@ -31,9 +34,11 @@ func TestErrorsFromTest(t *testing.T) { teams: []string{"@elastic/security-service-integrations"}, dataStream: "log", packageName: "cisco_umbrella", - serverless: false, }, { + dataError: dataError{ + serverless: false, + }, testCase: testCase{ Name: "system test: default", ClassName: "elastic_package_registry.metrics", @@ -43,9 +48,11 @@ func TestErrorsFromTest(t *testing.T) { teams: []string{"@elastic/ecosystem"}, dataStream: "metrics", packageName: "elastic_package_registry", - serverless: false, }, { + dataError: dataError{ + serverless: false, + }, testCase: testCase{ Name: "pipeline test: test-fortinet-7-4.log", ClassName: "fortinet_fortigate.log", @@ -65,9 +72,11 @@ func TestErrorsFromTest(t *testing.T) { packageName: "fortinet_fortigate", dataStream: "log", teams: []string{"@elastic/sec-deployment-and-devices"}, - serverless: false, }, { + dataError: dataError{ + serverless: false, + }, testCase: testCase{ Name: "system test: mssql", ClassName: "sql_input.", @@ -77,9 +86,11 @@ func TestErrorsFromTest(t *testing.T) { packageName: "sql_input", dataStream: "", teams: []string{"@elastic/obs-infraobs-integrations"}, - serverless: false, }, { + dataError: dataError{ + serverless: false, + }, testCase: testCase{ Name: "system test: mysql", ClassName: "sql_input.", @@ -89,7 +100,6 @@ func TestErrorsFromTest(t *testing.T) { packageName: "sql_input", dataStream: "", teams: []string{"@elastic/obs-infraobs-integrations"}, - serverless: false, }, }, }, diff --git a/magefile.go b/magefile.go index 0a90f3908ff..5ff8fa9301a 100644 --- a/magefile.go +++ b/magefile.go @@ -7,6 +7,7 @@ package main import ( + "context" "fmt" "io" "os" @@ -148,7 +149,7 @@ func ModTidy() error { return sh.RunV("go", "mod", "tidy") } -func ReportFailedTests(testResultsFolder string) error { +func ReportFailedTests(ctx context.Context, testResultsFolder string) error { stackVersion := os.Getenv("STACK_VERSION") serverlessEnv := os.Getenv("SERVERLESS") dryRunEnv := os.Getenv("DRY_RUN") @@ -201,5 +202,5 @@ func ReportFailedTests(testResultsFolder string) error { MaxTestsReported: maxIssues, DryRun: dryRun, } - return testsreporter.Check(testResultsFolder, options) + return testsreporter.Check(ctx, testResultsFolder, options) }