Report broad failures in daily CI builds (elastic#12408)

Update ReportFailedTests to report daily CI builds with broad
failures creating GitHub issues, or updating them in case
there was an existing one. A broad failure depends on the value
of the environment variable CI_MAX_TESTS_REPORTED

dev/testsreporter/_static/description.tmpl

@@ -1,11 +1,11 @@
 {{ .summary }}
-{{ if ne .failure "" -}}
+{{ if and (ne .failure nil) (ne .failure "") -}}
 Failure:
 ```
 {{ .failure }}
 ```
 {{- end }}
-{{- if ne .error "" -}}
+{{- if and (ne .error nil) (ne .error "") -}}
 Error:
 ```
 {{ .error }}

dev/testsreporter/_static/summary.tmpl

@@ -7,11 +7,19 @@
 {{ if .logsDB -}}
 - LogsDB: enabled
 {{ end -}}
+{{ if and (ne .packageName "") (ne .packageName nil) -}}
 - Package: {{ .packageName }}
 - Failing test: {{ .testName }}
-{{ if ne .dataStream "" -}}
+{{ end -}}
+{{ if and (ne .dataStream "") (ne .dataStream nil) -}}
 - DataStream: {{ .dataStream }}
 {{ end -}}
+{{ if and (ne .packages nil) (ne (len .packages) 0) -}}
+- Packages:
+{{- range .packages }}
+    - {{ . }}
+{{- end }}
+{{ end -}}
 {{ if ne (len .owners) 0 -}}
 - Owners:
 {{- range .owners }}

dev/testsreporter/builderror.go

@@ -0,0 +1,117 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package testsreporter
+
+import (
+	"fmt"
+	"strings"
+)
+
+const (
+	buildReportingTeam      = "@elastic/ecosystem"
+	buildReportingTeamLabel = "Team:Ecosystem"
+)
+
+type dataError struct {
+	errorLinks
+	serverless        bool
+	serverlessProject string
+	logsDB            bool
+	stackVersion      string
+}
+
+type buildError struct {
+	dataError
+	teams    []string
+	packages []string
+}
+
+type buildErrorOptions struct {
+	Serverless        bool
+	ServerlessProject string
+	LogsDB            bool
+	StackVersion      string
+	Packages          []string
+	BuildURL          string
+	PreviousBuilds    []string
+	ClosedIssueURL    string
+}
+
+// Ensures that buildError implements failureObserver interface
+var _ failureObserver = new(buildError)
+
+func newBuildError(options buildErrorOptions) (*buildError, error) {
+	b := buildError{
+		dataError: dataError{
+			serverless:        options.Serverless,
+			serverlessProject: options.ServerlessProject,
+			logsDB:            options.LogsDB,
+			stackVersion:      options.StackVersion,
+			errorLinks: errorLinks{
+				firstBuild:     options.BuildURL,
+				closedIssueURL: options.ClosedIssueURL,
+				previousBuilds: options.PreviousBuilds,
+			},
+		},
+		packages: options.Packages,
+		teams:    []string{buildReportingTeam},
+	}
+
+	return &b, nil
+}
+
+func (b *buildError) String() string {
+	var sb strings.Builder
+
+	if b.logsDB {
+		sb.WriteString("[LogsDB] ")
+	}
+	if b.serverless {
+		sb.WriteString(fmt.Sprintf("[Serverless %s] ", b.serverlessProject))
+	}
+	if b.stackVersion != "" {
+		sb.WriteString("[Stack ")
+		sb.WriteString(b.stackVersion)
+		sb.WriteString("] ")
+	}
+	sb.WriteString("Too many packages failing in daily job")
+
+	return sb.String()
+}
+
+func (p *buildError) FirstBuild() string {
+	return p.errorLinks.firstBuild
+}
+
+func (p *buildError) UpdateLinks(links errorLinks) {
+	p.errorLinks = links
+}
+
+func (p *buildError) Teams() []string {
+	return p.teams
+}
+
+func (p *buildError) SummaryData() map[string]any {
+	return map[string]any{
+		"stackVersion":      p.stackVersion,
+		"serverless":        p.serverless,
+		"serverlessProject": p.serverlessProject,
+		"logsDB":            p.logsDB,
+		"packages":          p.packages,
+		"owners":            p.teams,
+	}
+}
+
+func (p *buildError) DescriptionData() map[string]any {
+	return map[string]any{
+		"firstBuild":     p.errorLinks.firstBuild,
+		"closedIssueURL": p.errorLinks.closedIssueURL,
+		"previousBuilds": p.errorLinks.previousBuilds,
+	}
+}
+
+func (p *buildError) Labels() []string {
+	return []string{buildReportingTeamLabel}
+}

dev/testsreporter/builderror_test.go

@@ -0,0 +1,76 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package testsreporter
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewBuildError(t *testing.T) {
+	cases := []struct {
+		title         string
+		options       buildErrorOptions
+		expectedError bool
+		expected      buildError
+	}{
+		{
+			title: "Sample build error",
+			options: buildErrorOptions{
+				Serverless:        true,
+				ServerlessProject: "observability",
+				LogsDB:            false,
+				StackVersion:      "8.16.0-SNAPSHOT",
+				Packages: []string{
+					"elastic_package_registry",
+					"nginx",
+				},
+				BuildURL:       "https://buildkite.com/elastic/integrations/build/10",
+				ClosedIssueURL: "https://github.com/elastic/integrations/issues/2",
+				PreviousBuilds: []string{
+					"https://buildkite.com/elastic/integrations/builds/1",
+					"https://buildkite.com/elastic/integrations/builds/3",
+				},
+			},
+			expectedError: false,
+			expected: buildError{
+				dataError: dataError{
+					serverless:        true,
+					serverlessProject: "observability",
+					logsDB:            false,
+					stackVersion:      "8.16.0-SNAPSHOT",
+					errorLinks: errorLinks{
+						firstBuild:     "https://buildkite.com/elastic/integrations/build/10",
+						closedIssueURL: "https://github.com/elastic/integrations/issues/2",
+						previousBuilds: []string{
+							"https://buildkite.com/elastic/integrations/builds/1",
+							"https://buildkite.com/elastic/integrations/builds/3",
+						},
+					},
+				},
+				packages: []string{
+					"elastic_package_registry",
+					"nginx",
+				},
+				teams: []string{"@elastic/ecosystem"},
+			},
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.title, func(t *testing.T) {
+			buildError, err := newBuildError(c.options)
+			if c.expectedError {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+
+			assert.Equal(t, c.expected, *buildError)
+		})
+	}
+
+}

dev/testsreporter/failureobserver.go

@@ -11,4 +11,5 @@ type failureObserver interface {
 	SummaryData() map[string]interface{}
 	DescriptionData() map[string]interface{}
 	String() string
+	Labels() []string
 }

dev/testsreporter/format.go

@@ -7,6 +7,7 @@ package testsreporter
 import (
 	"bytes"
 	_ "embed"
+	"fmt"
 	"strings"
 	"text/template"
 )
@@ -32,25 +33,36 @@ func (r resultsFormatter) Owners() []string {
 	return r.result.Teams()
 }
 
-func (r resultsFormatter) Summary() string {
+func (r resultsFormatter) Summary() (string, error) {
 	var rendered bytes.Buffer
 	templ := template.Must(template.New("summary").Parse(summaryTmpl))
-	templ.Execute(&rendered, r.result.SummaryData())
-
-	return rendered.String()
+	data := r.result.SummaryData()
+	err := templ.Execute(&rendered, data)
+	if err != nil {
+		return "", fmt.Errorf("failed to render summary: %w", err)
+	}
+	return rendered.String(), nil
 }
 
-func (r resultsFormatter) Description() string {
+func (r resultsFormatter) Description() (string, error) {
 	var rendered bytes.Buffer
 	templ := template.Must(template.New("description").Parse(descriptionTmpl))
 
+	summary, err := r.Summary()
+	if err != nil {
+		return "", err
+	}
+
 	data := r.result.DescriptionData()
-	data["summary"] = r.Summary()
+	data["summary"] = summary
 	data["maxPreviousLinks"] = r.maxPreviousLinks
 
-	templ.Execute(&rendered, data)
+	err = templ.Execute(&rendered, data)
+	if err != nil {
+		return "", fmt.Errorf("failed to render description: %w", err)
+	}
 
-	return rendered.String()
+	return rendered.String(), nil
 }
 
 func truncateText(message string, maxLength int) string {