Add additional transaction labels with error details to requests.

changelog/fragments/1701111919-Enrich-transaction-APM-errors.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: enhancement
+
+# Change summary; a 80ish characters long description of the change.
+summary: Enrich transaction APM errors
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; a word indicating the component this changeset affects.
+component:
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: 3098

internal/pkg/api/error.go

@@ -8,13 +8,15 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"net/http"
 	"os"
 	"strings"
 	"time"
 
 	"github.com/elastic/fleet-server/v7/internal/pkg/apikey"
 	"github.com/elastic/fleet-server/v7/internal/pkg/dl"
+	"github.com/elastic/fleet-server/v7/internal/pkg/es"
 	"github.com/elastic/fleet-server/v7/internal/pkg/file"
 	"github.com/elastic/fleet-server/v7/internal/pkg/file/delivery"
 	"github.com/elastic/fleet-server/v7/internal/pkg/file/uploader"
@@ -518,13 +520,26 @@ func (er HTTPErrResp) Write(w http.ResponseWriter) error {
 func ErrorResp(w http.ResponseWriter, r *http.Request, err error) {
 	zlog := hlog.FromRequest(r)
 	resp := NewHTTPErrResp(err)
-	e := zlog.WithLevel(resp.Level).Err(err).Int(ECSHTTPResponseCode, resp.StatusCode)
+	e := zlog.WithLevel(resp.Level).Err(err).Int(ECSHTTPResponseCode, resp.StatusCode).Str("error.type", fmt.Sprintf("%T", err))
 	if ts, ok := logger.CtxStartTime(r.Context()); ok {
 		e = e.Int64(ECSEventDuration, time.Since(ts).Nanoseconds())
 	}
 	e.Msg("HTTP request error")
 
 	if resp.StatusCode >= 500 {
+		if trans := apm.TransactionFromContext(r.Context()); trans != nil {
+			esErr := &es.ErrElastic{}
+			if errors.As(err, &esErr) {
+				trans.Context.SetLabel("error.type", "ErrElastic")
+				trans.Context.SetLabel("error.details.status", esErr.Status)
+				trans.Context.SetLabel("error.details.type", esErr.Type)
+				trans.Context.SetLabel("error.details.reason", esErr.Reason)
+				trans.Context.SetLabel("error.details.cause.type", esErr.Cause.Type)
+				trans.Context.SetLabel("error.details.cause.reason", esErr.Cause.Reason)
+			} else {
+				trans.Context.SetLabel("error.type", fmt.Sprintf("%T", err))
+			}
+		}
 		apm.CaptureError(r.Context(), err).Send()
 	}
 

internal/pkg/api/error_test.go

@@ -0,0 +1,94 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package api
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/elastic/fleet-server/v7/internal/pkg/es"
+	testlog "github.com/elastic/fleet-server/v7/internal/pkg/testing/log"
+	"github.com/stretchr/testify/require"
+	"go.elastic.co/apm/v2"
+	"go.elastic.co/apm/v2/apmtest"
+)
+
+func Test_ErrorResp(t *testing.T) {
+	tests := []struct {
+		name         string
+		err          error
+		expectedTags map[string]interface{}
+	}{{
+		name: "generic error",
+		err:  fmt.Errorf("generic error"),
+	}, {
+		name: "elastic error",
+		err:  &es.ErrElastic{},
+		expectedTags: map[string]interface{}{
+			"error_type": "ErrElastic",
+		},
+	}, {
+		name: "wrapped elastic error",
+		err:  fmt.Errorf("wrapped error: %w", &es.ErrElastic{}),
+		expectedTags: map[string]interface{}{
+			"error_type": "ErrElastic",
+		},
+	}}
+
+	tracer := apmtest.NewRecordingTracer()
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			logger := testlog.SetLogger(t)
+			tracer.ResetPayloads()
+
+			tx := tracer.StartTransaction("test", "test")
+			ctx := apm.ContextWithTransaction(context.Background(), tx)
+			ctx = logger.WithContext(ctx)
+
+			wr := httptest.NewRecorder()
+			req, err := http.NewRequestWithContext(ctx, "GET", "http://localhost", nil)
+			require.NoError(t, err)
+
+			ErrorResp(wr, req, tc.err)
+			tx.End()
+			ch := make(chan struct{}, 1)
+			tracer.Flush(ch)
+
+			payloads := tracer.Payloads()
+			require.Len(t, payloads.Transactions, 1)
+			require.Len(t, payloads.Errors, 1)
+
+			tags := make(map[string]interface{})
+			for _, tag := range payloads.Transactions[0].Context.Tags {
+				tags[tag.Key] = tag.Value
+			}
+			for k, v := range tc.expectedTags {
+				require.Contains(t, tags, k, "expected tag is missing")
+				require.Equal(t, v, tags[k], "expected tag value does not match")
+			}
+		})
+	}
+}
+
+func Test_ErrorResp_NoTransaction(t *testing.T) {
+	tracer := apmtest.NewRecordingTracer()
+	ctx := testlog.SetLogger(t).WithContext(context.Background())
+
+	wr := httptest.NewRecorder()
+	req, err := http.NewRequestWithContext(ctx, "GET", "http://localhost", nil)
+	require.NoError(t, err)
+
+	ErrorResp(wr, req, fmt.Errorf("some error"))
+	ch := make(chan struct{}, 1)
+	tracer.Flush(ch)
+
+	payloads := tracer.Payloads()
+	require.Len(t, payloads.Transactions, 0)
+	require.Len(t, payloads.Errors, 0)
+}