Merge pull request #142 from instana/collect_http_headers

Collect custom HTTP headers

README.md

@@ -25,6 +25,7 @@ The Instana Go sensor consists of two parts:
   * [HTTP servers and clients](#http-servers-and-clients)
     * [Instrumenting HTTP request handling](#instrumenting-http-request-handling)
     * [Instrumenting HTTP request execution](#instrumenting-http-request-execution)
+    * [Capturing custom HTTP headers](#capturing-custom-http-headers)
   * [Database Calls](#database-calls)
     * [Instrumenting sql\.Open()](#instrumenting-sqlopen)
     * [Instrumenting sql\.OpenDB()](#instrumenting-sqlopendb)
@@ -56,7 +57,7 @@ func main() {
 }
 ```
 
-The `instana.InitSensor()` function takes an `*instana.Options` struct with the following optional fields:
+The `instana.InitSensor()` function takes an [`*instana.Options`][instana.Options] struct with the following optional fields:
 
 * **Service** - global service name that will be used to identify the program in the Instana backend
 * **AgentHost**, **AgentPort** - default to `localhost:42699`, set the coordinates of the Instana proxy agent
@@ -66,6 +67,7 @@ The `instana.InitSensor()` function takes an `*instana.Options` struct with the
 * **ForceTransmissionStartingAt** - the number of spans to collect before flushing the buffer to the agent
 * **MaxBufferedProfiles** - the maximum number of profiles to buffer
 * **IncludeProfilerFrames** - whether to include profiler calls into the profile or not
+* **Tracer** - [tracer-specific configuration][instana.TracerOptions] used by all tracers
 
 Once initialized, the sensor performs a host agent lookup using following list of addresses (in order of priority):
 
@@ -239,6 +241,31 @@ resp, err := client.Do(req.WithContext(ctx))
 
 The provided `parentSpan` is the incoming request from the request handler (see above) and provides the necessary tracing and span information to create a child span and inject it into the request.
 
+#### Capturing custom HTTP headers
+
+The HTTP instrumentation wrappers are capable of collecting HTTP headers and sending them along with the incoming/outgoing request spans. The list of case-insensitive header names can be provided both within `(instana.Options).Tracer.CollectableHTTPHeaders` field of the options object passed to `instana.InitSensor()` and in the [Host Agent Configuration file](https://www.instana.com/docs/setup_and_manage/host_agent/configuration/#capture-custom-http-headers). The latter setting takes precedence and requires agent Go trace plugin `com.instana.sensor-golang-trace` v1.3.0 and above:
+
+```go
+instana.InitSensor(&instana.Options{
+	// ...
+	Tracer: instana.TracerOptions{
+		// ...
+		CollectableHTTPHeaders: []string{"x-request-id", "x-loadtest-id"},
+	},
+})
+```
+
+This configuration is an equivalent of following settings in the [Host Agent Configuration file](https://www.instana.com/docs/setup_and_manage/host_agent/configuration/#capture-custom-http-headers):
+
+```
+com.instana.tracing:
+  extra-http-headers:
+    - 'x-request-id'
+    - 'x-loadtest-id'
+```
+
+By default the HTTP instrumentation does not collect any headers.
+
 ### Database Calls
 
 The Go sensor provides `instana.InstrumentSQLDriver()` and `instana.WrapSQLConnector()` (since Go v1.10+) to instrument SQL database calls made with `database/sql`. The tracer will then automatically capture the `Query` and `Exec` calls, gather information about the query, such as statement, execution time, etc. and forward them to be displayed as a part of the trace.
@@ -362,3 +389,5 @@ For more examples please consult the [godoc][godoc].
 [instana.TracingHandlerFunc]: https://pkg.go.dev/github.com/instana/go-sensor/?tab=doc#TracingHandlerFunc
 [instana.RoundTripper]: https://pkg.go.dev/github.com/instana/go-sensor/?tab=doc#RoundTripper
 [instana.HTTPSpanTags]: https://pkg.go.dev/github.com/instana/go-sensor/?tab=doc#HTTPSpanTags
+[instana.Options]: https://pkg.go.dev/github.com/instana/go-sensor/?tab=doc#Options
+[instana.TracerOptions]: https://pkg.go.dev/github.com/instana/go-sensor/?tab=doc#TracerOptions

agent.go

@@ -35,6 +35,7 @@ type agentResponse struct {
 		Matcher string   `json:"matcher"`
 		List    []string `json:"list"`
 	} `json:"secrets"`
+	ExtraHTTPHeaders []string `json:"extraHeaders"`
 }
 
 type discoveryS struct {
@@ -293,6 +294,8 @@ func (r *agentS) applyHostAgentSettings(resp agentResponse) {
 			sensor.options.Tracer.Secrets = m
 		}
 	}
+
+	sensor.options.Tracer.CollectableHTTPHeaders = resp.ExtraHTTPHeaders
 }
 
 func (r *agentS) setHost(host string) {

instrumentation_http.go

@@ -63,15 +63,34 @@ func TracingHandlerFunc(sensor *Sensor, pathTemplate string, handler http.Handle
 		span := tracer.StartSpan("g.http", opts...)
 		defer span.Finish()
 
+		var collectableHTTPHeaders []string
 		if t, ok := tracer.(Tracer); ok {
-			params := collectHTTPParams(req, t.Options().Secrets)
+			opts := t.Options()
+			collectableHTTPHeaders = opts.CollectableHTTPHeaders
+
+			params := collectHTTPParams(req, opts.Secrets)
 			if len(params) > 0 {
 				span.SetTag("http.params", params.Encode())
 			}
 		}
 
+		collectedHeaders := make(map[string]string)
+		// make sure collected headers are sent in case of panic/error
 		defer func() {
-			// Be sure to capture any kind of panic / error
+			if len(collectedHeaders) > 0 {
+				span.SetTag("http.header", collectedHeaders)
+			}
+		}()
+
+		// collect request headers
+		for _, h := range collectableHTTPHeaders {
+			if v := req.Header.Get(h); v != "" {
+				collectedHeaders[h] = v
+			}
+		}
+
+		defer func() {
+			// Be sure to capture any kind of panic/error
 			if err := recover(); err != nil {
 				if e, ok := err.(error); ok {
 					span.SetTag("http.error", e.Error())
@@ -94,6 +113,13 @@ func TracingHandlerFunc(sensor *Sensor, pathTemplate string, handler http.Handle
 		ctx = ContextWithSpan(ctx, span)
 		w3ctrace.TracingHandlerFunc(handler)(wrapped, req.WithContext(ctx))
 
+		// collect response headers
+		for _, h := range collectableHTTPHeaders {
+			if v := wrapped.Header().Get(h); v != "" {
+				collectedHeaders[h] = v
+			}
+		}
+
 		if wrapped.Status > 0 {
 			if wrapped.Status > http.StatusInternalServerError {
 				span.SetTag("http.error", http.StatusText(wrapped.Status))
@@ -136,20 +162,46 @@ func RoundTripper(sensor *Sensor, original http.RoundTripper) http.RoundTripper
 		req = cloneRequest(ContextWithSpan(ctx, span), req)
 		sensor.Tracer().Inject(span.Context(), ot.HTTPHeaders, ot.HTTPHeadersCarrier(req.Header))
 
+		var collectableHTTPHeaders []string
 		if t, ok := sensor.Tracer().(Tracer); ok {
-			params := collectHTTPParams(req, t.Options().Secrets)
+			opts := t.Options()
+			collectableHTTPHeaders = opts.CollectableHTTPHeaders
+
+			params := collectHTTPParams(req, opts.Secrets)
 			if len(params) > 0 {
 				span.SetTag("http.params", params.Encode())
 			}
 		}
 
+		collectedHeaders := make(map[string]string)
+		// make sure collected headers are sent in case of panic/error
+		defer func() {
+			if len(collectedHeaders) > 0 {
+				span.SetTag("http.header", collectedHeaders)
+			}
+		}()
+
+		// collect request headers
+		for _, h := range collectableHTTPHeaders {
+			if v := req.Header.Get(h); v != "" {
+				collectedHeaders[h] = v
+			}
+		}
+
 		resp, err := original.RoundTrip(req)
 		if err != nil {
 			span.SetTag("http.error", err.Error())
 			span.LogFields(otlog.Error(err))
 			return resp, err
 		}
 
+		// collect response headers
+		for _, h := range collectableHTTPHeaders {
+			if v := resp.Header.Get(h); v != "" {
+				collectedHeaders[h] = v
+			}
+		}
+
 		span.SetTag(string(ext.HTTPStatusCode), resp.StatusCode)
 
 		return resp, err

instrumentation_http_test.go

@@ -21,10 +21,14 @@ func TestTracingHandlerFunc_Write(t *testing.T) {
 	}, recorder))
 
 	h := instana.TracingHandlerFunc(s, "/{action}", func(w http.ResponseWriter, req *http.Request) {
+		w.Header().Set("X-Response", "true")
+		w.Header().Set("X-Custom-Header-2", "response")
 		fmt.Fprintln(w, "Ok")
 	})
 
 	req := httptest.NewRequest(http.MethodGet, "/test?q=term", nil)
+	req.Header.Set("Authorization", "Basic blah")
+	req.Header.Set("X-Custom-Header-1", "request")
 
 	rec := httptest.NewRecorder()
 	h.ServeHTTP(rec, req)
@@ -48,11 +52,15 @@ func TestTracingHandlerFunc_Write(t *testing.T) {
 	data := span.Data.(instana.HTTPSpanData)
 
 	assert.Equal(t, instana.HTTPSpanTags{
-		Host:         "example.com",
-		Status:       http.StatusOK,
-		Method:       "GET",
-		Path:         "/test",
-		Params:       "q=term",
+		Host:   "example.com",
+		Status: http.StatusOK,
+		Method: "GET",
+		Path:   "/test",
+		Params: "q=term",
+		Headers: map[string]string{
+			"x-custom-header-1": "request",
+			"x-custom-header-2": "response",
+		},
 		PathTemplate: "/{action}",
 	}, data.Tags)
 
@@ -337,11 +345,17 @@ func TestRoundTripper(t *testing.T) {
 		return &http.Response{
 			Status:     http.StatusText(http.StatusNotImplemented),
 			StatusCode: http.StatusNotImplemented,
+			Header: http.Header{
+				"X-Response":        []string{"true"},
+				"X-Custom-Header-2": []string{"response"},
+			},
 		}, nil
 	}))
 
 	ctx := instana.ContextWithSpan(context.Background(), parentSpan)
 	req := httptest.NewRequest("GET", "http://user:[email protected]/hello?q=term&sensitive_key=s3cr3t&myPassword=qwerty&SECRET_VALUE=1", nil)
+	req.Header.Set("X-Custom-Header-1", "request")
+	req.Header.Set("Authorization", "Basic blah")
 
 	_, err := rt.RoundTrip(req.WithContext(ctx))
 	require.NoError(t, err)
@@ -369,6 +383,10 @@ func TestRoundTripper(t *testing.T) {
 		Status: http.StatusNotImplemented,
 		URL:    "http://example.com/hello",
 		Params: "SECRET_VALUE=%3Credacted%3E&myPassword=%3Credacted%3E&q=term&sensitive_key=%3Credacted%3E",
+		Headers: map[string]string{
+			"x-custom-header-1": "request",
+			"x-custom-header-2": "response",
+		},
 	}, data.Tags)
 }
 

json_span.go

@@ -278,6 +278,8 @@ type HTTPSpanTags struct {
 	Path string `json:"path,omitempty"`
 	// Params are the request query string parameters
 	Params string `json:"params,omitempty"`
+	// Headers are the captured request/response headers
+	Headers map[string]string `json:"header,omitempty"`
 	// PathTemplate is the raw template string used to route the request
 	PathTemplate string `json:"path_tpl,omitempty"`
 	// The name:port of the host to which the request had been sent
@@ -303,6 +305,11 @@ func NewHTTPSpanTags(span *spanS) HTTPSpanTags {
 			readStringTag(&tags.Path, v)
 		case "http.params":
 			readStringTag(&tags.Params, v)
+		case "http.header":
+			if m, ok := v.(map[string]string); ok {
+				tags.Headers = m
+			}
+			tags.Headers = v.(map[string]string)
 		case "http.path_tpl":
 			readStringTag(&tags.PathTemplate, v)
 		case "http.host":

options.go

@@ -60,6 +60,6 @@ func (opts *Options) setDefaults() {
 	}
 
 	if opts.Tracer.Secrets == nil {
-		opts.Tracer = DefaultTracerOptions()
+		opts.Tracer.Secrets = DefaultSecretsMatcher()
 	}
 }

sensor_test.go

@@ -0,0 +1,21 @@
+package instana_test
+
+import (
+	"os"
+	"testing"
+
+	instana "github.com/instana/go-sensor"
+)
+
+const TestServiceName = "test_service"
+
+func TestMain(m *testing.M) {
+	instana.InitSensor(&instana.Options{
+		Service: TestServiceName,
+		Tracer: instana.TracerOptions{
+			CollectableHTTPHeaders: []string{"x-custom-header-1", "x-custom-header-2"},
+		},
+	})
+
+	os.Exit(m.Run())
+}

span_test.go

@@ -6,11 +6,11 @@ import (
 	"time"
 
 	instana "github.com/instana/go-sensor"
+	"github.com/instana/testify/assert"
+	"github.com/instana/testify/require"
 	ot "github.com/opentracing/opentracing-go"
 	ext "github.com/opentracing/opentracing-go/ext"
 	"github.com/opentracing/opentracing-go/log"
-	"github.com/instana/testify/assert"
-	"github.com/instana/testify/require"
 )
 
 func TestBasicSpan(t *testing.T) {
@@ -33,7 +33,7 @@ func TestBasicSpan(t *testing.T) {
 
 	require.IsType(t, instana.SDKSpanData{}, span.Data)
 	data := span.Data.(instana.SDKSpanData)
-	assert.Empty(t, data.Service)
+	assert.Equal(t, TestServiceName, data.Service)
 
 	assert.Equal(t, "test", data.Tags.Name)
 	assert.Nil(t, data.Tags.Custom["tags"])

tracer_options.go

@@ -14,6 +14,10 @@ type TracerOptions struct {
 	//
 	// See https://www.instana.com/docs/setup_and_manage/host_agent/configuration/#secrets for details
 	Secrets Matcher
+	// CollectableHTTPHeaders is a case-insensitive list of HTTP headers to be collected from HTTP requests and sent to the agent
+	//
+	// See https://www.instana.com/docs/setup_and_manage/host_agent/configuration/#capture-custom-http-headers for details
+	CollectableHTTPHeaders []string
 }
 
 // DefaultTracerOptions returns the default set of options to configure a tracer