From 9f6e897e245d04826672a17b3166300370d6eb9a Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Wed, 12 Aug 2020 17:18:30 +0200 Subject: [PATCH 1/6] Add (instana.TracerOptions).CollectableHTTPHeaders --- options.go | 2 +- tracer_options.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/options.go b/options.go index ea4b36357..995a2320a 100644 --- a/options.go +++ b/options.go @@ -60,6 +60,6 @@ func (opts *Options) setDefaults() { } if opts.Tracer.Secrets == nil { - opts.Tracer = DefaultTracerOptions() + opts.Tracer.Secrets = DefaultSecretsMatcher() } } diff --git a/tracer_options.go b/tracer_options.go index d7a368dc4..340c52d8e 100644 --- a/tracer_options.go +++ b/tracer_options.go @@ -14,6 +14,10 @@ type TracerOptions struct { // // See https://www.instana.com/docs/setup_and_manage/host_agent/configuration/#secrets for details Secrets Matcher + // CollectableHTTPHeaders is a case-insensitive list of HTTP headers to be collected from HTTP requests and sent to the agent + // + // See https://www.instana.com/docs/setup_and_manage/host_agent/configuration/#capture-custom-http-headers for details + CollectableHTTPHeaders []string } // DefaultTracerOptions returns the default set of options to configure a tracer From 828ee7ac93a6192ec4d61ec1eceb55fcc927f1f1 Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Wed, 12 Aug 2020 17:18:41 +0200 Subject: [PATCH 2/6] Populate (instana.TracerOptions).CollectableHTTPHeaders from agent response --- agent.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/agent.go b/agent.go index 35eb2f973..d880ba17b 100644 --- a/agent.go +++ b/agent.go @@ -35,6 +35,7 @@ type agentResponse struct { Matcher string `json:"matcher"` List []string `json:"list"` } `json:"secrets"` + ExtraHTTPHeaders []string `json:"extraHeaders"` } type discoveryS struct { @@ -293,6 +294,8 @@ func (r *agentS) applyHostAgentSettings(resp agentResponse) { sensor.options.Tracer.Secrets = m } } + + sensor.options.Tracer.CollectableHTTPHeaders = resp.ExtraHTTPHeaders } func (r *agentS) setHost(host string) { From 286cc40e3ddc6324960b4ee97de52a4f0184ed8d Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Wed, 12 Aug 2020 17:47:31 +0200 Subject: [PATCH 3/6] Add `http.header` to the list of supported tags in HTTP spans --- json_span.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/json_span.go b/json_span.go index 53b3a888a..82fbe172b 100644 --- a/json_span.go +++ b/json_span.go @@ -278,6 +278,8 @@ type HTTPSpanTags struct { Path string `json:"path,omitempty"` // Params are the request query string parameters Params string `json:"params,omitempty"` + // Headers are the captured request/response headers + Headers map[string]string `json:"header,omitempty"` // PathTemplate is the raw template string used to route the request PathTemplate string `json:"path_tpl,omitempty"` // The name:port of the host to which the request had been sent @@ -303,6 +305,11 @@ func NewHTTPSpanTags(span *spanS) HTTPSpanTags { readStringTag(&tags.Path, v) case "http.params": readStringTag(&tags.Params, v) + case "http.header": + if m, ok := v.(map[string]string); ok { + tags.Headers = m + } + tags.Headers = v.(map[string]string) case "http.path_tpl": readStringTag(&tags.PathTemplate, v) case "http.host": From 9b06afaaad90563d56f96347e73021c6028779e3 Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Wed, 12 Aug 2020 12:03:22 +0200 Subject: [PATCH 4/6] Pre-initialize sensor before test runs --- sensor_test.go | 18 ++++++++++++++++++ span_test.go | 6 +++--- 2 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 sensor_test.go diff --git a/sensor_test.go b/sensor_test.go new file mode 100644 index 000000000..f7d423f15 --- /dev/null +++ b/sensor_test.go @@ -0,0 +1,18 @@ +package instana_test + +import ( + "os" + "testing" + + instana "github.com/instana/go-sensor" +) + +const TestServiceName = "test_service" + +func TestMain(m *testing.M) { + instana.InitSensor(&instana.Options{ + Service: TestServiceName, + }) + + os.Exit(m.Run()) +} diff --git a/span_test.go b/span_test.go index 343deafd9..31e38c4d5 100644 --- a/span_test.go +++ b/span_test.go @@ -6,11 +6,11 @@ import ( "time" instana "github.com/instana/go-sensor" + "github.com/instana/testify/assert" + "github.com/instana/testify/require" ot "github.com/opentracing/opentracing-go" ext "github.com/opentracing/opentracing-go/ext" "github.com/opentracing/opentracing-go/log" - "github.com/instana/testify/assert" - "github.com/instana/testify/require" ) func TestBasicSpan(t *testing.T) { @@ -33,7 +33,7 @@ func TestBasicSpan(t *testing.T) { require.IsType(t, instana.SDKSpanData{}, span.Data) data := span.Data.(instana.SDKSpanData) - assert.Empty(t, data.Service) + assert.Equal(t, TestServiceName, data.Service) assert.Equal(t, "test", data.Tags.Name) assert.Nil(t, data.Tags.Custom["tags"]) From 7fa630eb039f0f158bfa7aff8be1bdab3f0d6510 Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Wed, 12 Aug 2020 18:26:35 +0200 Subject: [PATCH 5/6] Collect and send extra HTTP headers to the agent --- instrumentation_http.go | 58 ++++++++++++++++++++++++++++++++++-- instrumentation_http_test.go | 28 +++++++++++++---- sensor_test.go | 3 ++ 3 files changed, 81 insertions(+), 8 deletions(-) diff --git a/instrumentation_http.go b/instrumentation_http.go index 583fa60cc..bc6959e4d 100644 --- a/instrumentation_http.go +++ b/instrumentation_http.go @@ -63,15 +63,34 @@ func TracingHandlerFunc(sensor *Sensor, pathTemplate string, handler http.Handle span := tracer.StartSpan("g.http", opts...) defer span.Finish() + var collectableHTTPHeaders []string if t, ok := tracer.(Tracer); ok { - params := collectHTTPParams(req, t.Options().Secrets) + opts := t.Options() + collectableHTTPHeaders = opts.CollectableHTTPHeaders + + params := collectHTTPParams(req, opts.Secrets) if len(params) > 0 { span.SetTag("http.params", params.Encode()) } } + collectedHeaders := make(map[string]string) + // make sure collected headers are sent in case of panic/error defer func() { - // Be sure to capture any kind of panic / error + if len(collectedHeaders) > 0 { + span.SetTag("http.header", collectedHeaders) + } + }() + + // collect request headers + for _, h := range collectableHTTPHeaders { + if v := req.Header.Get(h); v != "" { + collectedHeaders[h] = v + } + } + + defer func() { + // Be sure to capture any kind of panic/error if err := recover(); err != nil { if e, ok := err.(error); ok { span.SetTag("http.error", e.Error()) @@ -94,6 +113,13 @@ func TracingHandlerFunc(sensor *Sensor, pathTemplate string, handler http.Handle ctx = ContextWithSpan(ctx, span) w3ctrace.TracingHandlerFunc(handler)(wrapped, req.WithContext(ctx)) + // collect response headers + for _, h := range collectableHTTPHeaders { + if v := wrapped.Header().Get(h); v != "" { + collectedHeaders[h] = v + } + } + if wrapped.Status > 0 { if wrapped.Status > http.StatusInternalServerError { span.SetTag("http.error", http.StatusText(wrapped.Status)) @@ -136,13 +162,32 @@ func RoundTripper(sensor *Sensor, original http.RoundTripper) http.RoundTripper req = cloneRequest(ContextWithSpan(ctx, span), req) sensor.Tracer().Inject(span.Context(), ot.HTTPHeaders, ot.HTTPHeadersCarrier(req.Header)) + var collectableHTTPHeaders []string if t, ok := sensor.Tracer().(Tracer); ok { - params := collectHTTPParams(req, t.Options().Secrets) + opts := t.Options() + collectableHTTPHeaders = opts.CollectableHTTPHeaders + + params := collectHTTPParams(req, opts.Secrets) if len(params) > 0 { span.SetTag("http.params", params.Encode()) } } + collectedHeaders := make(map[string]string) + // make sure collected headers are sent in case of panic/error + defer func() { + if len(collectedHeaders) > 0 { + span.SetTag("http.header", collectedHeaders) + } + }() + + // collect request headers + for _, h := range collectableHTTPHeaders { + if v := req.Header.Get(h); v != "" { + collectedHeaders[h] = v + } + } + resp, err := original.RoundTrip(req) if err != nil { span.SetTag("http.error", err.Error()) @@ -150,6 +195,13 @@ func RoundTripper(sensor *Sensor, original http.RoundTripper) http.RoundTripper return resp, err } + // collect response headers + for _, h := range collectableHTTPHeaders { + if v := resp.Header.Get(h); v != "" { + collectedHeaders[h] = v + } + } + span.SetTag(string(ext.HTTPStatusCode), resp.StatusCode) return resp, err diff --git a/instrumentation_http_test.go b/instrumentation_http_test.go index 789bbc848..19c8a0f65 100644 --- a/instrumentation_http_test.go +++ b/instrumentation_http_test.go @@ -21,10 +21,14 @@ func TestTracingHandlerFunc_Write(t *testing.T) { }, recorder)) h := instana.TracingHandlerFunc(s, "/{action}", func(w http.ResponseWriter, req *http.Request) { + w.Header().Set("X-Response", "true") + w.Header().Set("X-Custom-Header-2", "response") fmt.Fprintln(w, "Ok") }) req := httptest.NewRequest(http.MethodGet, "/test?q=term", nil) + req.Header.Set("Authorization", "Basic blah") + req.Header.Set("X-Custom-Header-1", "request") rec := httptest.NewRecorder() h.ServeHTTP(rec, req) @@ -48,11 +52,15 @@ func TestTracingHandlerFunc_Write(t *testing.T) { data := span.Data.(instana.HTTPSpanData) assert.Equal(t, instana.HTTPSpanTags{ - Host: "example.com", - Status: http.StatusOK, - Method: "GET", - Path: "/test", - Params: "q=term", + Host: "example.com", + Status: http.StatusOK, + Method: "GET", + Path: "/test", + Params: "q=term", + Headers: map[string]string{ + "x-custom-header-1": "request", + "x-custom-header-2": "response", + }, PathTemplate: "/{action}", }, data.Tags) @@ -337,11 +345,17 @@ func TestRoundTripper(t *testing.T) { return &http.Response{ Status: http.StatusText(http.StatusNotImplemented), StatusCode: http.StatusNotImplemented, + Header: http.Header{ + "X-Response": []string{"true"}, + "X-Custom-Header-2": []string{"response"}, + }, }, nil })) ctx := instana.ContextWithSpan(context.Background(), parentSpan) req := httptest.NewRequest("GET", "http://user:password@example.com/hello?q=term&sensitive_key=s3cr3t&myPassword=qwerty&SECRET_VALUE=1", nil) + req.Header.Set("X-Custom-Header-1", "request") + req.Header.Set("Authorization", "Basic blah") _, err := rt.RoundTrip(req.WithContext(ctx)) require.NoError(t, err) @@ -369,6 +383,10 @@ func TestRoundTripper(t *testing.T) { Status: http.StatusNotImplemented, URL: "http://example.com/hello", Params: "SECRET_VALUE=%3Credacted%3E&myPassword=%3Credacted%3E&q=term&sensitive_key=%3Credacted%3E", + Headers: map[string]string{ + "x-custom-header-1": "request", + "x-custom-header-2": "response", + }, }, data.Tags) } diff --git a/sensor_test.go b/sensor_test.go index f7d423f15..3968971d2 100644 --- a/sensor_test.go +++ b/sensor_test.go @@ -12,6 +12,9 @@ const TestServiceName = "test_service" func TestMain(m *testing.M) { instana.InitSensor(&instana.Options{ Service: TestServiceName, + Tracer: instana.TracerOptions{ + CollectableHTTPHeaders: []string{"x-custom-header-1", "x-custom-header-2"}, + }, }) os.Exit(m.Run()) From c0a353eff83821deef7aef5c243e9a167a175c19 Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Thu, 13 Aug 2020 15:24:08 +0200 Subject: [PATCH 6/6] Document custom HTTP headers collection --- README.md | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bd9e928eb..b8c1a74e3 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ The Instana Go sensor consists of two parts: * [HTTP servers and clients](#http-servers-and-clients) * [Instrumenting HTTP request handling](#instrumenting-http-request-handling) * [Instrumenting HTTP request execution](#instrumenting-http-request-execution) + * [Capturing custom HTTP headers](#capturing-custom-http-headers) * [Database Calls](#database-calls) * [Instrumenting sql\.Open()](#instrumenting-sqlopen) * [Instrumenting sql\.OpenDB()](#instrumenting-sqlopendb) @@ -56,7 +57,7 @@ func main() { } ``` -The `instana.InitSensor()` function takes an `*instana.Options` struct with the following optional fields: +The `instana.InitSensor()` function takes an [`*instana.Options`][instana.Options] struct with the following optional fields: * **Service** - global service name that will be used to identify the program in the Instana backend * **AgentHost**, **AgentPort** - default to `localhost:42699`, set the coordinates of the Instana proxy agent @@ -66,6 +67,7 @@ The `instana.InitSensor()` function takes an `*instana.Options` struct with the * **ForceTransmissionStartingAt** - the number of spans to collect before flushing the buffer to the agent * **MaxBufferedProfiles** - the maximum number of profiles to buffer * **IncludeProfilerFrames** - whether to include profiler calls into the profile or not +* **Tracer** - [tracer-specific configuration][instana.TracerOptions] used by all tracers Once initialized, the sensor performs a host agent lookup using following list of addresses (in order of priority): @@ -239,6 +241,31 @@ resp, err := client.Do(req.WithContext(ctx)) The provided `parentSpan` is the incoming request from the request handler (see above) and provides the necessary tracing and span information to create a child span and inject it into the request. +#### Capturing custom HTTP headers + +The HTTP instrumentation wrappers are capable of collecting HTTP headers and sending them along with the incoming/outgoing request spans. The list of case-insensitive header names can be provided both within `(instana.Options).Tracer.CollectableHTTPHeaders` field of the options object passed to `instana.InitSensor()` and in the [Host Agent Configuration file](https://www.instana.com/docs/setup_and_manage/host_agent/configuration/#capture-custom-http-headers). The latter setting takes precedence and requires agent Go trace plugin `com.instana.sensor-golang-trace` v1.3.0 and above: + +```go +instana.InitSensor(&instana.Options{ + // ... + Tracer: instana.TracerOptions{ + // ... + CollectableHTTPHeaders: []string{"x-request-id", "x-loadtest-id"}, + }, +}) +``` + +This configuration is an equivalent of following settings in the [Host Agent Configuration file](https://www.instana.com/docs/setup_and_manage/host_agent/configuration/#capture-custom-http-headers): + +``` +com.instana.tracing: + extra-http-headers: + - 'x-request-id' + - 'x-loadtest-id' +``` + +By default the HTTP instrumentation does not collect any headers. + ### Database Calls The Go sensor provides `instana.InstrumentSQLDriver()` and `instana.WrapSQLConnector()` (since Go v1.10+) to instrument SQL database calls made with `database/sql`. The tracer will then automatically capture the `Query` and `Exec` calls, gather information about the query, such as statement, execution time, etc. and forward them to be displayed as a part of the trace. @@ -362,3 +389,5 @@ For more examples please consult the [godoc][godoc]. [instana.TracingHandlerFunc]: https://pkg.go.dev/github.com/instana/go-sensor/?tab=doc#TracingHandlerFunc [instana.RoundTripper]: https://pkg.go.dev/github.com/instana/go-sensor/?tab=doc#RoundTripper [instana.HTTPSpanTags]: https://pkg.go.dev/github.com/instana/go-sensor/?tab=doc#HTTPSpanTags +[instana.Options]: https://pkg.go.dev/github.com/instana/go-sensor/?tab=doc#Options +[instana.TracerOptions]: https://pkg.go.dev/github.com/instana/go-sensor/?tab=doc#TracerOptions