Merge pull request #141 from instana/filter_secrets

Collect HTTP request parameters

README.md

@@ -21,6 +21,7 @@ The Instana Go sensor consists of two parts:
 * [Common Operations](#common-operations)
   * [Setting the sensor log output](#setting-the-sensor-log-output)
   * [Trace Context Propagation](#trace-context-propagation)
+  * [Secrets Filtering](#secrets-filtering)
   * [HTTP servers and clients](#http-servers-and-clients)
     * [Instrumenting HTTP request handling](#instrumenting-http-request-handling)
     * [Instrumenting HTTP request execution](#instrumenting-http-request-execution)
@@ -177,8 +178,17 @@ func MyFunc(ctx context.Context) {
 }
 ```
 
+### Secrets Filtering
+
+Certain instrumentations provided by the Go sensor package, e.g. the [HTTP servers and clients](#http-servers-and-clients) wrappers, collect data that may contain sensitive information, such as passwords, keys and secrets. To avoid leaking these values the Go sensor replaces them with `<redacted>` before sending to the agent. The list of parameter name matchers is defined in `com.instana.secrets` section of the [Host Agent Configuration file](https://www.instana.com/docs/setup_and_manage/host_agent/configuration/#secrets) and will be sent to the in-app tracer during the announcement phase (requires agent Go trace plugin `com.instana.sensor-golang-trace` v1.3.0 and above).
+
+The default setting for the secrets matcher is `contains-ignore-case` with following list of terms: `key`, `password`, `secret`. This would redact the value of a parameter which name _contains_ any of these strings ignoring the case.
+
 ### HTTP servers and clients
 
+The Go sensor module provides instrumentation for clients and servers that use `net/http` package. Once activated (see below) this
+instrumentation automatically collects information about incoming and outgoing requests and sends it to the Instana agent. See the [instana.HTTPSpanTags][instana.HTTPSpanTags] documentation to learn which call details are collected.
+
 #### Instrumenting HTTP request handling
 
 With support to wrap a `http.HandlerFunc`, Instana quickly adds the possibility to trace requests and collect child spans, executed in the context of the request span.
@@ -351,3 +361,4 @@ For more examples please consult the [godoc][godoc].
 [pkg.go.dev]: https://pkg.go.dev/github.com/instana/go-sensor
 [instana.TracingHandlerFunc]: https://pkg.go.dev/github.com/instana/go-sensor/?tab=doc#TracingHandlerFunc
 [instana.RoundTripper]: https://pkg.go.dev/github.com/instana/go-sensor/?tab=doc#RoundTripper
+[instana.HTTPSpanTags]: https://pkg.go.dev/github.com/instana/go-sensor/?tab=doc#HTTPSpanTags

adapters.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"runtime"
 
+	"github.com/opentracing/opentracing-go"
 	ot "github.com/opentracing/opentracing-go"
 	"github.com/opentracing/opentracing-go/ext"
 	otlog "github.com/opentracing/opentracing-go/log"
@@ -13,6 +14,14 @@ import (
 type SpanSensitiveFunc func(span ot.Span)
 type ContextSensitiveFunc func(span ot.Span, ctx context.Context)
 
+// Tracer extends the opentracing.Tracer interface
+type Tracer interface {
+	opentracing.Tracer
+
+	// Options gets the current tracer options
+	Options() TracerOptions
+}
+
 // Sensor is used to inject tracing information into requests
 type Sensor struct {
 	tracer ot.Tracer

agent.go

@@ -29,8 +29,12 @@ const (
 )
 
 type agentResponse struct {
-	Pid    uint32 `json:"pid"`
-	HostID string `json:"agentUuid"`
+	Pid     uint32 `json:"pid"`
+	HostID  string `json:"agentUuid"`
+	Secrets struct {
+		Matcher string   `json:"matcher"`
+		List    []string `json:"list"`
+	} `json:"secrets"`
 }
 
 type discoveryS struct {
@@ -278,8 +282,17 @@ func (r *agentS) fullRequestResponse(url string, method string, data interface{}
 	return ret, err
 }
 
-func (r *agentS) setFrom(from *fromS) {
-	r.from = from
+func (r *agentS) applyHostAgentSettings(resp agentResponse) {
+	r.from = newHostAgentFromS(int(resp.Pid), resp.HostID)
+
+	if resp.Secrets.Matcher != "" {
+		m, err := NamedMatcher(resp.Secrets.Matcher, resp.Secrets.List)
+		if err != nil {
+			r.logger.Warn("failed to apply secrets matcher configuration: %s", err)
+		} else {
+			sensor.options.Tracer.Secrets = m
+		}
+	}
 }
 
 func (r *agentS) setHost(host string) {

fsm.go

@@ -119,26 +119,30 @@ func (r *fsmS) lookupSuccess(host string) {
 }
 
 func (r *fsmS) announceSensor(e *f.Event) {
-	cb := func(b bool, from *fromS) {
-		if b {
-			r.agent.logger.Info("Host agent available. We're in business. Announced pid:", from.EntityID)
-			r.agent.setFrom(from)
-			r.retries = maximumRetries
-			r.fsm.Event(eAnnounce)
-		} else {
+	cb := func(success bool, resp agentResponse) {
+		if !success {
 			r.agent.logger.Error("Cannot announce sensor. Scheduling retry.")
 			r.retries--
-			if r.retries > 0 {
-				r.scheduleRetry(e, r.announceSensor)
-			} else {
+			if r.retries == 0 {
 				r.fsm.Event(eInit)
+				return
 			}
+
+			r.scheduleRetry(e, r.announceSensor)
+
+			return
 		}
+
+		r.agent.logger.Info("Host agent available. We're in business. Announced pid:", resp.Pid)
+		r.agent.applyHostAgentSettings(resp)
+
+		r.retries = maximumRetries
+		r.fsm.Event(eAnnounce)
 	}
 
 	r.agent.logger.Debug("announcing sensor to the agent")
 
-	go func(cb func(b bool, from *fromS)) {
+	go func(cb func(success bool, resp agentResponse)) {
 		defer func() {
 			if err := recover(); err != nil {
 				r.agent.logger.Debug("Announce recovered:", err)
@@ -201,9 +205,9 @@ func (r *fsmS) announceSensor(e *f.Event) {
 			}
 		}
 
-		ret := &agentResponse{}
-		_, err := r.agent.requestResponse(r.agent.makeURL(agentDiscoveryURL), "PUT", d, ret)
-		cb(err == nil, newHostAgentFromS(int(ret.Pid), ret.HostID))
+		var resp agentResponse
+		_, err := r.agent.requestResponse(r.agent.makeURL(agentDiscoveryURL), "PUT", d, &resp)
+		cb(err == nil, resp)
 	}(cb)
 }
 

instrumentation_http.go

@@ -63,6 +63,13 @@ func TracingHandlerFunc(sensor *Sensor, pathTemplate string, handler http.Handle
 		span := tracer.StartSpan("g.http", opts...)
 		defer span.Finish()
 
+		if t, ok := tracer.(Tracer); ok {
+			params := collectHTTPParams(req, t.Options().Secrets)
+			if len(params) > 0 {
+				span.SetTag("http.params", params.Encode())
+			}
+		}
+
 		defer func() {
 			// Be sure to capture any kind of panic / error
 			if err := recover(); err != nil {
@@ -129,6 +136,13 @@ func RoundTripper(sensor *Sensor, original http.RoundTripper) http.RoundTripper
 		req = cloneRequest(ContextWithSpan(ctx, span), req)
 		sensor.Tracer().Inject(span.Context(), ot.HTTPHeaders, ot.HTTPHeadersCarrier(req.Header))
 
+		if t, ok := sensor.Tracer().(Tracer); ok {
+			params := collectHTTPParams(req, t.Options().Secrets)
+			if len(params) > 0 {
+				span.SetTag("http.params", params.Encode())
+			}
+		}
+
 		resp, err := original.RoundTrip(req)
 		if err != nil {
 			span.SetTag("http.error", err.Error())
@@ -167,6 +181,18 @@ func (rt tracingRoundTripper) RoundTrip(req *http.Request) (*http.Response, erro
 	return rt(req)
 }
 
+func collectHTTPParams(req *http.Request, matcher Matcher) url.Values {
+	params := cloneURLValues(req.URL.Query())
+
+	for k := range params {
+		if matcher.Match(k) {
+			params[k] = []string{"<redacted>"}
+		}
+	}
+
+	return params
+}
+
 // The following code is ported from $GOROOT/src/net/http/clone.go with minor changes
 // for compatibility with Go versions prior to 1.13
 //

instrumentation_http_test.go

@@ -24,7 +24,7 @@ func TestTracingHandlerFunc_Write(t *testing.T) {
 		fmt.Fprintln(w, "Ok")
 	})
 
-	req := httptest.NewRequest(http.MethodGet, "/test?q=classified", nil)
+	req := httptest.NewRequest(http.MethodGet, "/test?q=term", nil)
 
 	rec := httptest.NewRecorder()
 	h.ServeHTTP(rec, req)
@@ -52,6 +52,7 @@ func TestTracingHandlerFunc_Write(t *testing.T) {
 		Status:       http.StatusOK,
 		Method:       "GET",
 		Path:         "/test",
+		Params:       "q=term",
 		PathTemplate: "/{action}",
 	}, data.Tags)
 
@@ -69,7 +70,7 @@ func TestTracingHandlerFunc_WriteHeaders(t *testing.T) {
 	})
 
 	rec := httptest.NewRecorder()
-	h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/test?q=classified", nil))
+	h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/test?q=term", nil))
 
 	assert.Equal(t, http.StatusNotFound, rec.Code)
 
@@ -89,9 +90,57 @@ func TestTracingHandlerFunc_WriteHeaders(t *testing.T) {
 		Method: "GET",
 		Host:   "example.com",
 		Path:   "/test",
+		Params: "q=term",
 	}, data.Tags)
 }
 
+func TestTracingHandlerFunc_SecretsFiltering(t *testing.T) {
+	recorder := instana.NewTestRecorder()
+	s := instana.NewSensorWithTracer(instana.NewTracerWithEverything(&instana.Options{
+		Service: "go-sensor-test",
+	}, recorder))
+
+	h := instana.TracingHandlerFunc(s, "/{action}", func(w http.ResponseWriter, req *http.Request) {
+		fmt.Fprintln(w, "Ok")
+	})
+
+	req := httptest.NewRequest(http.MethodGet, "/test?q=term&sensitive_key=s3cr3t&myPassword=qwerty&SECRET_VALUE=1", nil)
+
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, req)
+
+	assert.Equal(t, http.StatusOK, rec.Code)
+	assert.Equal(t, "Ok\n", rec.Body.String())
+
+	spans := recorder.GetQueuedSpans()
+	require.Len(t, spans, 1)
+
+	span := spans[0]
+	assert.Equal(t, 0, span.Ec)
+	assert.EqualValues(t, instana.EntrySpanKind, span.Kind)
+	assert.False(t, span.Synthetic)
+	assert.Empty(t, span.CorrelationType)
+	assert.Empty(t, span.CorrelationID)
+
+	assert.Nil(t, span.ForeignParent)
+
+	require.IsType(t, instana.HTTPSpanData{}, span.Data)
+	data := span.Data.(instana.HTTPSpanData)
+
+	assert.Equal(t, instana.HTTPSpanTags{
+		Host:         "example.com",
+		Status:       http.StatusOK,
+		Method:       "GET",
+		Path:         "/test",
+		Params:       "SECRET_VALUE=%3Credacted%3E&myPassword=%3Credacted%3E&q=term&sensitive_key=%3Credacted%3E",
+		PathTemplate: "/{action}",
+	}, data.Tags)
+
+	// check whether the trace context has been sent back to the client
+	assert.Equal(t, instana.FormatID(span.TraceID), rec.Header().Get(instana.FieldT))
+	assert.Equal(t, instana.FormatID(span.SpanID), rec.Header().Get(instana.FieldS))
+}
+
 func TestTracingHandlerFunc_Error(t *testing.T) {
 	recorder := instana.NewTestRecorder()
 	s := instana.NewSensorWithTracer(instana.NewTracerWithEverything(&instana.Options{}, recorder))
@@ -249,7 +298,7 @@ func TestTracingHandlerFunc_PanicHandling(t *testing.T) {
 
 	rec := httptest.NewRecorder()
 	assert.Panics(t, func() {
-		h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/test?q=classified", nil))
+		h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/test?q=term", nil))
 	})
 
 	spans := recorder.GetQueuedSpans()
@@ -268,6 +317,7 @@ func TestTracingHandlerFunc_PanicHandling(t *testing.T) {
 		Method: "GET",
 		Host:   "example.com",
 		Path:   "/test",
+		Params: "q=term",
 		Error:  "something went wrong",
 	}, data.Tags)
 }
@@ -291,7 +341,7 @@ func TestRoundTripper(t *testing.T) {
 	}))
 
 	ctx := instana.ContextWithSpan(context.Background(), parentSpan)
-	req := httptest.NewRequest("GET", "http://user:[email protected]/hello", nil)
+	req := httptest.NewRequest("GET", "http://user:[email protected]/hello?q=term&sensitive_key=s3cr3t&myPassword=qwerty&SECRET_VALUE=1", nil)
 
 	_, err := rt.RoundTrip(req.WithContext(ctx))
 	require.NoError(t, err)
@@ -318,6 +368,7 @@ func TestRoundTripper(t *testing.T) {
 		Method: "GET",
 		Status: http.StatusNotImplemented,
 		URL:    "http://example.com/hello",
+		Params: "SECRET_VALUE=%3Credacted%3E&myPassword=%3Credacted%3E&q=term&sensitive_key=%3Credacted%3E",
 	}, data.Tags)
 }
 
@@ -353,7 +404,7 @@ func TestRoundTripper_Error(t *testing.T) {
 	}))
 
 	ctx := instana.ContextWithSpan(context.Background(), s.Tracer().StartSpan("parent"))
-	req := httptest.NewRequest("GET", "http://example.com/hello", nil)
+	req := httptest.NewRequest("GET", "http://example.com/hello?q=term&key=s3cr3t", nil)
 
 	_, err := rt.RoundTrip(req.WithContext(ctx))
 	assert.Error(t, err)
@@ -371,6 +422,7 @@ func TestRoundTripper_Error(t *testing.T) {
 	assert.Equal(t, instana.HTTPSpanTags{
 		Method: "GET",
 		URL:    "http://example.com/hello",
+		Params: "key=%3Credacted%3E&q=term",
 		Error:  "something went wrong",
 	}, data.Tags)
 }

json_span.go

@@ -276,6 +276,8 @@ type HTTPSpanTags struct {
 	Method string `json:"method,omitempty"`
 	// Path is the path part of the request URL
 	Path string `json:"path,omitempty"`
+	// Params are the request query string parameters
+	Params string `json:"params,omitempty"`
 	// PathTemplate is the raw template string used to route the request
 	PathTemplate string `json:"path_tpl,omitempty"`
 	// The name:port of the host to which the request had been sent
@@ -299,6 +301,8 @@ func NewHTTPSpanTags(span *spanS) HTTPSpanTags {
 			readStringTag(&tags.Method, v)
 		case "http.path":
 			readStringTag(&tags.Path, v)
+		case "http.params":
+			readStringTag(&tags.Params, v)
 		case "http.path_tpl":
 			readStringTag(&tags.PathTemplate, v)
 		case "http.host":