From 4cd3ff46734e13783f01f2f65fb3d39d017bdec0 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 16 May 2024 08:56:33 +0930 Subject: [PATCH] x-pack/filebeat/input/cel: add default user-agent to http requests --- CHANGELOG.next.asciidoc | 1 + .../filebeat/docs/inputs/input-cel.asciidoc | 2 +- x-pack/filebeat/input/cel/input.go | 20 ++- x-pack/filebeat/input/cel/input_test.go | 140 +++++++++++++++++- 4 files changed, 159 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2b870c03f99..c460560da8b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -267,6 +267,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Update CEL mito extensions to v1.11.0 to improve type checking. {pull}39460[39460] - Improve logging of request and response with request trace logging in error conditions. {pull}39455[39455] - Add HTTP metrics to CEL input. {issue}39501[39501] {pull}39503[39503] +- Add default user-agent to CEL HTTP requests. {issue}39502[39502] {pull}39587[39587] *Auditbeat* diff --git a/x-pack/filebeat/docs/inputs/input-cel.asciidoc b/x-pack/filebeat/docs/inputs/input-cel.asciidoc index 7ec869e42cc..3c77f750c11 100644 --- a/x-pack/filebeat/docs/inputs/input-cel.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-cel.asciidoc @@ -242,7 +242,7 @@ As noted above the `cel` input provides functions, macros, and global variables * {mito_docs}/lib#Debug[Debug] — the debug handler registers a logger with the name extension `cel_debug` and calls to the CEL `debug` function are emitted to that logger. ** {mito_docs}/lib#hdr-Debug[Debug] -In addition to the extensions provided in the packages listed above, a global variable `useragent` is also provided which gives the user CEL program access to the {beatname_lc} user-agent string. +In addition to the extensions provided in the packages listed above, a global variable `useragent` is also provided which gives the user CEL program access to the {beatname_lc} user-agent string. By default, this value is assigned to all requests' user-agent headers unless the CEL program has already set the user-agent header value. Programs wishing to not provide a user-agent, should set this header to the empty string, `""`. The CEL environment enables the https://pkg.go.dev/github.com/google/cel-go/cel#OptionalTypes[optional types] library using the version defined {mito_docs}/lib#OptionalTypesVersion[here]. diff --git a/x-pack/filebeat/input/cel/input.go b/x-pack/filebeat/input/cel/input.go index 759809e6e80..f0795377b0e 100644 --- a/x-pack/filebeat/input/cel/input.go +++ b/x-pack/filebeat/input/cel/input.go @@ -64,7 +64,8 @@ const ( root = "state" ) -// The Filebeat user-agent is provided to the program as useragent. +// The Filebeat user-agent is provided to the program as useragent. If a request +// is not given a user-agent string, this user agent is added to the request. var userAgent = useragent.UserAgent("Filebeat", version.GetDefaultVersion(), version.Commit(), version.BuildTime().String()) func Plugin(log *logp.Logger, store inputcursor.StateStore) v2.Plugin { @@ -758,6 +759,11 @@ func newClient(ctx context.Context, cfg config, log *logp.Logger, reg *monitorin return authClient, trace, nil } + c.Transport = userAgentDecorator{ + UserAgent: userAgent, + Transport: c.Transport, + } + return c, trace, nil } @@ -856,6 +862,18 @@ func retryErrorHandler(max int, log *logp.Logger) retryablehttp.ErrorHandler { } } +type userAgentDecorator struct { + UserAgent string + Transport http.RoundTripper +} + +func (t userAgentDecorator) RoundTrip(r *http.Request) (*http.Response, error) { + if _, ok := r.Header["User-Agent"]; !ok { + r.Header.Set("User-Agent", t.UserAgent) + } + return t.Transport.RoundTrip(r) +} + func newRateLimiterFromConfig(cfg *ResourceConfig) *rate.Limiter { r := rate.Inf b := 1 diff --git a/x-pack/filebeat/input/cel/input_test.go b/x-pack/filebeat/input/cel/input_test.go index 1ee7704f826..7a2372fce3b 100644 --- a/x-pack/filebeat/input/cel/input_test.go +++ b/x-pack/filebeat/input/cel/input_test.go @@ -558,6 +558,142 @@ var inputTests = []struct { }, }, }, + { + name: "GET_request_check_user_agent_default", + server: newTestServer(httptest.NewServer), + config: map[string]interface{}{ + "interval": 1, + "program": ` + get(state.url).Body.as(body, { + "events": [body.decode_json()] + }) + `, + }, + handler: func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + msg := `{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}` + if r.UserAgent() != userAgent { + w.WriteHeader(http.StatusBadRequest) + msg = fmt.Sprintf(`{"error":"expected user agent was %#q"}`, userAgent) + } + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusBadRequest) + msg = fmt.Sprintf(`{"error":"expected method was %#q"}`, http.MethodGet) + } + + w.Write([]byte(msg)) + }, + want: []map[string]interface{}{ + { + "hello": []interface{}{ + map[string]interface{}{ + "world": "moon", + }, + map[string]interface{}{ + "space": []interface{}{ + map[string]interface{}{ + "cake": "pumpkin", + }, + }, + }, + }, + }, + }, + }, + { + name: "GET_request_check_user_agent_user_defined", + server: newTestServer(httptest.NewServer), + config: map[string]interface{}{ + "interval": 1, + "program": ` + get_request(state.url).with({ + "Header": { + "User-Agent": ["custom user agent"] + } + }).do_request().Body.as(body, { + "events": [body.decode_json()] + }) + `, + }, + handler: func(w http.ResponseWriter, r *http.Request) { + const customUserAgent = "custom user agent" + + w.Header().Set("content-type", "application/json") + msg := `{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}` + if r.UserAgent() != customUserAgent { + w.WriteHeader(http.StatusBadRequest) + msg = fmt.Sprintf(`{"error":"expected user agent was %#q"}`, customUserAgent) + } + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusBadRequest) + msg = fmt.Sprintf(`{"error":"expected method was %#q"}`, http.MethodGet) + } + + w.Write([]byte(msg)) + }, + want: []map[string]interface{}{ + { + "hello": []interface{}{ + map[string]interface{}{ + "world": "moon", + }, + map[string]interface{}{ + "space": []interface{}{ + map[string]interface{}{ + "cake": "pumpkin", + }, + }, + }, + }, + }, + }, + }, + { + name: "GET_request_check_user_agent_none", + server: newTestServer(httptest.NewServer), + config: map[string]interface{}{ + "interval": 1, + "program": ` + get_request(state.url).with({ + "Header": { + "User-Agent": [""] + } + }).do_request().Body.as(body, { + "events": [body.decode_json()] + }) + `, + }, + handler: func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + msg := `{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}` + if _, ok := r.Header["User-Agent"]; ok { + w.WriteHeader(http.StatusBadRequest) + msg = fmt.Sprintf(`{"error":"expected no user agent header, but got %#q"}`, r.UserAgent()) + } + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusBadRequest) + msg = fmt.Sprintf(`{"error":"expected method was %#q"}`, http.MethodGet) + } + + w.Write([]byte(msg)) + }, + want: []map[string]interface{}{ + { + "hello": []interface{}{ + map[string]interface{}{ + "world": "moon", + }, + map[string]interface{}{ + "space": []interface{}{ + map[string]interface{}{ + "cake": "pumpkin", + }, + }, + }, + }, + }, + }, + }, { name: "GET_request_TLS", server: newTestServer(httptest.NewTLSServer), @@ -1576,13 +1712,13 @@ func defaultHandler(expectedMethod, expectedBody string) http.HandlerFunc { switch { case r.Method != expectedMethod: w.WriteHeader(http.StatusBadRequest) - msg = fmt.Sprintf(`{"error":"expected method was %q"}`, expectedMethod) + msg = fmt.Sprintf(`{"error":"expected method was %#q"}`, expectedMethod) case expectedBody != "": body, _ := io.ReadAll(r.Body) r.Body.Close() if expectedBody != string(body) { w.WriteHeader(http.StatusBadRequest) - msg = fmt.Sprintf(`{"error":"expected body was %q"}`, expectedBody) + msg = fmt.Sprintf(`{"error":"expected body was %#q"}`, expectedBody) } }