Merge remote-tracking branch 'origin' into retestsched

.ci/bump-golang.yml

@@ -120,6 +120,15 @@ targets:
       content: ':go-version: {{ source "latestGoVersion" }}'
       file: libbeat/docs/version.asciidoc
       matchpattern: ':go-version: \d+.\d+.\d+'
+  update-devcontainer:
+    name: "Update .devcontainer/devcontainer.json"
+    sourceid: gomod
+    scmid: githubConfig
+    kind: file
+    spec:
+      content: 'go:1-{{ source "gomod" }}'
+      file: ./.devcontainer/devcontainer.json
+      matchpattern: 'go:1-\d+.\d+'
   update-auditbeat-dockerfile:
     name: "Update Auditbeat Dockerfile"
     sourceid: latestGoVersion

.devcontainer/devcontainer.json

@@ -0,0 +1,31 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/go
+// If you're wondering how to get SSH working for git pushes, make sure to ssh-add ~/.ssh/<YOURPRIVATEKEY> on your host system
+// to make sure the ssh agent has your key
+{
+	"name": "Beats Development Container",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/go:1-1.20-bullseye",
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	"features": {
+		"ghcr.io/devcontainers/features/node:1": {},
+		"ghcr.io/devcontainers/features/python:1": {},
+		"ghcr.io/devcontainers/features/github-cli:1": {},
+		"ghcr.io/devcontainers/features/common-utils:2": {},
+		"ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}
+	},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// Mage is installed this way, and not via the feature plugin because that plugin was
+	// broken for me, and mage install is simple enough
+	"postCreateCommand": "cd /opt/; sudo mkdir mage; sudo chown $USER:$(id -g) mage; git clone --depth=1 https://github.com/magefile/mage && cd mage && go run bootstrap.go"
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

CHANGELOG-developer.next.asciidoc

@@ -65,6 +65,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - The `queue.Batch` API now provides access to individual events instead of an array. {pull}31699[31699]
 - Rename `queue.Batch.ACK()` to `queue.Batch.Done()`. {pull}31903[31903]
 - `queue.ACKListener` has been removed. Queue configurations now accept an explicit callback function for ACK handling. {pull}35078[35078]
+- Split split httpmon out of x-pack/filebeat/input/internal/httplog. {pull}36385[36385]
 
 ==== Bugfixes
 

CHANGELOG.next.asciidoc

@@ -127,6 +127,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add option in SQL module to execute queries for all dbs. {pull}35688[35688]
 - Add support for api_key authentication in elasticsearch module  {pull}36274[36274]
 - Add remaining dimensions for azure storage account to make them available for tsdb enablement. {pull}36331[36331]
+- Add missing 'TransactionType' dimension for Azure Storage Account. {pull}36413[36413]
 
 *Osquerybeat*
 
@@ -149,6 +150,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add warning message to SysV init scripts for RPM-based systems that lack `/etc/rc.d/init.d/functions`. {issue}35708[35708] {pull}36188[36188]
 - Mark `translate_sid` processor is GA. {issue}36279[36279] {pull}36280[36280]
 - dns processor: Add support for forward lookups (`A`, `AAAA`, and `TXT`). {issue}11416[11416] {pull}36394[36394]
+- Mark `syslog` processor as GA, improve docs about how processor handles syslog messages. {issue}36416[36416] {pull}36417[36417]
 
 *Auditbeat*
 

libbeat/processors/syslog/docs/syslog.asciidoc

@@ -5,7 +5,12 @@
 <titleabbrev>syslog</titleabbrev>
 ++++
 
-experimental[]
+The syslog processor parses RFC 3146 and/or RFC 5424 formatted syslog messages
+that are stored in a field. The processor itself does not handle receiving syslog
+messages from external sources. This is done through an input, such as the TCP
+input. Certain integrations, when enabled through configuration, will embed the
+syslog processor to process syslog messages, such as Custom TCP Logs and
+Custom UDP Logs.
 
 [float]
 ==== Configuration

testing/environments/snapshot.yml

@@ -3,7 +3,7 @@
 version: '2.3'
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.10.0-0aabd193-SNAPSHOT
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0-76b03348-SNAPSHOT
     # When extend is used it merges healthcheck.tests, see:
     # https://github.com/docker/compose/issues/8962
     # healthcheck:
@@ -31,7 +31,7 @@ services:
     - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles"
 
   logstash:
-    image: docker.elastic.co/logstash/logstash:8.10.0-0aabd193-SNAPSHOT
+    image: docker.elastic.co/logstash/logstash:8.11.0-76b03348-SNAPSHOT
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"]
       retries: 600
@@ -44,7 +44,7 @@ services:
       - 5055:5055
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:8.10.0-0aabd193-SNAPSHOT
+    image: docker.elastic.co/kibana/kibana:8.11.0-76b03348-SNAPSHOT
     environment:
     - "ELASTICSEARCH_USERNAME=kibana_system_user"
     - "ELASTICSEARCH_PASSWORD=testing"

x-pack/filebeat/input/httpjson/input.go

@@ -29,6 +29,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/monitoring/inputmon"
 	"github.com/elastic/beats/v7/libbeat/version"
 	"github.com/elastic/beats/v7/x-pack/filebeat/input/internal/httplog"
+	"github.com/elastic/beats/v7/x-pack/filebeat/input/internal/httpmon"
 	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/elastic-agent-libs/mapstr"
 	"github.com/elastic/elastic-agent-libs/monitoring"
@@ -256,7 +257,7 @@ func newNetHTTPClient(ctx context.Context, cfg *requestConfig, log *logp.Logger,
 	}
 
 	if reg != nil {
-		netHTTPClient.Transport = httplog.NewMetricsRoundTripper(netHTTPClient.Transport, reg)
+		netHTTPClient.Transport = httpmon.NewMetricsRoundTripper(netHTTPClient.Transport, reg)
 	}
 
 	netHTTPClient.CheckRedirect = checkRedirect(cfg, log)

x-pack/filebeat/input/internal/httplog/roundtripper.go

@@ -17,17 +17,12 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/rcrowley/go-metrics"
 	"go.uber.org/atomic"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
-
-	"github.com/elastic/elastic-agent-libs/monitoring"
-	"github.com/elastic/elastic-agent-libs/monitoring/adapter"
 )
 
 var _ http.RoundTripper = (*LoggingRoundTripper)(nil)
-var _ http.RoundTripper = (*MetricsRoundTripper)(nil)
 
 // TraceIDKey is key used to add a trace.id value to the context of HTTP
 // requests. The value will be logged by LoggingRoundTripper.
@@ -212,179 +207,3 @@ func copyBody(b io.ReadCloser) (r io.ReadCloser, body []byte, err error) {
 	}
 	return io.NopCloser(&buf), buf.Bytes(), nil
 }
-
-// MetricsRoundTripper is an http.RoundTripper that monitors requests and responses.
-type MetricsRoundTripper struct {
-	transport http.RoundTripper
-
-	metrics *httpMetrics
-}
-
-type httpMetrics struct {
-	reqs          *monitoring.Uint // total number of requests
-	reqErrs       *monitoring.Uint // total number of request errors
-	reqDelete     *monitoring.Uint // number of DELETE requests
-	reqGet        *monitoring.Uint // number of GET requests
-	reqHead       *monitoring.Uint // number of HEAD requests
-	reqOptions    *monitoring.Uint // number of OPTIONS requests
-	reqPatch      *monitoring.Uint // number of PATCH requests
-	reqPost       *monitoring.Uint // number of POST requests
-	reqPut        *monitoring.Uint // number of PUT requests
-	reqsAccSize   *monitoring.Uint // accumulated request body size
-	reqsSize      metrics.Sample   // histogram of the request body size
-	resps         *monitoring.Uint // total number of responses
-	respErrs      *monitoring.Uint // total number of response errors
-	resp1xx       *monitoring.Uint // number of 1xx responses
-	resp2xx       *monitoring.Uint // number of 2xx responses
-	resp3xx       *monitoring.Uint // number of 3xx responses
-	resp4xx       *monitoring.Uint // number of 4xx responses
-	resp5xx       *monitoring.Uint // number of 5xx responses
-	respsAccSize  *monitoring.Uint // accumulated response body size
-	respsSize     metrics.Sample   // histogram of the response body size
-	roundTripTime metrics.Sample   // histogram of the round trip (request -> response) time
-}
-
-// NewMetricsRoundTripper returns a MetricsRoundTripper that sends requests and
-// responses metrics to the provided input monitoring registry.
-// It will register all http related metrics into the provided registry, but it is not responsible
-// for its lifecyle.
-func NewMetricsRoundTripper(next http.RoundTripper, reg *monitoring.Registry) *MetricsRoundTripper {
-	return &MetricsRoundTripper{
-		transport: next,
-		metrics:   newHTTPMetrics(reg),
-	}
-}
-
-func newHTTPMetrics(reg *monitoring.Registry) *httpMetrics {
-	if reg == nil {
-		return nil
-	}
-
-	out := &httpMetrics{
-		reqs:          monitoring.NewUint(reg, "http_request_total"),
-		reqErrs:       monitoring.NewUint(reg, "http_request_errors_total"),
-		reqDelete:     monitoring.NewUint(reg, "http_request_delete_total"),
-		reqGet:        monitoring.NewUint(reg, "http_request_get_total"),
-		reqHead:       monitoring.NewUint(reg, "http_request_head_total"),
-		reqOptions:    monitoring.NewUint(reg, "http_request_options_total"),
-		reqPatch:      monitoring.NewUint(reg, "http_request_patch_total"),
-		reqPost:       monitoring.NewUint(reg, "http_request_post_total"),
-		reqPut:        monitoring.NewUint(reg, "http_request_put_total"),
-		reqsAccSize:   monitoring.NewUint(reg, "http_request_body_bytes_total"),
-		reqsSize:      metrics.NewUniformSample(1024),
-		resps:         monitoring.NewUint(reg, "http_response_total"),
-		respErrs:      monitoring.NewUint(reg, "http_response_errors_total"),
-		resp1xx:       monitoring.NewUint(reg, "http_response_1xx_total"),
-		resp2xx:       monitoring.NewUint(reg, "http_response_2xx_total"),
-		resp3xx:       monitoring.NewUint(reg, "http_response_3xx_total"),
-		resp4xx:       monitoring.NewUint(reg, "http_response_4xx_total"),
-		resp5xx:       monitoring.NewUint(reg, "http_response_5xx_total"),
-		respsAccSize:  monitoring.NewUint(reg, "http_response_body_bytes_total"),
-		respsSize:     metrics.NewUniformSample(1024),
-		roundTripTime: metrics.NewUniformSample(1024),
-	}
-
-	_ = adapter.GetGoMetrics(reg, "http_request_body_bytes", adapter.Accept).
-		GetOrRegister("histogram", metrics.NewHistogram(out.reqsSize))
-	_ = adapter.GetGoMetrics(reg, "http_response_body_bytes", adapter.Accept).
-		GetOrRegister("histogram", metrics.NewHistogram(out.respsSize))
-	_ = adapter.GetGoMetrics(reg, "http_round_trip_time", adapter.Accept).
-		GetOrRegister("histogram", metrics.NewHistogram(out.roundTripTime))
-
-	return out
-}
-
-// RoundTrip implements the http.RoundTripper interface, sending
-// request and response metrics to the underlying registry.
-//
-//	http_request_total
-//	http_request_errors_total
-//	http_request_delete_total
-//	http_request_get_total
-//	http_request_head_total
-//	http_request_options_total
-//	http_request_patch_total
-//	http_request_post_total
-//	http_request_put_total
-//	http_request_body_bytes_total
-//	http_request_body_bytes
-//	http_response_total
-//	http_response_errors_total
-//	http_response_1xx_total
-//	http_response_2xx_total
-//	http_response_3xx_total
-//	http_response_4xx_total
-//	http_response_5xx_total
-//	http_response_body_bytes_total
-//	http_response_body_bytes
-//	http_round_trip_time
-func (rt *MetricsRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
-	if rt.metrics == nil {
-		return rt.transport.RoundTrip(req)
-	}
-
-	rt.metrics.reqs.Add(1)
-
-	rt.monitorByMethod(req.Method)
-
-	if req.ContentLength >= 0 {
-		rt.metrics.reqsAccSize.Add(uint64(req.ContentLength))
-		rt.metrics.reqsSize.Update(req.ContentLength)
-	}
-
-	reqStart := time.Now()
-	resp, err := rt.transport.RoundTrip(req)
-	rt.metrics.roundTripTime.Update(time.Since(reqStart).Nanoseconds())
-
-	if resp != nil {
-		rt.metrics.resps.Add(1)
-	}
-
-	if resp == nil || err != nil {
-		rt.metrics.respErrs.Add(1)
-		return resp, err
-	}
-
-	rt.monitorByStatusCode(resp.StatusCode)
-
-	if resp.ContentLength >= 0 {
-		rt.metrics.respsAccSize.Add(uint64(resp.ContentLength))
-		rt.metrics.respsSize.Update(resp.ContentLength)
-	}
-
-	return resp, err
-}
-
-func (rt *MetricsRoundTripper) monitorByMethod(method string) {
-	switch method {
-	case http.MethodDelete:
-		rt.metrics.reqDelete.Add(1)
-	case http.MethodGet:
-		rt.metrics.reqGet.Add(1)
-	case http.MethodHead:
-		rt.metrics.reqHead.Add(1)
-	case http.MethodOptions:
-		rt.metrics.reqOptions.Add(1)
-	case http.MethodPatch:
-		rt.metrics.reqPatch.Add(1)
-	case http.MethodPost:
-		rt.metrics.reqPost.Add(1)
-	case http.MethodPut:
-		rt.metrics.reqPut.Add(1)
-	}
-}
-
-func (rt *MetricsRoundTripper) monitorByStatusCode(code int) {
-	switch code / 100 {
-	case 1:
-		rt.metrics.resp1xx.Add(1)
-	case 2:
-		rt.metrics.resp2xx.Add(1)
-	case 3:
-		rt.metrics.resp3xx.Add(1)
-	case 4:
-		rt.metrics.resp4xx.Add(1)
-	case 5:
-		rt.metrics.resp5xx.Add(1)
-	}
-}