Merge branch 'main' into refactor/drop-uber-atomic

.buildkite/auditbeat/auditbeat-pipeline.yml

@@ -27,6 +27,33 @@ env:
   TEST_COVERAGE: "true"
 
 steps:
+  - group: "Auditbeat Check/Update"
+    key: "auditbeat-check-update"
+
+    steps:
+      - label: "Run check/update"
+        command: |
+          make -C auditbeat check update
+          make check-no-changes
+        retry:
+          automatic:
+            - limit: 3
+        agents:
+          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+          cpu: "4000m"
+          memory: "8Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "auditbeat: check/update"
+
+  - wait: ~
+    # with PRs, we want to run mandatory tests only if check/update step succeed
+    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in check/update step
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "auditbeat-check-update"
+
   - group: "Auditbeat Mandatory Testing"
     key: "auditbeat-mandatory-tests"
 

.buildkite/scripts/changesets.sh

@@ -15,7 +15,7 @@ definePattern() {
 }
 
 defineExclusions() {
-  exclude="^$beatPath\/module\/(.*(?<!\.asciidoc|\.png))$"
+  exclude="^$beatPath\/module\/(.*(?<!\.asciidoc|\.png|Dockerfile))$"
 }
 
 defineFromCommit() {

.buildkite/x-pack/pipeline.xpack.auditbeat.yml

@@ -27,6 +27,33 @@ env:
   TEST_COVERAGE: "true"
 
 steps:
+  - group: "x-pack/auditbeat Check/Update"
+    key: "x-pack-auditbeat-check-update"
+
+    steps:
+      - label: "Run check/update"
+        command: |
+          make -C x-pack/auditbeat check update
+          make check-no-changes
+        retry:
+          automatic:
+            - limit: 3
+        agents:
+          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+          cpu: "4000m"
+          memory: "8Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "x-pack/auditbeat: check/update"
+
+  - wait: ~
+    # with PRs, we want to run mandatory tests only if check/update step succeed
+    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in check/update step
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "x-pack-auditbeat-check-update"
+
   - group: "x-pack/auditbeat Mandatory Tests"
     key: "x-pack-auditbeat-mandatory-tests"
 

.go-version

@@ -1 +1 @@
-1.21.10
+1.21.11

.golangci.yml

@@ -124,7 +124,7 @@ linters-settings:
 
   gosimple:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.10"
+    go: "1.21.11"
 
   nakedret:
     # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
@@ -142,19 +142,19 @@ linters-settings:
 
   staticcheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.10"
+    go: "1.21.11"
     checks: ["all"]
 
   stylecheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.10"
+    go: "1.21.11"
     # Disabled:
     # ST1005: error strings should not be capitalized
     checks: ["all", "-ST1005"]
 
   unused:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.10"
+    go: "1.21.11"
 
   gosec:
     excludes:

CHANGELOG.asciidoc

@@ -3,6 +3,17 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-8.14.1]]
+=== Beats version 8.14.1
+https://github.com/elastic/beats/compare/v8.14.0\...v8.14.1[View commits]
+
+==== Bugfixes
+
+*Heartbeat*
+
+- Fix import of browser plugin for Agentbeat. {pull}39818[39818]
+
+
 [[release-notes-8.14.0]]
 === Beats version 8.14.0
 https://github.com/elastic/beats/compare/v8.13.4\...v8.14.0[View commits]

CHANGELOG.next.asciidoc

@@ -145,18 +145,13 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Heartbeat*
 
-- Fix panics when parsing dereferencing invalid parsed url. {pull}34702[34702]
-- Fix setuid root when running under cgroups v2. {pull}37794[37794]
-- Adjust State loader to only retry when response code status is 5xx {pull}37981[37981]
-- Reset prctl dumpable flag after cap drop. {pull}38269[38269]
-- Redact synthexec cmd output. {pull}39535[39535]
-- Fix import of browser plugin for agentbeat. {pull}39818[39818]
 
 *Heartbeat*
 
 
 *Metricbeat*
 
+- Fix `namespace` filter option at Kubernetes provider level. {pull}39881[39881]
 - Fix Azure Monitor 429 error by causing metricbeat to retry the request again. {pull}38294[38294]
 - Fix fields not being parsed correctly in postgresql/database {issue}25301[25301] {pull}37720[37720]
 - rabbitmq/queue - Change the mapping type of `rabbitmq.queue.consumers.utilisation.pct` to `scaled_float` from `long` because the values fall within the range of `[0.0, 1.0]`. Previously, conversion to integer resulted in reporting either `0` or `1`.
@@ -191,7 +186,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Beats will now connect to older Elasticsearch instances by default {pull}36884[36884]
 - Raise up logging level to warning when attempting to configure beats with unknown fields from autodiscovered events/environments
 - elasticsearch output now supports `idle_connection_timeout`. {issue}35616[35615] {pull}36843[36843]
-- Update to Go 1.21.10. {pull}39467[39467]
+- Update to Go 1.21.11. {pull}39851[39851]
 - Enable early event encoding in the Elasticsearch output, improving cpu and memory use {pull}38572[38572]
 - The environment variable `BEATS_ADD_CLOUD_METADATA_PROVIDERS` overrides configured/default `add_cloud_metadata` providers {pull}38669[38669]
 - Introduce log message for not supported annotations for Hints based autodiscover {pull}38213[38213]
@@ -362,65 +357,3 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 ==== Known Issues
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21.10
+FROM golang:1.21.11
 
 RUN \
     apt-get update \

dev-tools/kubernetes/filebeat/Dockerfile.debug

@@ -1,4 +1,4 @@
-FROM golang:1.21.10 as builder
+FROM golang:1.21.11 as builder
 
 ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
 

dev-tools/kubernetes/heartbeat/Dockerfile.debug

@@ -1,4 +1,4 @@
-FROM golang:1.21.10 as builder
+FROM golang:1.21.11 as builder
 
 ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
 

dev-tools/kubernetes/metricbeat/Dockerfile.debug

@@ -1,4 +1,4 @@
-FROM golang:1.21.10 as builder
+FROM golang:1.21.11 as builder
 
 ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
 

filebeat/tests/system/test_crawler.py

@@ -485,7 +485,7 @@ def test_tail_files(self):
         self.wait_until(
             lambda: self.log_contains(
                 "Start next scan"),
-            max_timeout=5)
+            max_timeout=10)
 
         with open(testfile, 'a') as f:
             # write additional lines
@@ -596,7 +596,7 @@ def test_encodings(self):
         # run filebeat
         filebeat = self.start_beat()
         self.wait_until(lambda: self.output_has(lines=len(encodings)),
-                        max_timeout=15)
+                        max_timeout=25)
 
         # write another line in all files
         for _, enc_py, text in encodings:

filebeat/tests/system/test_harvester.py

@@ -858,6 +858,6 @@ def test_debug_reader(self):
 
         # 13 on unix, 14 on windows.
         self.wait_until(lambda: self.log_contains(re.compile(
-            'Matching null byte found at offset (13|14)')), max_timeout=5)
+            'Matching null byte found at offset (13|14)')), max_timeout=10)
 
         filebeat.check_kill_and_wait()

filebeat/tests/system/test_registrar.py

@@ -252,9 +252,11 @@ def test_registry_file_update_permissions(self):
 
         self.assertEqual(self.file_permissions(os.path.join(registry_path, "log.json")), "0o600")
 
+        registry_home = "a/b/c/d/registry_x"
+        registry_path = os.path.join(registry_home, "filebeat")
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/*",
-            registry_home="a/b/c/registry_x",
+            registry_home=registry_home,
             registry_file_permissions=0o640
         )
 
@@ -266,7 +268,7 @@ def test_registry_file_update_permissions(self):
         # the logging and actual writing the file. Seems to happen on Windows.
         self.wait_until(
             lambda: self.has_registry(registry_path),
-            max_timeout=1)
+            max_timeout=10)
 
         # Wait a moment to make sure registry is completely written
         time.sleep(1)
@@ -950,7 +952,7 @@ def test_restart_state(self):
             path=os.path.abspath(self.working_dir) + "/log/*",
             close_inactive="200ms",
             ignore_older="2000ms",
-            clean_inactive="3s",
+            clean_inactive="10s",
         )
 
         filebeat = self.start_beat()
@@ -976,7 +978,7 @@ def test_restart_state_reset(self):
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/*",
             clean_inactive="10s",
-            ignore_older="5s"
+            ignore_older="9s"
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -1003,7 +1005,7 @@ def test_restart_state_reset(self):
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/test2.log",
             clean_inactive="10s",
-            ignore_older="5s",
+            ignore_older="9s",
         )
 
         filebeat = self.start_beat(output="filebeat2.log")
@@ -1137,7 +1139,7 @@ def test_restart_state_reset_ttl_no_clean_inactive(self):
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/test.log",
             clean_inactive="10s",
-            ignore_older="5s"
+            ignore_older="9s"
         )
         os.mkdir(self.working_dir + "/log/")
 

filebeat/tests/system/test_shutdown.py

@@ -27,7 +27,13 @@ def test_shutdown(self):
         )
         for i in range(1, 5):
             proc = self.start_beat(logging_args=["-e", "-v"])
-            time.sleep(.5)
+
+            # Flaky on MacOS, see https://github.com/elastic/beats/issues/39613#issuecomment-2158812325
+            # we need to wait a bit longer for filebeat to start
+            if platform.system() == "Darwin":
+                time.sleep(10)
+            else:
+                time.sleep(.5)
             proc.check_kill_and_wait()
 
     @unittest.skip("Skipped as flaky: https://github.com/elastic/beats/issues/14647")

go.mod

@@ -2,7 +2,7 @@ module github.com/elastic/beats/v7
 
 go 1.21.0
 
-toolchain go1.21.10
+toolchain go1.21.11
 
 require (
 	cloud.google.com/go/bigquery v1.55.0

heartbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21.10
+FROM golang:1.21.11
 
 RUN \
     apt-get update \