From 639fc23430d70816b0a1262307a6ff2d0e98fd2d Mon Sep 17 00:00:00 2001 From: ev1yehor <146825775+ev1yehor@users.noreply.github.com> Date: Thu, 13 Jun 2024 15:54:04 +0300 Subject: [PATCH 1/8] Migrate github checks to buildkite for auditbeat and x-pack/auditbeat (#39868) * Update * image update --- .buildkite/auditbeat/auditbeat-pipeline.yml | 27 ++++++++++++++++++ .../x-pack/pipeline.xpack.auditbeat.yml | 27 ++++++++++++++++++ .github/workflows/check-auditbeat.yml | 28 ------------------- 3 files changed, 54 insertions(+), 28 deletions(-) delete mode 100644 .github/workflows/check-auditbeat.yml diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 2ac4a8911a1..f5ca39b3645 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -27,6 +27,33 @@ env: TEST_COVERAGE: "true" steps: + - group: "Auditbeat Check/Update" + key: "auditbeat-check-update" + + steps: + - label: "Run check/update" + command: | + make -C auditbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "auditbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "auditbeat-check-update" + - group: "Auditbeat Mandatory Testing" key: "auditbeat-mandatory-tests" diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml index a851f681d33..9552cc14ef9 100644 --- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml @@ -27,6 +27,33 @@ env: TEST_COVERAGE: "true" steps: + - group: "x-pack/auditbeat Check/Update" + key: "x-pack-auditbeat-check-update" + + steps: + - label: "Run check/update" + command: | + make -C x-pack/auditbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "x-pack/auditbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "x-pack-auditbeat-check-update" + - group: "x-pack/auditbeat Mandatory Tests" key: "x-pack-auditbeat-mandatory-tests" diff --git a/.github/workflows/check-auditbeat.yml b/.github/workflows/check-auditbeat.yml deleted file mode 100644 index bbc96242687..00000000000 --- a/.github/workflows/check-auditbeat.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: check-auditbeat - -on: - pull_request: - paths: - - '.github/workflows/check-auditbeat.yml' - - 'auditbeat/**' - - 'x-pack/auditbeat/**' - -env: - BEAT_MODULE: 'auditbeat' - -permissions: - contents: read - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - name: Run check/update - run: | - go install github.com/magefile/mage - make -C ${{ env.BEAT_MODULE }} check update - make check-no-changes From b5155a177edea5ca97b6f5aaecc020d95d8350f7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 13 Jun 2024 08:57:32 -0400 Subject: [PATCH 2/8] [Automation] Bump Golang version to 1.21.11 (#39851) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: Update Auditbeat Dockerfile Made with ❤️️ by updatecli * chore: Update Filebeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update from vsphere Dockerfile Made with ❤️️ by updatecli * chore: Update Heartbeat Dockerfile Made with ❤️️ by updatecli * chore: Update NATS module Dockerfile Made with ❤️️ by updatecli * chore: Update version.asciidoc Made with ❤️️ by updatecli * chore: Update Metricbeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update Metricbeat Dockerfile Made with ❤️️ by updatecli * chore: Update HTTP module Dockerfile Made with ❤️️ by updatecli * chore: Update .go-version Made with ❤️️ by updatecli * chore: Update go.mod Made with ❤️️ by updatecli * chore: Update Functionbeat Dockerfile Made with ❤️️ by updatecli * chore: Update Packetbeat Dockerfile Made with ❤️️ by updatecli * chore: Update stan Dockerfile Made with ❤️️ by updatecli * chore: Update .golangci.yml Made with ❤️️ by updatecli * chore: Update Heartbeat debug Dockerfile Made with ❤️️ by updatecli * Update changelog. * Run go mod tidy. * chore: Update .go-version Made with ❤️️ by updatecli * chore: Update Packetbeat Dockerfile Made with ❤️️ by updatecli * chore: Update Filebeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update Heartbeat Dockerfile Made with ❤️️ by updatecli * chore: Update version.asciidoc Made with ❤️️ by updatecli * chore: Update Metricbeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update stan Dockerfile Made with ❤️️ by updatecli * chore: Update go.mod Made with ❤️️ by updatecli * chore: Update Metricbeat Dockerfile Made with ❤️️ by updatecli * chore: Update Heartbeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update HTTP module Dockerfile Made with ❤️️ by updatecli * chore: Update NATS module Dockerfile Made with ❤️️ by updatecli * chore: Update from vsphere Dockerfile Made with ❤️️ by updatecli * chore: Update Functionbeat Dockerfile Made with ❤️️ by updatecli * chore: Update .golangci.yml Made with ❤️️ by updatecli * chore: Update Auditbeat Dockerfile Made with ❤️️ by updatecli * Exclude Dockerfile from module detection Signed-off-by: Alexandros Sapranidis * Update go.mod Signed-off-by: Alexandros Sapranidis * chore: Update from vsphere Dockerfile Made with ❤️️ by updatecli * chore: Update Heartbeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update Metricbeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update Functionbeat Dockerfile Made with ❤️️ by updatecli * chore: Update go.mod Made with ❤️️ by updatecli * chore: Update Heartbeat Dockerfile Made with ❤️️ by updatecli * chore: Update Filebeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update NATS module Dockerfile Made with ❤️️ by updatecli * chore: Update stan Dockerfile Made with ❤️️ by updatecli * chore: Update HTTP module Dockerfile Made with ❤️️ by updatecli * chore: Update Metricbeat Dockerfile Made with ❤️️ by updatecli * chore: Update version.asciidoc Made with ❤️️ by updatecli * chore: Update .golangci.yml Made with ❤️️ by updatecli * chore: Update Packetbeat Dockerfile Made with ❤️️ by updatecli * chore: Update Auditbeat Dockerfile Made with ❤️️ by updatecli * chore: Update .go-version Made with ❤️️ by updatecli * go mod tidy --------- Signed-off-by: Alexandros Sapranidis Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Craig MacKenzie Co-authored-by: Alexandros Sapranidis --- .buildkite/scripts/changesets.sh | 2 +- .go-version | 2 +- .golangci.yml | 8 ++++---- CHANGELOG.next.asciidoc | 2 +- auditbeat/Dockerfile | 2 +- dev-tools/kubernetes/filebeat/Dockerfile.debug | 2 +- dev-tools/kubernetes/heartbeat/Dockerfile.debug | 2 +- dev-tools/kubernetes/metricbeat/Dockerfile.debug | 2 +- go.mod | 2 +- heartbeat/Dockerfile | 2 +- libbeat/docs/version.asciidoc | 2 +- metricbeat/Dockerfile | 2 +- metricbeat/module/http/_meta/Dockerfile | 2 +- metricbeat/module/nats/_meta/Dockerfile | 2 +- metricbeat/module/vsphere/_meta/Dockerfile | 2 +- packetbeat/Dockerfile | 2 +- x-pack/functionbeat/Dockerfile | 2 +- x-pack/metricbeat/module/stan/_meta/Dockerfile | 2 +- 18 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.buildkite/scripts/changesets.sh b/.buildkite/scripts/changesets.sh index 7e79d2d3164..1ab80edc585 100644 --- a/.buildkite/scripts/changesets.sh +++ b/.buildkite/scripts/changesets.sh @@ -15,7 +15,7 @@ definePattern() { } defineExclusions() { - exclude="^$beatPath\/module\/(.*(? Date: Thu, 13 Jun 2024 15:15:40 +0200 Subject: [PATCH 3/8] docs: Prepare Changelog for 8.14.1 (#39864) (#39882) * docs: Close changelog for 8.14.1 * Update CHANGELOG.asciidoc * Update CHANGELOG.next.asciidoc --------- Co-authored-by: Pierre HILBERT (cherry picked from commit f68af7f3d287df593abcca6b62ed5cff8a0fe01b) Co-authored-by: Elastic Machine Co-authored-by: Lisa Cawley --- CHANGELOG.asciidoc | 11 ++++++ CHANGELOG.next.asciidoc | 68 ----------------------------------- libbeat/docs/release.asciidoc | 1 + 3 files changed, 12 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 83844526f9d..6d12414d6e2 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -3,6 +3,17 @@ :issue: https://github.com/elastic/beats/issues/ :pull: https://github.com/elastic/beats/pull/ +[[release-notes-8.14.1]] +=== Beats version 8.14.1 +https://github.com/elastic/beats/compare/v8.14.0\...v8.14.1[View commits] + +==== Bugfixes + +*Heartbeat* + +- Fix import of browser plugin for Agentbeat. {pull}39818[39818] + + [[release-notes-8.14.0]] === Beats version 8.14.0 https://github.com/elastic/beats/compare/v8.13.4\...v8.14.0[View commits] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 33bbcc055bc..88656f51363 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -145,12 +145,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Heartbeat* -- Fix panics when parsing dereferencing invalid parsed url. {pull}34702[34702] -- Fix setuid root when running under cgroups v2. {pull}37794[37794] -- Adjust State loader to only retry when response code status is 5xx {pull}37981[37981] -- Reset prctl dumpable flag after cap drop. {pull}38269[38269] -- Redact synthexec cmd output. {pull}39535[39535] -- Fix import of browser plugin for agentbeat. {pull}39818[39818] *Heartbeat* @@ -362,65 +356,3 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] ==== Known Issues - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/libbeat/docs/release.asciidoc b/libbeat/docs/release.asciidoc index bad3f87b38d..3dc94738d12 100644 --- a/libbeat/docs/release.asciidoc +++ b/libbeat/docs/release.asciidoc @@ -8,6 +8,7 @@ This section summarizes the changes in each release. Also read <> for more detail about changes that affect upgrade. +* <> * <> * <> * <> From 2f91cf1b07d1e6bb72df4858cb324d5c44bfda21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Constan=C3=A7a=20Manteigas?= <113898685+constanca-m@users.noreply.github.com> Date: Thu, 13 Jun 2024 15:26:52 +0200 Subject: [PATCH 4/8] [Kubernetes Provider] Apply namespace filter to watchers (#39881) * Apply namespace filter Signed-off-by: constanca * Apply namespace filter Signed-off-by: constanca * Fix namespace docs description Signed-off-by: constanca --------- Signed-off-by: constanca --- CHANGELOG.next.asciidoc | 1 + .../autodiscover/providers/kubernetes/pod.go | 20 ++++++++++++------- .../providers/kubernetes/service.go | 5 +++-- libbeat/docs/shared-autodiscover.asciidoc | 7 ++----- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 88656f51363..15e4fd614db 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -151,6 +151,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Metricbeat* +- Fix `namespace` filter option at Kubernetes provider level. {pull}39881[39881] - Fix Azure Monitor 429 error by causing metricbeat to retry the request again. {pull}38294[38294] - Fix fields not being parsed correctly in postgresql/database {issue}25301[25301] {pull}37720[37720] - rabbitmq/queue - Change the mapping type of `rabbitmq.queue.consumers.utilisation.pct` to `scaled_float` from `long` because the values fall within the range of `[0.0, 1.0]`. Previously, conversion to integer resulted in reporting either `0` or `1`. diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go index b11faac4931..c5f9c721eb9 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod.go +++ b/libbeat/autodiscover/providers/kubernetes/pod.go @@ -100,9 +100,9 @@ func NewPodEventer(uuid uuid.UUID, cfg *conf.C, client k8s.Interface, publish fu if metaConf.Node.Enabled() || config.Hints.Enabled() { options := kubernetes.WatchOptions{ - SyncTimeout: config.SyncPeriod, - Node: config.Node, - Namespace: config.Namespace, + SyncTimeout: config.SyncPeriod, + Node: config.Node, + HonorReSyncs: true, } nodeWatcher, err = kubernetes.NewNamedWatcher("node", client, &kubernetes.Node{}, options, nil) if err != nil { @@ -112,20 +112,24 @@ func NewPodEventer(uuid uuid.UUID, cfg *conf.C, client k8s.Interface, publish fu if metaConf.Namespace.Enabled() || config.Hints.Enabled() { namespaceWatcher, err = kubernetes.NewNamedWatcher("namespace", client, &kubernetes.Namespace{}, kubernetes.WatchOptions{ - SyncTimeout: config.SyncPeriod, + SyncTimeout: config.SyncPeriod, + Namespace: config.Namespace, + HonorReSyncs: true, }, nil) if err != nil { logger.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Namespace{}, err) } } - // Resource is Pod so we need to create watchers for Replicasets and Jobs that it might belongs to + // Resource is Pod, so we need to create watchers for Replicasets and Jobs that it might belong to // in order to be able to retrieve 2nd layer Owner metadata like in case of: // Deployment -> Replicaset -> Pod // CronJob -> job -> Pod if metaConf.Deployment { replicaSetWatcher, err = kubernetes.NewNamedWatcher("resource_metadata_enricher_rs", client, &kubernetes.ReplicaSet{}, kubernetes.WatchOptions{ - SyncTimeout: config.SyncPeriod, + SyncTimeout: config.SyncPeriod, + Namespace: config.Namespace, + HonorReSyncs: true, }, nil) if err != nil { logger.Errorf("Error creating watcher for %T due to error %+v", &kubernetes.ReplicaSet{}, err) @@ -133,7 +137,9 @@ func NewPodEventer(uuid uuid.UUID, cfg *conf.C, client k8s.Interface, publish fu } if metaConf.CronJob { jobWatcher, err = kubernetes.NewNamedWatcher("resource_metadata_enricher_job", client, &kubernetes.Job{}, kubernetes.WatchOptions{ - SyncTimeout: config.SyncPeriod, + SyncTimeout: config.SyncPeriod, + Namespace: config.Namespace, + HonorReSyncs: true, }, nil) if err != nil { logger.Errorf("Error creating watcher for %T due to error %+v", &kubernetes.Job{}, err) diff --git a/libbeat/autodiscover/providers/kubernetes/service.go b/libbeat/autodiscover/providers/kubernetes/service.go index ba62dda9c47..e9e71c921bd 100644 --- a/libbeat/autodiscover/providers/kubernetes/service.go +++ b/libbeat/autodiscover/providers/kubernetes/service.go @@ -74,8 +74,9 @@ func NewServiceEventer(uuid uuid.UUID, cfg *conf.C, client k8s.Interface, publis if metaConf.Namespace.Enabled() || config.Hints.Enabled() { namespaceWatcher, err = kubernetes.NewNamedWatcher("namespace", client, &kubernetes.Namespace{}, kubernetes.WatchOptions{ - SyncTimeout: config.SyncPeriod, - Namespace: config.Namespace, + SyncTimeout: config.SyncPeriod, + Namespace: config.Namespace, + HonorReSyncs: true, }, nil) if err != nil { return nil, fmt.Errorf("couldn't create watcher for %T due to error %w", &kubernetes.Namespace{}, err) diff --git a/libbeat/docs/shared-autodiscover.asciidoc b/libbeat/docs/shared-autodiscover.asciidoc index 1e1ea567b7c..83d44b498ac 100644 --- a/libbeat/docs/shared-autodiscover.asciidoc +++ b/libbeat/docs/shared-autodiscover.asciidoc @@ -140,10 +140,7 @@ The `kubernetes` autodiscover provider has the following configuration settings: `node`:: (Optional) Specify the node to scope {beatname_lc} to in case it cannot be accurately detected, as when running {beatname_lc} in host network mode. -`namespace`:: (Optional) Select the namespace from which to collect the - metadata. If it is not set, the processor collects metadata from all - namespaces. It is unset by default. The namespace configuration only applies to - kubernetes resources that are namespace scoped. +`namespace`:: (Optional) Select the namespace from which to collect the events from the resources. If it is not set, the provider collects them from all namespaces. It is unset by default. The namespace configuration only applies to kubernetes resources that are namespace scoped and if `unique` field is set to `false`. `cleanup_timeout`:: (Optional) Specify the time of inactivity before stopping the running configuration for a container, ifeval::["{beatname_lc}"=="filebeat"] @@ -196,7 +193,7 @@ Example: `unique`:: (Optional) Defaults to `false`. Marking an autodiscover provider as unique results into making the provider to enable the provided templates only when it will gain the leader lease. - This setting can only be combined with `cluster` scope. When `unique` is enabled enabled, `resource` + This setting can only be combined with `cluster` scope. When `unique` is enabled, `resource` and `add_resource_metadata` settings are not taken into account. `leader_lease`:: (Optional) Defaults to +{beatname_lc}-cluster-leader+. This will be name of the lock lease. One can monitor the status of the lease with `kubectl describe lease beats-cluster-leader`. From f9fec1e4ff85085093682a1017640cc2e4b097b8 Mon Sep 17 00:00:00 2001 From: VihasMakwana <121151420+VihasMakwana@users.noreply.github.com> Date: Thu, 13 Jun 2024 18:58:16 +0530 Subject: [PATCH 5/8] filebeat: Fix flaky test case on macOS (#39860) * fix: fix a flaky test on macos * fix: fix more such test cases * fix: only update ignore_older * fix: also fix flaky test_restart_state * fix: fix CI --- filebeat/tests/system/test_crawler.py | 4 ++-- filebeat/tests/system/test_harvester.py | 2 +- filebeat/tests/system/test_registrar.py | 14 ++++++++------ filebeat/tests/system/test_shutdown.py | 8 +++++++- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/filebeat/tests/system/test_crawler.py b/filebeat/tests/system/test_crawler.py index 2bea57223fe..39f0d454124 100644 --- a/filebeat/tests/system/test_crawler.py +++ b/filebeat/tests/system/test_crawler.py @@ -485,7 +485,7 @@ def test_tail_files(self): self.wait_until( lambda: self.log_contains( "Start next scan"), - max_timeout=5) + max_timeout=10) with open(testfile, 'a') as f: # write additional lines @@ -596,7 +596,7 @@ def test_encodings(self): # run filebeat filebeat = self.start_beat() self.wait_until(lambda: self.output_has(lines=len(encodings)), - max_timeout=15) + max_timeout=25) # write another line in all files for _, enc_py, text in encodings: diff --git a/filebeat/tests/system/test_harvester.py b/filebeat/tests/system/test_harvester.py index 3b40f3a6730..9099abc699e 100644 --- a/filebeat/tests/system/test_harvester.py +++ b/filebeat/tests/system/test_harvester.py @@ -858,6 +858,6 @@ def test_debug_reader(self): # 13 on unix, 14 on windows. self.wait_until(lambda: self.log_contains(re.compile( - 'Matching null byte found at offset (13|14)')), max_timeout=5) + 'Matching null byte found at offset (13|14)')), max_timeout=10) filebeat.check_kill_and_wait() diff --git a/filebeat/tests/system/test_registrar.py b/filebeat/tests/system/test_registrar.py index 53af186dbf9..b08eda2777c 100644 --- a/filebeat/tests/system/test_registrar.py +++ b/filebeat/tests/system/test_registrar.py @@ -252,9 +252,11 @@ def test_registry_file_update_permissions(self): self.assertEqual(self.file_permissions(os.path.join(registry_path, "log.json")), "0o600") + registry_home = "a/b/c/d/registry_x" + registry_path = os.path.join(registry_home, "filebeat") self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", - registry_home="a/b/c/registry_x", + registry_home=registry_home, registry_file_permissions=0o640 ) @@ -266,7 +268,7 @@ def test_registry_file_update_permissions(self): # the logging and actual writing the file. Seems to happen on Windows. self.wait_until( lambda: self.has_registry(registry_path), - max_timeout=1) + max_timeout=10) # Wait a moment to make sure registry is completely written time.sleep(1) @@ -950,7 +952,7 @@ def test_restart_state(self): path=os.path.abspath(self.working_dir) + "/log/*", close_inactive="200ms", ignore_older="2000ms", - clean_inactive="3s", + clean_inactive="10s", ) filebeat = self.start_beat() @@ -976,7 +978,7 @@ def test_restart_state_reset(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", clean_inactive="10s", - ignore_older="5s" + ignore_older="9s" ) os.mkdir(self.working_dir + "/log/") @@ -1003,7 +1005,7 @@ def test_restart_state_reset(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/test2.log", clean_inactive="10s", - ignore_older="5s", + ignore_older="9s", ) filebeat = self.start_beat(output="filebeat2.log") @@ -1137,7 +1139,7 @@ def test_restart_state_reset_ttl_no_clean_inactive(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/test.log", clean_inactive="10s", - ignore_older="5s" + ignore_older="9s" ) os.mkdir(self.working_dir + "/log/") diff --git a/filebeat/tests/system/test_shutdown.py b/filebeat/tests/system/test_shutdown.py index 8f18337435f..8e781837730 100644 --- a/filebeat/tests/system/test_shutdown.py +++ b/filebeat/tests/system/test_shutdown.py @@ -27,7 +27,13 @@ def test_shutdown(self): ) for i in range(1, 5): proc = self.start_beat(logging_args=["-e", "-v"]) - time.sleep(.5) + + # Flaky on MacOS, see https://github.com/elastic/beats/issues/39613#issuecomment-2158812325 + # we need to wait a bit longer for filebeat to start + if platform.system() == "Darwin": + time.sleep(10) + else: + time.sleep(.5) proc.check_kill_and_wait() @unittest.skip("Skipped as flaky: https://github.com/elastic/beats/issues/14647") From de63284e8e45152fd99798c07732f8f90eab2d75 Mon Sep 17 00:00:00 2001 From: niraj-elastic <124254029+niraj-elastic@users.noreply.github.com> Date: Thu, 13 Jun 2024 21:20:00 +0530 Subject: [PATCH 6/8] [oracle] Missing total.bytes field in tablespace datastream (#39787) * fix total.bytes for tablespace * update changelog * fix CI errors * fix CI * fix CI issues * address review comments * address review comment * update changelog * address review comment * update visualization * update visualization name --------- Co-authored-by: Kush Rana <89848966+kush-elastic@users.noreply.github.com> --- CHANGELOG.next.asciidoc | 1 + .../862e2c20-9bf0-11e9-a61b-f742ed613c57.json | 2 +- .../05acae50-9bf0-11e9-a61b-f742ed613c57.json | 37 ++++++++++++++++--- .../module/oracle/tablespace/data.go | 20 +++++----- .../module/oracle/tablespace/data_test.go | 12 +++--- .../module/oracle/tablespace/mocks_test.go | 10 ++--- .../oracle/tablespace/temp_free_space.go | 21 ++++------- .../oracle/tablespace/used_and_free_space.go | 20 ++++------ 8 files changed, 70 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 15e4fd614db..372b8e35244 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -162,6 +162,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix issue where beats may report incorrect metrics for its own process when running inside a container {pull}39627[39627] - Fix for MySQL/Performance - Query failure for MySQL versions below v8.0.1, for performance metric `quantile_95`. {pull}38710[38710] - Normalize AWS RDS CPU Utilization values before making the metadata API call. {pull}39664[39664] +- Fix query logic for temp and non-temp tablespaces in Oracle module. {issue}38051[38051] {pull}39787[39787] *Osquerybeat* diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/862e2c20-9bf0-11e9-a61b-f742ed613c57.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/862e2c20-9bf0-11e9-a61b-f742ed613c57.json index f093d13a3e4..216971d16b5 100644 --- a/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/862e2c20-9bf0-11e9-a61b-f742ed613c57.json +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/862e2c20-9bf0-11e9-a61b-f742ed613c57.json @@ -46,7 +46,7 @@ }, "panelIndex": "2", "panelRefName": "panel_2", - "title": "Tablespace Total Size", + "title": "Tablespace Max Total Size", "type": "visualization", "version": "8.0.0-SNAPSHOT" }, diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json index 603fcc02ec2..27de5af579a 100644 --- a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json @@ -4,7 +4,7 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": {} }, - "title": "Tablespace Total Size [Metricbeat Oracle]", + "title": "Tablespace Max Total Size [Metricbeat Oracle]", "uiStateJSON": {}, "version": 1, "visState": { @@ -24,13 +24,40 @@ "fill": 0.5, "formatter": "bytes", "id": "61ca57f1-469d-11e7-af02-69e470af7417", - "label": "Tablespace total size", + "label": "Tablespace max total size", "line_width": 1, "metrics": [ { - "field": "oracle.tablespace.space.total.bytes", + "agg_with": "avg", + "field": "oracle.tablespace.space.used.bytes", "id": "61ca57f2-469d-11e7-af02-69e470af7417", - "type": "avg" + "type": "max" + }, + { + "agg_with": "avg", + "colors": [ + "#68BC00" + ], + "field": "oracle.tablespace.space.free.bytes", + "id": "e04e8f40-24cd-4066-b12c-da0db0ff73d4", + "type": "max" + }, + { + "id": "2cf57800-8b54-41fa-a877-159b49699a50", + "script": "params.used_bytes + params.free_bytes", + "type": "math", + "variables": [ + { + "field": "61ca57f2-469d-11e7-af02-69e470af7417", + "id": "631a44d5-d18a-4743-bea0-6f61930fd65f", + "name": "used_bytes" + }, + { + "field": "e04e8f40-24cd-4066-b12c-da0db0ff73d4", + "id": "c255d24c-3a29-4879-b999-77af43d97c6b", + "name": "free_bytes" + } + ] } ], "point_size": 1, @@ -48,7 +75,7 @@ "type": "timeseries", "use_kibana_indexes": false }, - "title": "Tablespace Total Size [Metricbeat Oracle]", + "title": "Tablespace Max Total Size [Metricbeat Oracle]", "type": "metrics" } }, diff --git a/x-pack/metricbeat/module/oracle/tablespace/data.go b/x-pack/metricbeat/module/oracle/tablespace/data.go index 30c86d766da..82f93a060b1 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/data.go +++ b/x-pack/metricbeat/module/oracle/tablespace/data.go @@ -16,8 +16,9 @@ import ( // extract is the E of a ETL processing. Gets the data files, used/free space and temp free space data that is fetch // by doing queries to Oracle -func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMethods) (out *extractedData, err error) { - out = &extractedData{} +func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMethods) (*extractedData, error) { + out := &extractedData{} + var err error if out.dataFiles, err = extractor.dataFilesData(ctx); err != nil { return nil, fmt.Errorf("error getting data_files: %w", err) @@ -31,23 +32,23 @@ func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMeth return nil, fmt.Errorf("error getting free space data: %w", err) } - return + return out, nil } // transform is the T of an ETL (refer to the 'extract' method above if you need to see the origin). Transforms the data // to create a Kibana/Elasticsearch friendly JSON. Data from Oracle is pretty fragmented by design so a lot of data // was necessary. Data is organized by Tablespace entity (Tablespaces might contain one or more data files) -func (m *MetricSet) transform(in *extractedData) (out map[string]mapstr.M) { - out = make(map[string]mapstr.M, 0) +func (m *MetricSet) transform(in *extractedData) map[string]mapstr.M { + out := make(map[string]mapstr.M, 0) - for _, dataFile := range in.dataFiles { - m.addDataFileData(&dataFile, out) + for i := range in.dataFiles { + m.addDataFileData(&in.dataFiles[i], out) } m.addUsedAndFreeSpaceData(in.freeSpace, out) m.addTempFreeSpaceData(in.tempFreeSpace, out) - return + return out } func (m *MetricSet) extractAndTransform(ctx context.Context) ([]mb.Event, error) { @@ -78,7 +79,7 @@ func (m *MetricSet) addTempFreeSpaceData(tempFreeSpaces []tempFreeSpace, out map name := val.(string) if name == "TEMP" { for _, tempFreeSpaceTable := range tempFreeSpaces { - oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.total.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.TablespaceSize}) + oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.total.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.TotalSpaceBytes}) oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.used.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.UsedSpaceBytes}) oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.free.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.FreeSpace}) } @@ -101,6 +102,7 @@ func (m *MetricSet) addUsedAndFreeSpaceData(freeSpaces []usedAndFreeSpace, out m if name == freeSpaceTable.TablespaceName { oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.free.bytes", &oracle.Int64Value{NullInt64: freeSpaceTable.TotalFreeBytes}) oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.used.bytes", &oracle.Int64Value{NullInt64: freeSpaceTable.TotalUsedBytes}) + oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.total.bytes", &oracle.Int64Value{NullInt64: freeSpaceTable.TotalSpaceBytes}) } } } diff --git a/x-pack/metricbeat/module/oracle/tablespace/data_test.go b/x-pack/metricbeat/module/oracle/tablespace/data_test.go index 02e4bdd4528..9c6990a2598 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/data_test.go +++ b/x-pack/metricbeat/module/oracle/tablespace/data_test.go @@ -12,13 +12,13 @@ import ( "github.com/stretchr/testify/assert" ) -var expectedResults = []string{`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux01.dbf","online_status":"ONLINE","size":{"bytes":9999990,"free":{"bytes":99999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`, - `{"data_file":{"id":181,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux02.dbf","online_status":"ONLINE","size":{"bytes":9999991,"free":{"bytes":99999995},"max":{"bytes":9999995}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`, - `{"data_file":{"id":182,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux03.dbf","online_status":"ONLINE","size":{"bytes":9999992,"free":{"bytes":99999996},"max":{"bytes":9999996}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`, - `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/system01.dbf","online_status":"ONLINE","size":{"bytes":999990,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSTEM","space":{"free":{"bytes":9990},"used":{"bytes":9991}}}`, +var expectedResults = []string{`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux01.dbf","online_status":"ONLINE","size":{"bytes":9999990,"free":{"bytes":99999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`, + `{"data_file":{"id":181,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux02.dbf","online_status":"ONLINE","size":{"bytes":9999991,"free":{"bytes":99999995},"max":{"bytes":9999995}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`, + `{"data_file":{"id":182,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux03.dbf","online_status":"ONLINE","size":{"bytes":9999992,"free":{"bytes":99999996},"max":{"bytes":9999996}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`, + `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/system01.dbf","online_status":"ONLINE","size":{"bytes":999990,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSTEM","space":{"free":{"bytes":9990},"total":{"bytes":99999},"used":{"bytes":9991}}}`, `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/temp012017-03-02_07-54-38-075-AM.dbf","online_status":"ONLINE","size":{"bytes":999991,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"TEMP","space":{"free":{"bytes":99999},"total":{"bytes":99999},"used":{"bytes":99999}}}`, - `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/undotbs01.dbf","online_status":"ONLINE","size":{"bytes":999992,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"UNDOTBS1","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`, - `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/users01.dbf","online_status":"ONLINE","size":{"bytes":999993,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"USERS","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`} + `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/undotbs01.dbf","online_status":"ONLINE","size":{"bytes":999992,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"UNDOTBS1","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`, + `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/users01.dbf","online_status":"ONLINE","size":{"bytes":999993,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"USERS","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`} var notExpectedEvents = []string{`{}`, `{"foo":"bar"}`} diff --git a/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go b/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go index 12348236bcc..9f5800166af 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go +++ b/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go @@ -77,16 +77,16 @@ func (h happyDataFiles) dataFilesData(_ context.Context) ([]dataFile, error) { type happyTempFreeSpaceData struct{} func (happyTempFreeSpaceData) tempFreeSpaceData(_ context.Context) ([]tempFreeSpace, error) { - return []tempFreeSpace{{TablespaceName: "TEMP", TablespaceSize: sql.NullInt64{Valid: true, Int64: 99999}, UsedSpaceBytes: sql.NullInt64{Valid: true, Int64: 99999}, FreeSpace: sql.NullInt64{Int64: 99999, Valid: true}}}, nil + return []tempFreeSpace{{TablespaceName: "TEMP", TotalSpaceBytes: sql.NullInt64{Valid: true, Int64: 99999}, UsedSpaceBytes: sql.NullInt64{Valid: true, Int64: 99999}, FreeSpace: sql.NullInt64{Int64: 99999, Valid: true}}}, nil } type happyFreeSpaceData struct{} func (happyFreeSpaceData) usedAndFreeSpaceData(_ context.Context) ([]usedAndFreeSpace, error) { return []usedAndFreeSpace{ - {TablespaceName: "SYSTEM", TotalFreeBytes: sql.NullInt64{Int64: 9990, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}}, - {TablespaceName: "SYSAUX", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}}, - {TablespaceName: "UNDOTBS1", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}}, - {TablespaceName: "USERS", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}}, + {TablespaceName: "SYSTEM", TotalFreeBytes: sql.NullInt64{Int64: 9990, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}}, + {TablespaceName: "SYSAUX", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}}, + {TablespaceName: "UNDOTBS1", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}}, + {TablespaceName: "USERS", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}}, }, nil } diff --git a/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go b/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go index efc8401e2c0..f4b592d2b75 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go +++ b/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go @@ -11,31 +11,24 @@ import ( ) type tempFreeSpace struct { - TablespaceName string - TablespaceSize sql.NullInt64 - UsedSpaceBytes sql.NullInt64 - FreeSpace sql.NullInt64 -} - -func (d *tempFreeSpace) hash() string { - return d.TablespaceName -} - -func (d *tempFreeSpace) eventKey() string { - return d.TablespaceName + TablespaceName string + TotalSpaceBytes sql.NullInt64 + UsedSpaceBytes sql.NullInt64 + FreeSpace sql.NullInt64 } func (e *tablespaceExtractor) tempFreeSpaceData(ctx context.Context) ([]tempFreeSpace, error) { - rows, err := e.db.QueryContext(ctx, "SELECT TABLESPACE_NAME, TABLESPACE_SIZE, ALLOCATED_SPACE, FREE_SPACE FROM DBA_TEMP_FREE_SPACE") + rows, err := e.db.QueryContext(ctx, `SELECT t.TABLESPACE_NAME, (SELECT SUM(BYTES) FROM DBA_DATA_FILES) + (SELECT SUM(BYTES) FROM DBA_TEMP_FILES) AS TOTAL_SUM, t.ALLOCATED_SPACE, t.FREE_SPACE FROM DBA_TEMP_FREE_SPACE t `) if err != nil { return nil, fmt.Errorf("error executing query: %w", err) } + defer rows.Close() results := make([]tempFreeSpace, 0) for rows.Next() { dest := tempFreeSpace{} - if err = rows.Scan(&dest.TablespaceName, &dest.TablespaceSize, &dest.UsedSpaceBytes, &dest.FreeSpace); err != nil { + if err = rows.Scan(&dest.TablespaceName, &dest.TotalSpaceBytes, &dest.UsedSpaceBytes, &dest.FreeSpace); err != nil { return nil, err } results = append(results, dest) diff --git a/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go b/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go index b17b249808d..e327badcf02 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go +++ b/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go @@ -11,30 +11,24 @@ import ( ) type usedAndFreeSpace struct { - TablespaceName string - TotalFreeBytes sql.NullInt64 - TotalUsedBytes sql.NullInt64 -} - -func (d *usedAndFreeSpace) hash() string { - return d.TablespaceName -} - -func (d *usedAndFreeSpace) eventKey() string { - return d.TablespaceName + TablespaceName string + TotalSpaceBytes sql.NullInt64 + TotalFreeBytes sql.NullInt64 + TotalUsedBytes sql.NullInt64 } func (e *tablespaceExtractor) usedAndFreeSpaceData(ctx context.Context) ([]usedAndFreeSpace, error) { - rows, err := e.db.QueryContext(ctx, "SELECT b.tablespace_name, tbs_size used, a.free_space free FROM (SELECT tablespace_name, sum(bytes) AS free_space FROM dba_free_space GROUP BY tablespace_name) a, (SELECT tablespace_name, sum(bytes) AS tbs_size FROM dba_data_files GROUP BY tablespace_name) b WHERE a.tablespace_name(+)=b.tablespace_name") + rows, err := e.db.QueryContext(ctx, `SELECT b.tablespace_name, (b.tbs_size - NVL(a.free_space, 0)) AS used, NVL(a.free_space, 0) AS free, (SELECT SUM(bytes) FROM DBA_DATA_FILES) + (SELECT SUM(bytes) FROM DBA_TEMP_FILES) AS total_sum FROM (SELECT tablespace_name, SUM(bytes) AS free_space FROM DBA_FREE_SPACE GROUP BY tablespace_name) a RIGHT JOIN (SELECT tablespace_name, SUM(bytes) AS tbs_size FROM DBA_DATA_FILES GROUP BY tablespace_name) b ON a.tablespace_name = b.tablespace_name`) if err != nil { return nil, fmt.Errorf("error executing query: %w", err) } + defer rows.Close() results := make([]usedAndFreeSpace, 0) for rows.Next() { dest := usedAndFreeSpace{} - if err = rows.Scan(&dest.TablespaceName, &dest.TotalUsedBytes, &dest.TotalFreeBytes); err != nil { + if err = rows.Scan(&dest.TablespaceName, &dest.TotalUsedBytes, &dest.TotalFreeBytes, &dest.TotalSpaceBytes); err != nil { return nil, err } results = append(results, dest) From 974ace65bda4c9f68e5b002fad0b14b942a8bed2 Mon Sep 17 00:00:00 2001 From: Alex K <8418476+fearful-symmetry@users.noreply.github.com> Date: Thu, 13 Jun 2024 08:52:44 -0700 Subject: [PATCH 7/8] Add section on cgroupns usage (#39889) --- metricbeat/docs/running-on-docker.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metricbeat/docs/running-on-docker.asciidoc b/metricbeat/docs/running-on-docker.asciidoc index 72cfed9757d..26ba27658b8 100644 --- a/metricbeat/docs/running-on-docker.asciidoc +++ b/metricbeat/docs/running-on-docker.asciidoc @@ -21,6 +21,7 @@ docker run \ --mount type=bind,source=/var/run/dbus/system_bus_socket,target=/hostfs/var/run/dbus/system_bus_socket,readonly \ <4> --env DBUS_SYSTEM_BUS_ADDRESS='unix:path=/hostfs/var/run/dbus/system_bus_socket' \ <4> --net=host \ <5> + --cgroupns=host \ <6> {dockerimage} -e -system.hostfs=/hostfs ---- @@ -45,6 +46,7 @@ both require access to dbus. Mount the dbus socket and set the `DBUS_SYSTEM_BUS_ to make this file contain the host's network devices is to use the `--net=host` flag. This is due to Linux namespacing; simply bind mounting the host's `/proc` to `/hostfs/proc` is not sufficient. +<6> Runs the container using the host's cgroup namespace, instead of a private namespace. While this is optional, <> may produce more correct cgroup metrics when running in host mode. NOTE: The special filesystems +/proc+ and +/sys+ are only available if the host system is running Linux. Attempts to bind-mount these filesystems will From 9d2b48da6dbb16b3ed65717669acb06fda7da245 Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Thu, 13 Jun 2024 20:33:14 +0200 Subject: [PATCH 8/8] refactor: replace uber atomic with stdlib atomic types (#39853) * refactor: replace uber atomic with stdlib atomic types Go 1.19 added new atomic types. Migrate usage of go.uber.org/atomic to sync/atomic and drop the dependency. * refactor: store concrete type instead of pointer * Update input.go * lint: fix linting errors --- NOTICE.txt | 58 +++++++++---------- go.mod | 2 +- .../filebeat/input/http_endpoint/handler.go | 8 +-- x-pack/filebeat/input/http_endpoint/input.go | 8 +-- .../input/internal/httplog/roundtripper.go | 23 ++++---- .../manager/aws/event_stack_poller_test.go | 4 +- 6 files changed, 50 insertions(+), 53 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index f6881a796c0..c476eea8226 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -24852,35 +24852,6 @@ Contents of probable licence file $GOMODCACHE/go.mongodb.org/mongo-driver@v1.5.1 limitations under the License. --------------------------------------------------------------------------------- -Dependency : go.uber.org/atomic -Version: v1.11.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/go.uber.org/atomic@v1.11.0/LICENSE.txt: - -Copyright (c) 2016 Uber Technologies, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - -------------------------------------------------------------------------------- Dependency : go.uber.org/multierr Version: v1.11.0 @@ -55030,6 +55001,35 @@ Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/otel/trace@v1. limitations under the License. +-------------------------------------------------------------------------------- +Dependency : go.uber.org/atomic +Version: v1.11.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/go.uber.org/atomic@v1.11.0/LICENSE.txt: + +Copyright (c) 2016 Uber Technologies, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : go.uber.org/goleak Version: v1.3.0 diff --git a/go.mod b/go.mod index 57a93552539..3647392e0c2 100644 --- a/go.mod +++ b/go.mod @@ -150,7 +150,7 @@ require ( go.elastic.co/ecszap v1.0.2 go.elastic.co/go-licence-detector v0.6.0 go.etcd.io/bbolt v1.3.6 - go.uber.org/atomic v1.11.0 + go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.22.0 diff --git a/x-pack/filebeat/input/http_endpoint/handler.go b/x-pack/filebeat/input/http_endpoint/handler.go index b799248a935..4f3fdd550aa 100644 --- a/x-pack/filebeat/input/http_endpoint/handler.go +++ b/x-pack/filebeat/input/http_endpoint/handler.go @@ -18,13 +18,13 @@ import ( "sort" "strconv" "strings" + "sync/atomic" "time" "github.com/google/cel-go/cel" "github.com/google/cel-go/checker/decls" "github.com/google/cel-go/common/types" "github.com/google/cel-go/common/types/ref" - "go.uber.org/atomic" "go.uber.org/zap" "go.uber.org/zap/zapcore" "google.golang.org/protobuf/types/known/structpb" @@ -53,8 +53,8 @@ type handler struct { publish func(beat.Event) log *logp.Logger validator apiValidator - txBaseID string // Random value to make transaction IDs unique. - txIDCounter *atomic.Uint64 // Transaction ID counter that is incremented for each request. + txBaseID string // Random value to make transaction IDs unique. + txIDCounter atomic.Uint64 // Transaction ID counter that is incremented for each request. reqLogger *zap.Logger host, scheme string @@ -290,7 +290,7 @@ func (h *handler) logRequest(txID string, r *http.Request, status int, respBody } func (h *handler) nextTxID() string { - count := h.txIDCounter.Inc() + count := h.txIDCounter.Add(1) return h.formatTxID(count) } diff --git a/x-pack/filebeat/input/http_endpoint/input.go b/x-pack/filebeat/input/http_endpoint/input.go index 6737a9b9aa0..7f0440deb60 100644 --- a/x-pack/filebeat/input/http_endpoint/input.go +++ b/x-pack/filebeat/input/http_endpoint/input.go @@ -24,7 +24,6 @@ import ( "github.com/rcrowley/go-metrics" "go.elastic.co/ecszap" - "go.uber.org/atomic" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -327,10 +326,9 @@ func (s *server) getErr() error { func newHandler(ctx context.Context, c config, prg *program, pub func(beat.Event), log *logp.Logger, metrics *inputMetrics) http.Handler { h := &handler{ - ctx: ctx, - log: log, - txBaseID: newID(), - txIDCounter: atomic.NewUint64(0), + ctx: ctx, + log: log, + txBaseID: newID(), publish: pub, metrics: metrics, diff --git a/x-pack/filebeat/input/internal/httplog/roundtripper.go b/x-pack/filebeat/input/internal/httplog/roundtripper.go index ce68147a2a7..9e60cb60942 100644 --- a/x-pack/filebeat/input/internal/httplog/roundtripper.go +++ b/x-pack/filebeat/input/internal/httplog/roundtripper.go @@ -15,9 +15,9 @@ import ( "net/http" "net/http/httputil" "strconv" + "sync/atomic" "time" - "go.uber.org/atomic" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -36,22 +36,21 @@ type contextKey string // responses to the provided logger. Transaction creation is logged to log. func NewLoggingRoundTripper(next http.RoundTripper, logger *zap.Logger, maxBodyLen int, log *logp.Logger) *LoggingRoundTripper { return &LoggingRoundTripper{ - transport: next, - maxBodyLen: maxBodyLen, - txLog: logger, - txBaseID: newID(), - txIDCounter: atomic.NewUint64(0), - log: log, + transport: next, + maxBodyLen: maxBodyLen, + txLog: logger, + txBaseID: newID(), + log: log, } } // LoggingRoundTripper is an http.RoundTripper that logs requests and responses. type LoggingRoundTripper struct { transport http.RoundTripper - maxBodyLen int // The maximum length of a body. Longer bodies will be truncated. - txLog *zap.Logger // Destination logger. - txBaseID string // Random value to make transaction IDs unique. - txIDCounter *atomic.Uint64 // Transaction ID counter that is incremented for each request. + maxBodyLen int // The maximum length of a body. Longer bodies will be truncated. + txLog *zap.Logger // Destination logger. + txBaseID string // Random value to make transaction IDs unique. + txIDCounter atomic.Uint64 // Transaction ID counter that is incremented for each request. log *logp.Logger } @@ -220,7 +219,7 @@ func (rt *LoggingRoundTripper) TxID() string { // nextTxID returns the next transaction.id value. It increments the internal // request counter. func (rt *LoggingRoundTripper) nextTxID() string { - count := rt.txIDCounter.Inc() + count := rt.txIDCounter.Add(1) return rt.formatTxID(count) } diff --git a/x-pack/functionbeat/manager/aws/event_stack_poller_test.go b/x-pack/functionbeat/manager/aws/event_stack_poller_test.go index fe7473d0f3d..3679f07fb54 100644 --- a/x-pack/functionbeat/manager/aws/event_stack_poller_test.go +++ b/x-pack/functionbeat/manager/aws/event_stack_poller_test.go @@ -7,6 +7,7 @@ package aws import ( "context" "strconv" + "sync/atomic" "testing" "time" @@ -14,7 +15,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/cloudformation" "github.com/stretchr/testify/assert" - "go.uber.org/atomic" "github.com/elastic/elastic-agent-libs/logp" ) @@ -29,7 +29,7 @@ func (m *mockEventHandler) sync(event types.StackEvent) bool { if m.skipCount.Load() >= m.skipEvents { return false } - m.skipCount.Inc() + m.skipCount.Add(1) return true }