From bb898001f52042c72ab87651096cb63bee9a9e58 Mon Sep 17 00:00:00 2001 From: ev1yehor <146825775+ev1yehor@users.noreply.github.com> Date: Wed, 12 Jun 2024 13:21:39 +0300 Subject: [PATCH 01/26] Migrate github check for Filebeat and x-pack/filebeat to Buildkite pipeline (#39751) * Update filebeat-pipeline.yml * Update filebeat-pipeline.yml * Update filebeat-pipeline.yml * Update filebeat-pipeline.yml * Update filebeat-pipeline.yml * Update filebeat-pipeline.yml * Update filebeat-pipeline.yml * Update filebeat-pipeline.yml * Update filebeat-pipeline.yml * Update filebeat-pipeline.yml * Update filebeat-pipeline.yml * Migrate check * Update filebeat-pipeline.yml * Update pipeline.xpack.filebeat.yml * Update pipeline.xpack.filebeat.yml * Update pipeline.xpack.filebeat.yml * Update --- .buildkite/filebeat/filebeat-pipeline.yml | 26 +++++++++++++++ .buildkite/x-pack/pipeline.xpack.filebeat.yml | 27 ++++++++++++++++ .github/workflows/check-filebeat.yml | 32 ------------------- 3 files changed, 53 insertions(+), 32 deletions(-) delete mode 100644 .github/workflows/check-filebeat.yml diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 87f1925a372..849720d1e2d 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -27,6 +27,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "Filebeat Check/Update" + key: "filebeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C filebeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.3" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "filebeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "filebeat-check-update" + - group: "Filebeat Mandatory Tests" key: "filebeat-mandatory-tests" steps: diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index 1aa1c51ca5f..ce45e798239 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -26,6 +26,33 @@ env: TEST_COVERAGE: "true" steps: + - group: "x-pack/filebeat Check/Update" + key: "x-pack-filebeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C x-pack/filebeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.3" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "x-pack-filebeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "x-pack-filebeat-check-update" + + - group: "x-pack/filebeat Mandatory Tests" key: "x-pack-filebeat-mandatory-tests" steps: diff --git a/.github/workflows/check-filebeat.yml b/.github/workflows/check-filebeat.yml deleted file mode 100644 index 930a04ec5e5..00000000000 --- a/.github/workflows/check-filebeat.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: check-filebeat - -on: - pull_request: - paths: - - '.github/workflows/check-filebeat.yml' - - 'filebeat/**' - - 'x-pack/filebeat/**' - -env: - BEAT_MODULE: 'filebeat' - -permissions: - contents: read - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - name: Update package lists - run: sudo apt-get update - - name: Install libsystemd-dev - run: sudo apt-get install -y libsystemd-dev - - name: Run check/update - run: | - go install github.com/magefile/mage - make -C ${{ env.BEAT_MODULE }} check update - make check-no-changes From f8679db91a544f6439ce3d4b4c8f2098abd2fcf0 Mon Sep 17 00:00:00 2001 From: ev1yehor <146825775+ev1yehor@users.noreply.github.com> Date: Wed, 12 Jun 2024 16:13:56 +0300 Subject: [PATCH 02/26] Migrate github checks to buildkite for packetbeat and x-pack/packetbeat (#39871) * update * update --- .buildkite/packetbeat/pipeline.packetbeat.yml | 26 +++++++++++++++ .../x-pack/pipeline.xpack.packetbeat.yml | 26 +++++++++++++++ .github/workflows/check-packetbeat.yml | 32 ------------------- 3 files changed, 52 insertions(+), 32 deletions(-) delete mode 100644 .github/workflows/check-packetbeat.yml diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index c9a5c9f06f2..40e5f0f4b6f 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -25,6 +25,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "Packetbeat Check/Update" + key: "packetbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C packetbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "packetbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "packetbeat-check-update" + - group: "packetbeat Mandatory Tests" key: "packetbeat-mandatory-tests" steps: diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml index 542fee787c1..71027426992 100644 --- a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml @@ -27,6 +27,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "x-pack/packetbeat Check/Update" + key: "x-pack-packetbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C x-pack/packetbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "x-pack/packetbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "x-pack-packetbeat-check-update" + - group: "x-pack/packetbeat Mandatory Tests" key: "x-pack-packetbeat-mandatory-tests" diff --git a/.github/workflows/check-packetbeat.yml b/.github/workflows/check-packetbeat.yml deleted file mode 100644 index ba05b6c0160..00000000000 --- a/.github/workflows/check-packetbeat.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: check-packetbeat - -on: - pull_request: - paths: - - '.github/workflows/check-packetbeat.yml' - - 'packetbeat/**' - - 'x-pack/packetbeat/**' - -env: - BEAT_MODULE: 'packetbeat' - -permissions: - contents: read - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - name: Update package lists - run: sudo apt-get update - - name: Install libpcap-dev - run: sudo apt-get install -y libpcap-dev - - name: Run check/update - run: | - go install github.com/magefile/mage - make -C ${{ env.BEAT_MODULE }} check update - make check-no-changes From 02a57f76489ac349d24fd2a36818559ff314a624 Mon Sep 17 00:00:00 2001 From: ev1yehor <146825775+ev1yehor@users.noreply.github.com> Date: Wed, 12 Jun 2024 16:14:12 +0300 Subject: [PATCH 03/26] Migrate github checks to buildkite for libbeat and x-pack/libbeat (#39869) * Update * Update pipeline.xpack.libbeat.yml * Update pipeline.xpack.libbeat.yml * update --- .buildkite/libbeat/pipeline.libbeat.yml | 26 ++++++++++++++++ .buildkite/x-pack/pipeline.xpack.libbeat.yml | 26 ++++++++++++++++ .github/workflows/check-libbeat.yml | 32 -------------------- 3 files changed, 52 insertions(+), 32 deletions(-) delete mode 100644 .github/workflows/check-libbeat.yml diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index fd485279858..7ebe5d76f98 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -16,6 +16,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "libbeat Check/Update" + key: "libbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C libbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "libbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "libbeat-check-update" + - group: "Mandatory Tests" key: "mandatory-tests" steps: diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index 33356c78c46..93f57713ee3 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -24,6 +24,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "x-pack/libbeat Check/Update" + key: "x-pack-libbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C x-pack/libbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "x-pack/libbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "x-pack-libbeat-check-update" + - group: "x-pack/libbeat Mandatory Tests" key: "x-pack-libbeat-mandatory-tests" diff --git a/.github/workflows/check-libbeat.yml b/.github/workflows/check-libbeat.yml deleted file mode 100644 index 27e03701b85..00000000000 --- a/.github/workflows/check-libbeat.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: check-libbeat - -on: - pull_request: - paths: - - '.github/workflows/check-packetbeat.yml' - - 'libbeat/**' - - 'x-pack/libbeat/**' - -env: - BEAT_MODULE: 'libbeat' - -permissions: - contents: read - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - name: Update package lists - run: sudo apt-get update - - name: Install libpcap-dev - run: sudo apt-get install -y libpcap-dev - - name: Run check/update - run: | - go install github.com/magefile/mage - make -C ${{ env.BEAT_MODULE }} check update - make check-no-changes From 8bcf0babf2a7fd397520858d0a7712d4d3d5fd31 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 12 Jun 2024 16:27:48 +0300 Subject: [PATCH 04/26] Allow github-actions bot to trigger builds (#39874) This commit adds GitHub-actions to the list of bots that are allowed to trigger BK builds. Signed-off-by: Alexandros Sapranidis --- .buildkite/pull-requests.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 55affae4128..b2cbb06e70f 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -5,7 +5,7 @@ "pipelineSlug": "beats", "allow_org_users": true, "allowed_repo_permissions": ["admin", "write"], - "allowed_list": ["dependabot[bot]", "mergify[bot]"], + "allowed_list": ["dependabot[bot]", "mergify[bot]", "github-actions[bot]"], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, @@ -21,7 +21,7 @@ "pipelineSlug": "beats-xpack-elastic-agent", "allow_org_users": true, "allowed_repo_permissions": ["admin", "write"], - "allowed_list": ["dependabot[bot]", "mergify[bot]"], + "allowed_list": ["dependabot[bot]", "mergify[bot]", "github-actions[bot]"], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, From 60ba7bb46d0396f71ae54c9d3cd0979f3919cf5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tanja=20Mili=C4=8Di=C4=87?= <156105538+tanja-milicic@users.noreply.github.com> Date: Wed, 12 Jun 2024 16:02:27 +0200 Subject: [PATCH 05/26] Update opendashboards.asciidoc to rename index pattern to data view (#39827) --- libbeat/docs/shared/opendashboards.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/docs/shared/opendashboards.asciidoc b/libbeat/docs/shared/opendashboards.asciidoc index 7e73fbc8f4b..1b942504998 100644 --- a/libbeat/docs/shared/opendashboards.asciidoc +++ b/libbeat/docs/shared/opendashboards.asciidoc @@ -13,7 +13,7 @@ include::{libbeat-dir}/tab-widgets/open-kibana-widget.asciidoc[] -- . In the side navigation, click *Discover*. To see {beatname_uc} data, make -sure the predefined +{beatname_lc}-*+ index pattern is selected. +sure the predefined +{beatname_lc}-*+ data view is selected. + -- TIP: If you don’t see data in {kib}, try changing the time filter to a larger From 09707a2a279e7b8c0a55f3b19ac76d3754013140 Mon Sep 17 00:00:00 2001 From: VihasMakwana <121151420+VihasMakwana@users.noreply.github.com> Date: Wed, 12 Jun 2024 20:46:20 +0530 Subject: [PATCH 06/26] [docs][filebeat]: add 4th step in migrate-to-filestream (#39863) * docs: add 4th step in migrate-to-filestream * Update filebeat/docs/howto/migrate-to-filestream.asciidoc Co-authored-by: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> --------- Co-authored-by: Pierre HILBERT Co-authored-by: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> --- filebeat/docs/howto/migrate-to-filestream.asciidoc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/filebeat/docs/howto/migrate-to-filestream.asciidoc b/filebeat/docs/howto/migrate-to-filestream.asciidoc index 30057fab725..55c05a5b46d 100644 --- a/filebeat/docs/howto/migrate-to-filestream.asciidoc +++ b/filebeat/docs/howto/migrate-to-filestream.asciidoc @@ -234,6 +234,13 @@ The example configuration shown earlier needs to be adjusted as well: |backoff.max |=== +=== Step 4 + +The events produced by `filestream` input with `take_over: true` contain a `take_over` tag. +You can filter on this tag in Kibana and see the events which came from a filestream in the "take_over" mode. + +Once you start receiving events with this tag, you can remove `take_over: true` and restart the fileinput again. + === If something went wrong If for whatever reason you'd like to revert the configuration after running the migrated configuration From dc876a0a1dfb89c23a69a9f2fac01d2f55301cae Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 12 Jun 2024 16:55:31 +0000 Subject: [PATCH 07/26] [updatecli] update elastic stack version for testing 8.15.0-04e5793a (#39670) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Denis --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 5b6dd6ffd00..a5fb79fb02c 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.15.0-9bce1e63-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.15.0-d2070ed2-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.15.0-9bce1e63-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.15.0-d2070ed2-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.15.0-9bce1e63-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.15.0-d2070ed2-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 9e2ef861d608d9299985a87569229207dc4a3bd3 Mon Sep 17 00:00:00 2001 From: ev1yehor <146825775+ev1yehor@users.noreply.github.com> Date: Thu, 13 Jun 2024 00:44:25 +0300 Subject: [PATCH 08/26] Update (#39879) --- .buildkite/winlogbeat/pipeline.winlogbeat.yml | 26 +++++++++++++++++ .../x-pack/pipeline.xpack.winlogbeat.yml | 26 +++++++++++++++++ .github/workflows/check-winlogbeat.yml | 28 ------------------- 3 files changed, 52 insertions(+), 28 deletions(-) delete mode 100644 .github/workflows/check-winlogbeat.yml diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml index c598224438f..4d8e468595d 100644 --- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -23,6 +23,32 @@ env: DOCKER_PULL: 0 steps: + - group: "Winlogbeat Check/Update" + key: "winlogbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C winlogbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "winlogbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "winlogbeat-check-update" + - group: "Winlogbeat Mandatory Tests" key: "winlogbeat-mandatory-tests" diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index 401517960aa..fc8ce29a407 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -19,6 +19,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "x-pack/winlogbeat Check/Update" + key: "x-pack-winlogbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C x-pack/winlogbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "x-pack/winlogbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "x-pack-winlogbeat-check-update" + - group: "x-pack/winlogbeat Mandatory Tests" key: "x-pack-winlogbeat-mandatory-tests" steps: diff --git a/.github/workflows/check-winlogbeat.yml b/.github/workflows/check-winlogbeat.yml deleted file mode 100644 index a79c4bef209..00000000000 --- a/.github/workflows/check-winlogbeat.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: check-winlogbeat - -on: - pull_request: - paths: - - '.github/workflows/check-winlogbeat.yml' - - 'winlogbeat/**' - - 'x-pack/winlogbeat/**' - -env: - BEAT_MODULE: 'winlogbeat' - -permissions: - contents: read - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - name: Run check/update - run: | - go install github.com/magefile/mage - make -C ${{ env.BEAT_MODULE }} check update - make check-no-changes From ea9575a157b5cdffe342caea952e08abd591e2b3 Mon Sep 17 00:00:00 2001 From: subham sarkar Date: Thu, 13 Jun 2024 13:37:44 +0530 Subject: [PATCH 09/26] x-pack/filebeat/input/httpjson: Close connections properly (#39790) --- CHANGELOG-developer.next.asciidoc | 1 + x-pack/filebeat/input/httpjson/policy.go | 6 ++++++ x-pack/filebeat/input/httpjson/request.go | 7 +------ 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 995b52511b8..29c9ce99f49 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -100,6 +100,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only. - Skip flakey metrics test on windows in filebeat httpjson input. {issue}39676[39676] {pull}39678[39678] - Fix flakey test on Windows 2022 in packetbeat/route. {issue}39698[39698] {pull}39822[39822] - Fix bug in minimum length for request trace logging. {pull}39834[39834] +- Close connections properly in Filbeat's HTTPJSON input. {pull}39790[39790] ==== Added diff --git a/x-pack/filebeat/input/httpjson/policy.go b/x-pack/filebeat/input/httpjson/policy.go index 0c671cb85bb..43360c1ed0f 100644 --- a/x-pack/filebeat/input/httpjson/policy.go +++ b/x-pack/filebeat/input/httpjson/policy.go @@ -91,6 +91,12 @@ func (p *Policy) CustomRetryPolicy(ctx context.Context, resp *http.Response, err // errors and may relate to outages on the server side. This will catch // invalid response codes as well, like 0 and 999. if resp.StatusCode == 0 || (resp.StatusCode >= 500 && resp.StatusCode != 501) { + defer func() { + if resp.Body != nil { + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + } + }() return true, nil } diff --git a/x-pack/filebeat/input/httpjson/request.go b/x-pack/filebeat/input/httpjson/request.go index 3e63f026716..b15f3db51b1 100644 --- a/x-pack/filebeat/input/httpjson/request.go +++ b/x-pack/filebeat/input/httpjson/request.go @@ -676,6 +676,7 @@ func (r *requester) processChainPaginationEvents(ctx context.Context, trCtx *tra if err != nil { return -1, fmt.Errorf("failed to collect response: %w", err) } + // store data according to response type if i == len(r.requestFactories)-1 && len(ids) != 0 { finalResps = append(finalResps, httpResp) @@ -702,12 +703,6 @@ func (r *requester) processChainPaginationEvents(ctx context.Context, trCtx *tra n += p.eventCount() } - defer func() { - if httpResp != nil && httpResp.Body != nil { - httpResp.Body.Close() - } - }() - return n, nil } From eadec29bfa121038abfa94c315343b2d3ee8ba62 Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Thu, 13 Jun 2024 12:01:23 +0300 Subject: [PATCH 10/26] added docker-compose to changesets for x-pack/filebeat and x-pack/metricbeat (#39880) Added docker-compose to changesets for x-pack/filebeat and x-pack/metricbeat --- .buildkite/pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 52768481bff..712b6200697 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -116,6 +116,7 @@ steps: - .buildkite/x-pack/pipeline.xpack.filebeat.yml - .buildkite/scripts - .buildkite/hooks/ + - .buildkite/deploy/docker/docker-compose.yml #OSS - go.mod - pytest.ini @@ -219,6 +220,7 @@ steps: - .buildkite/x-pack/pipeline.xpack.metricbeat.yml - .buildkite/scripts - .buildkite/hooks/ + - .buildkite/deploy/docker/docker-compose.yml #OSS - go.mod - pytest.ini From 639fc23430d70816b0a1262307a6ff2d0e98fd2d Mon Sep 17 00:00:00 2001 From: ev1yehor <146825775+ev1yehor@users.noreply.github.com> Date: Thu, 13 Jun 2024 15:54:04 +0300 Subject: [PATCH 11/26] Migrate github checks to buildkite for auditbeat and x-pack/auditbeat (#39868) * Update * image update --- .buildkite/auditbeat/auditbeat-pipeline.yml | 27 ++++++++++++++++++ .../x-pack/pipeline.xpack.auditbeat.yml | 27 ++++++++++++++++++ .github/workflows/check-auditbeat.yml | 28 ------------------- 3 files changed, 54 insertions(+), 28 deletions(-) delete mode 100644 .github/workflows/check-auditbeat.yml diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 2ac4a8911a1..f5ca39b3645 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -27,6 +27,33 @@ env: TEST_COVERAGE: "true" steps: + - group: "Auditbeat Check/Update" + key: "auditbeat-check-update" + + steps: + - label: "Run check/update" + command: | + make -C auditbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "auditbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "auditbeat-check-update" + - group: "Auditbeat Mandatory Testing" key: "auditbeat-mandatory-tests" diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml index a851f681d33..9552cc14ef9 100644 --- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml @@ -27,6 +27,33 @@ env: TEST_COVERAGE: "true" steps: + - group: "x-pack/auditbeat Check/Update" + key: "x-pack-auditbeat-check-update" + + steps: + - label: "Run check/update" + command: | + make -C x-pack/auditbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "x-pack/auditbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "x-pack-auditbeat-check-update" + - group: "x-pack/auditbeat Mandatory Tests" key: "x-pack-auditbeat-mandatory-tests" diff --git a/.github/workflows/check-auditbeat.yml b/.github/workflows/check-auditbeat.yml deleted file mode 100644 index bbc96242687..00000000000 --- a/.github/workflows/check-auditbeat.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: check-auditbeat - -on: - pull_request: - paths: - - '.github/workflows/check-auditbeat.yml' - - 'auditbeat/**' - - 'x-pack/auditbeat/**' - -env: - BEAT_MODULE: 'auditbeat' - -permissions: - contents: read - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - name: Run check/update - run: | - go install github.com/magefile/mage - make -C ${{ env.BEAT_MODULE }} check update - make check-no-changes From b5155a177edea5ca97b6f5aaecc020d95d8350f7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 13 Jun 2024 08:57:32 -0400 Subject: [PATCH 12/26] [Automation] Bump Golang version to 1.21.11 (#39851) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: Update Auditbeat Dockerfile Made with ❤️️ by updatecli * chore: Update Filebeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update from vsphere Dockerfile Made with ❤️️ by updatecli * chore: Update Heartbeat Dockerfile Made with ❤️️ by updatecli * chore: Update NATS module Dockerfile Made with ❤️️ by updatecli * chore: Update version.asciidoc Made with ❤️️ by updatecli * chore: Update Metricbeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update Metricbeat Dockerfile Made with ❤️️ by updatecli * chore: Update HTTP module Dockerfile Made with ❤️️ by updatecli * chore: Update .go-version Made with ❤️️ by updatecli * chore: Update go.mod Made with ❤️️ by updatecli * chore: Update Functionbeat Dockerfile Made with ❤️️ by updatecli * chore: Update Packetbeat Dockerfile Made with ❤️️ by updatecli * chore: Update stan Dockerfile Made with ❤️️ by updatecli * chore: Update .golangci.yml Made with ❤️️ by updatecli * chore: Update Heartbeat debug Dockerfile Made with ❤️️ by updatecli * Update changelog. * Run go mod tidy. * chore: Update .go-version Made with ❤️️ by updatecli * chore: Update Packetbeat Dockerfile Made with ❤️️ by updatecli * chore: Update Filebeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update Heartbeat Dockerfile Made with ❤️️ by updatecli * chore: Update version.asciidoc Made with ❤️️ by updatecli * chore: Update Metricbeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update stan Dockerfile Made with ❤️️ by updatecli * chore: Update go.mod Made with ❤️️ by updatecli * chore: Update Metricbeat Dockerfile Made with ❤️️ by updatecli * chore: Update Heartbeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update HTTP module Dockerfile Made with ❤️️ by updatecli * chore: Update NATS module Dockerfile Made with ❤️️ by updatecli * chore: Update from vsphere Dockerfile Made with ❤️️ by updatecli * chore: Update Functionbeat Dockerfile Made with ❤️️ by updatecli * chore: Update .golangci.yml Made with ❤️️ by updatecli * chore: Update Auditbeat Dockerfile Made with ❤️️ by updatecli * Exclude Dockerfile from module detection Signed-off-by: Alexandros Sapranidis * Update go.mod Signed-off-by: Alexandros Sapranidis * chore: Update from vsphere Dockerfile Made with ❤️️ by updatecli * chore: Update Heartbeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update Metricbeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update Functionbeat Dockerfile Made with ❤️️ by updatecli * chore: Update go.mod Made with ❤️️ by updatecli * chore: Update Heartbeat Dockerfile Made with ❤️️ by updatecli * chore: Update Filebeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update NATS module Dockerfile Made with ❤️️ by updatecli * chore: Update stan Dockerfile Made with ❤️️ by updatecli * chore: Update HTTP module Dockerfile Made with ❤️️ by updatecli * chore: Update Metricbeat Dockerfile Made with ❤️️ by updatecli * chore: Update version.asciidoc Made with ❤️️ by updatecli * chore: Update .golangci.yml Made with ❤️️ by updatecli * chore: Update Packetbeat Dockerfile Made with ❤️️ by updatecli * chore: Update Auditbeat Dockerfile Made with ❤️️ by updatecli * chore: Update .go-version Made with ❤️️ by updatecli * go mod tidy --------- Signed-off-by: Alexandros Sapranidis Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Craig MacKenzie Co-authored-by: Alexandros Sapranidis --- .buildkite/scripts/changesets.sh | 2 +- .go-version | 2 +- .golangci.yml | 8 ++++---- CHANGELOG.next.asciidoc | 2 +- auditbeat/Dockerfile | 2 +- dev-tools/kubernetes/filebeat/Dockerfile.debug | 2 +- dev-tools/kubernetes/heartbeat/Dockerfile.debug | 2 +- dev-tools/kubernetes/metricbeat/Dockerfile.debug | 2 +- go.mod | 2 +- heartbeat/Dockerfile | 2 +- libbeat/docs/version.asciidoc | 2 +- metricbeat/Dockerfile | 2 +- metricbeat/module/http/_meta/Dockerfile | 2 +- metricbeat/module/nats/_meta/Dockerfile | 2 +- metricbeat/module/vsphere/_meta/Dockerfile | 2 +- packetbeat/Dockerfile | 2 +- x-pack/functionbeat/Dockerfile | 2 +- x-pack/metricbeat/module/stan/_meta/Dockerfile | 2 +- 18 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.buildkite/scripts/changesets.sh b/.buildkite/scripts/changesets.sh index 7e79d2d3164..1ab80edc585 100644 --- a/.buildkite/scripts/changesets.sh +++ b/.buildkite/scripts/changesets.sh @@ -15,7 +15,7 @@ definePattern() { } defineExclusions() { - exclude="^$beatPath\/module\/(.*(? Date: Thu, 13 Jun 2024 15:15:40 +0200 Subject: [PATCH 13/26] docs: Prepare Changelog for 8.14.1 (#39864) (#39882) * docs: Close changelog for 8.14.1 * Update CHANGELOG.asciidoc * Update CHANGELOG.next.asciidoc --------- Co-authored-by: Pierre HILBERT (cherry picked from commit f68af7f3d287df593abcca6b62ed5cff8a0fe01b) Co-authored-by: Elastic Machine Co-authored-by: Lisa Cawley --- CHANGELOG.asciidoc | 11 ++++++ CHANGELOG.next.asciidoc | 68 ----------------------------------- libbeat/docs/release.asciidoc | 1 + 3 files changed, 12 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 83844526f9d..6d12414d6e2 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -3,6 +3,17 @@ :issue: https://github.com/elastic/beats/issues/ :pull: https://github.com/elastic/beats/pull/ +[[release-notes-8.14.1]] +=== Beats version 8.14.1 +https://github.com/elastic/beats/compare/v8.14.0\...v8.14.1[View commits] + +==== Bugfixes + +*Heartbeat* + +- Fix import of browser plugin for Agentbeat. {pull}39818[39818] + + [[release-notes-8.14.0]] === Beats version 8.14.0 https://github.com/elastic/beats/compare/v8.13.4\...v8.14.0[View commits] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 33bbcc055bc..88656f51363 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -145,12 +145,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Heartbeat* -- Fix panics when parsing dereferencing invalid parsed url. {pull}34702[34702] -- Fix setuid root when running under cgroups v2. {pull}37794[37794] -- Adjust State loader to only retry when response code status is 5xx {pull}37981[37981] -- Reset prctl dumpable flag after cap drop. {pull}38269[38269] -- Redact synthexec cmd output. {pull}39535[39535] -- Fix import of browser plugin for agentbeat. {pull}39818[39818] *Heartbeat* @@ -362,65 +356,3 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] ==== Known Issues - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/libbeat/docs/release.asciidoc b/libbeat/docs/release.asciidoc index bad3f87b38d..3dc94738d12 100644 --- a/libbeat/docs/release.asciidoc +++ b/libbeat/docs/release.asciidoc @@ -8,6 +8,7 @@ This section summarizes the changes in each release. Also read <> for more detail about changes that affect upgrade. +* <> * <> * <> * <> From 2f91cf1b07d1e6bb72df4858cb324d5c44bfda21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Constan=C3=A7a=20Manteigas?= <113898685+constanca-m@users.noreply.github.com> Date: Thu, 13 Jun 2024 15:26:52 +0200 Subject: [PATCH 14/26] [Kubernetes Provider] Apply namespace filter to watchers (#39881) * Apply namespace filter Signed-off-by: constanca * Apply namespace filter Signed-off-by: constanca * Fix namespace docs description Signed-off-by: constanca --------- Signed-off-by: constanca --- CHANGELOG.next.asciidoc | 1 + .../autodiscover/providers/kubernetes/pod.go | 20 ++++++++++++------- .../providers/kubernetes/service.go | 5 +++-- libbeat/docs/shared-autodiscover.asciidoc | 7 ++----- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 88656f51363..15e4fd614db 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -151,6 +151,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Metricbeat* +- Fix `namespace` filter option at Kubernetes provider level. {pull}39881[39881] - Fix Azure Monitor 429 error by causing metricbeat to retry the request again. {pull}38294[38294] - Fix fields not being parsed correctly in postgresql/database {issue}25301[25301] {pull}37720[37720] - rabbitmq/queue - Change the mapping type of `rabbitmq.queue.consumers.utilisation.pct` to `scaled_float` from `long` because the values fall within the range of `[0.0, 1.0]`. Previously, conversion to integer resulted in reporting either `0` or `1`. diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go index b11faac4931..c5f9c721eb9 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod.go +++ b/libbeat/autodiscover/providers/kubernetes/pod.go @@ -100,9 +100,9 @@ func NewPodEventer(uuid uuid.UUID, cfg *conf.C, client k8s.Interface, publish fu if metaConf.Node.Enabled() || config.Hints.Enabled() { options := kubernetes.WatchOptions{ - SyncTimeout: config.SyncPeriod, - Node: config.Node, - Namespace: config.Namespace, + SyncTimeout: config.SyncPeriod, + Node: config.Node, + HonorReSyncs: true, } nodeWatcher, err = kubernetes.NewNamedWatcher("node", client, &kubernetes.Node{}, options, nil) if err != nil { @@ -112,20 +112,24 @@ func NewPodEventer(uuid uuid.UUID, cfg *conf.C, client k8s.Interface, publish fu if metaConf.Namespace.Enabled() || config.Hints.Enabled() { namespaceWatcher, err = kubernetes.NewNamedWatcher("namespace", client, &kubernetes.Namespace{}, kubernetes.WatchOptions{ - SyncTimeout: config.SyncPeriod, + SyncTimeout: config.SyncPeriod, + Namespace: config.Namespace, + HonorReSyncs: true, }, nil) if err != nil { logger.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Namespace{}, err) } } - // Resource is Pod so we need to create watchers for Replicasets and Jobs that it might belongs to + // Resource is Pod, so we need to create watchers for Replicasets and Jobs that it might belong to // in order to be able to retrieve 2nd layer Owner metadata like in case of: // Deployment -> Replicaset -> Pod // CronJob -> job -> Pod if metaConf.Deployment { replicaSetWatcher, err = kubernetes.NewNamedWatcher("resource_metadata_enricher_rs", client, &kubernetes.ReplicaSet{}, kubernetes.WatchOptions{ - SyncTimeout: config.SyncPeriod, + SyncTimeout: config.SyncPeriod, + Namespace: config.Namespace, + HonorReSyncs: true, }, nil) if err != nil { logger.Errorf("Error creating watcher for %T due to error %+v", &kubernetes.ReplicaSet{}, err) @@ -133,7 +137,9 @@ func NewPodEventer(uuid uuid.UUID, cfg *conf.C, client k8s.Interface, publish fu } if metaConf.CronJob { jobWatcher, err = kubernetes.NewNamedWatcher("resource_metadata_enricher_job", client, &kubernetes.Job{}, kubernetes.WatchOptions{ - SyncTimeout: config.SyncPeriod, + SyncTimeout: config.SyncPeriod, + Namespace: config.Namespace, + HonorReSyncs: true, }, nil) if err != nil { logger.Errorf("Error creating watcher for %T due to error %+v", &kubernetes.Job{}, err) diff --git a/libbeat/autodiscover/providers/kubernetes/service.go b/libbeat/autodiscover/providers/kubernetes/service.go index ba62dda9c47..e9e71c921bd 100644 --- a/libbeat/autodiscover/providers/kubernetes/service.go +++ b/libbeat/autodiscover/providers/kubernetes/service.go @@ -74,8 +74,9 @@ func NewServiceEventer(uuid uuid.UUID, cfg *conf.C, client k8s.Interface, publis if metaConf.Namespace.Enabled() || config.Hints.Enabled() { namespaceWatcher, err = kubernetes.NewNamedWatcher("namespace", client, &kubernetes.Namespace{}, kubernetes.WatchOptions{ - SyncTimeout: config.SyncPeriod, - Namespace: config.Namespace, + SyncTimeout: config.SyncPeriod, + Namespace: config.Namespace, + HonorReSyncs: true, }, nil) if err != nil { return nil, fmt.Errorf("couldn't create watcher for %T due to error %w", &kubernetes.Namespace{}, err) diff --git a/libbeat/docs/shared-autodiscover.asciidoc b/libbeat/docs/shared-autodiscover.asciidoc index 1e1ea567b7c..83d44b498ac 100644 --- a/libbeat/docs/shared-autodiscover.asciidoc +++ b/libbeat/docs/shared-autodiscover.asciidoc @@ -140,10 +140,7 @@ The `kubernetes` autodiscover provider has the following configuration settings: `node`:: (Optional) Specify the node to scope {beatname_lc} to in case it cannot be accurately detected, as when running {beatname_lc} in host network mode. -`namespace`:: (Optional) Select the namespace from which to collect the - metadata. If it is not set, the processor collects metadata from all - namespaces. It is unset by default. The namespace configuration only applies to - kubernetes resources that are namespace scoped. +`namespace`:: (Optional) Select the namespace from which to collect the events from the resources. If it is not set, the provider collects them from all namespaces. It is unset by default. The namespace configuration only applies to kubernetes resources that are namespace scoped and if `unique` field is set to `false`. `cleanup_timeout`:: (Optional) Specify the time of inactivity before stopping the running configuration for a container, ifeval::["{beatname_lc}"=="filebeat"] @@ -196,7 +193,7 @@ Example: `unique`:: (Optional) Defaults to `false`. Marking an autodiscover provider as unique results into making the provider to enable the provided templates only when it will gain the leader lease. - This setting can only be combined with `cluster` scope. When `unique` is enabled enabled, `resource` + This setting can only be combined with `cluster` scope. When `unique` is enabled, `resource` and `add_resource_metadata` settings are not taken into account. `leader_lease`:: (Optional) Defaults to +{beatname_lc}-cluster-leader+. This will be name of the lock lease. One can monitor the status of the lease with `kubectl describe lease beats-cluster-leader`. From f9fec1e4ff85085093682a1017640cc2e4b097b8 Mon Sep 17 00:00:00 2001 From: VihasMakwana <121151420+VihasMakwana@users.noreply.github.com> Date: Thu, 13 Jun 2024 18:58:16 +0530 Subject: [PATCH 15/26] filebeat: Fix flaky test case on macOS (#39860) * fix: fix a flaky test on macos * fix: fix more such test cases * fix: only update ignore_older * fix: also fix flaky test_restart_state * fix: fix CI --- filebeat/tests/system/test_crawler.py | 4 ++-- filebeat/tests/system/test_harvester.py | 2 +- filebeat/tests/system/test_registrar.py | 14 ++++++++------ filebeat/tests/system/test_shutdown.py | 8 +++++++- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/filebeat/tests/system/test_crawler.py b/filebeat/tests/system/test_crawler.py index 2bea57223fe..39f0d454124 100644 --- a/filebeat/tests/system/test_crawler.py +++ b/filebeat/tests/system/test_crawler.py @@ -485,7 +485,7 @@ def test_tail_files(self): self.wait_until( lambda: self.log_contains( "Start next scan"), - max_timeout=5) + max_timeout=10) with open(testfile, 'a') as f: # write additional lines @@ -596,7 +596,7 @@ def test_encodings(self): # run filebeat filebeat = self.start_beat() self.wait_until(lambda: self.output_has(lines=len(encodings)), - max_timeout=15) + max_timeout=25) # write another line in all files for _, enc_py, text in encodings: diff --git a/filebeat/tests/system/test_harvester.py b/filebeat/tests/system/test_harvester.py index 3b40f3a6730..9099abc699e 100644 --- a/filebeat/tests/system/test_harvester.py +++ b/filebeat/tests/system/test_harvester.py @@ -858,6 +858,6 @@ def test_debug_reader(self): # 13 on unix, 14 on windows. self.wait_until(lambda: self.log_contains(re.compile( - 'Matching null byte found at offset (13|14)')), max_timeout=5) + 'Matching null byte found at offset (13|14)')), max_timeout=10) filebeat.check_kill_and_wait() diff --git a/filebeat/tests/system/test_registrar.py b/filebeat/tests/system/test_registrar.py index 53af186dbf9..b08eda2777c 100644 --- a/filebeat/tests/system/test_registrar.py +++ b/filebeat/tests/system/test_registrar.py @@ -252,9 +252,11 @@ def test_registry_file_update_permissions(self): self.assertEqual(self.file_permissions(os.path.join(registry_path, "log.json")), "0o600") + registry_home = "a/b/c/d/registry_x" + registry_path = os.path.join(registry_home, "filebeat") self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", - registry_home="a/b/c/registry_x", + registry_home=registry_home, registry_file_permissions=0o640 ) @@ -266,7 +268,7 @@ def test_registry_file_update_permissions(self): # the logging and actual writing the file. Seems to happen on Windows. self.wait_until( lambda: self.has_registry(registry_path), - max_timeout=1) + max_timeout=10) # Wait a moment to make sure registry is completely written time.sleep(1) @@ -950,7 +952,7 @@ def test_restart_state(self): path=os.path.abspath(self.working_dir) + "/log/*", close_inactive="200ms", ignore_older="2000ms", - clean_inactive="3s", + clean_inactive="10s", ) filebeat = self.start_beat() @@ -976,7 +978,7 @@ def test_restart_state_reset(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", clean_inactive="10s", - ignore_older="5s" + ignore_older="9s" ) os.mkdir(self.working_dir + "/log/") @@ -1003,7 +1005,7 @@ def test_restart_state_reset(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/test2.log", clean_inactive="10s", - ignore_older="5s", + ignore_older="9s", ) filebeat = self.start_beat(output="filebeat2.log") @@ -1137,7 +1139,7 @@ def test_restart_state_reset_ttl_no_clean_inactive(self): self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/test.log", clean_inactive="10s", - ignore_older="5s" + ignore_older="9s" ) os.mkdir(self.working_dir + "/log/") diff --git a/filebeat/tests/system/test_shutdown.py b/filebeat/tests/system/test_shutdown.py index 8f18337435f..8e781837730 100644 --- a/filebeat/tests/system/test_shutdown.py +++ b/filebeat/tests/system/test_shutdown.py @@ -27,7 +27,13 @@ def test_shutdown(self): ) for i in range(1, 5): proc = self.start_beat(logging_args=["-e", "-v"]) - time.sleep(.5) + + # Flaky on MacOS, see https://github.com/elastic/beats/issues/39613#issuecomment-2158812325 + # we need to wait a bit longer for filebeat to start + if platform.system() == "Darwin": + time.sleep(10) + else: + time.sleep(.5) proc.check_kill_and_wait() @unittest.skip("Skipped as flaky: https://github.com/elastic/beats/issues/14647") From de63284e8e45152fd99798c07732f8f90eab2d75 Mon Sep 17 00:00:00 2001 From: niraj-elastic <124254029+niraj-elastic@users.noreply.github.com> Date: Thu, 13 Jun 2024 21:20:00 +0530 Subject: [PATCH 16/26] [oracle] Missing total.bytes field in tablespace datastream (#39787) * fix total.bytes for tablespace * update changelog * fix CI errors * fix CI * fix CI issues * address review comments * address review comment * update changelog * address review comment * update visualization * update visualization name --------- Co-authored-by: Kush Rana <89848966+kush-elastic@users.noreply.github.com> --- CHANGELOG.next.asciidoc | 1 + .../862e2c20-9bf0-11e9-a61b-f742ed613c57.json | 2 +- .../05acae50-9bf0-11e9-a61b-f742ed613c57.json | 37 ++++++++++++++++--- .../module/oracle/tablespace/data.go | 20 +++++----- .../module/oracle/tablespace/data_test.go | 12 +++--- .../module/oracle/tablespace/mocks_test.go | 10 ++--- .../oracle/tablespace/temp_free_space.go | 21 ++++------- .../oracle/tablespace/used_and_free_space.go | 20 ++++------ 8 files changed, 70 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 15e4fd614db..372b8e35244 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -162,6 +162,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix issue where beats may report incorrect metrics for its own process when running inside a container {pull}39627[39627] - Fix for MySQL/Performance - Query failure for MySQL versions below v8.0.1, for performance metric `quantile_95`. {pull}38710[38710] - Normalize AWS RDS CPU Utilization values before making the metadata API call. {pull}39664[39664] +- Fix query logic for temp and non-temp tablespaces in Oracle module. {issue}38051[38051] {pull}39787[39787] *Osquerybeat* diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/862e2c20-9bf0-11e9-a61b-f742ed613c57.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/862e2c20-9bf0-11e9-a61b-f742ed613c57.json index f093d13a3e4..216971d16b5 100644 --- a/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/862e2c20-9bf0-11e9-a61b-f742ed613c57.json +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/862e2c20-9bf0-11e9-a61b-f742ed613c57.json @@ -46,7 +46,7 @@ }, "panelIndex": "2", "panelRefName": "panel_2", - "title": "Tablespace Total Size", + "title": "Tablespace Max Total Size", "type": "visualization", "version": "8.0.0-SNAPSHOT" }, diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json index 603fcc02ec2..27de5af579a 100644 --- a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json @@ -4,7 +4,7 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": {} }, - "title": "Tablespace Total Size [Metricbeat Oracle]", + "title": "Tablespace Max Total Size [Metricbeat Oracle]", "uiStateJSON": {}, "version": 1, "visState": { @@ -24,13 +24,40 @@ "fill": 0.5, "formatter": "bytes", "id": "61ca57f1-469d-11e7-af02-69e470af7417", - "label": "Tablespace total size", + "label": "Tablespace max total size", "line_width": 1, "metrics": [ { - "field": "oracle.tablespace.space.total.bytes", + "agg_with": "avg", + "field": "oracle.tablespace.space.used.bytes", "id": "61ca57f2-469d-11e7-af02-69e470af7417", - "type": "avg" + "type": "max" + }, + { + "agg_with": "avg", + "colors": [ + "#68BC00" + ], + "field": "oracle.tablespace.space.free.bytes", + "id": "e04e8f40-24cd-4066-b12c-da0db0ff73d4", + "type": "max" + }, + { + "id": "2cf57800-8b54-41fa-a877-159b49699a50", + "script": "params.used_bytes + params.free_bytes", + "type": "math", + "variables": [ + { + "field": "61ca57f2-469d-11e7-af02-69e470af7417", + "id": "631a44d5-d18a-4743-bea0-6f61930fd65f", + "name": "used_bytes" + }, + { + "field": "e04e8f40-24cd-4066-b12c-da0db0ff73d4", + "id": "c255d24c-3a29-4879-b999-77af43d97c6b", + "name": "free_bytes" + } + ] } ], "point_size": 1, @@ -48,7 +75,7 @@ "type": "timeseries", "use_kibana_indexes": false }, - "title": "Tablespace Total Size [Metricbeat Oracle]", + "title": "Tablespace Max Total Size [Metricbeat Oracle]", "type": "metrics" } }, diff --git a/x-pack/metricbeat/module/oracle/tablespace/data.go b/x-pack/metricbeat/module/oracle/tablespace/data.go index 30c86d766da..82f93a060b1 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/data.go +++ b/x-pack/metricbeat/module/oracle/tablespace/data.go @@ -16,8 +16,9 @@ import ( // extract is the E of a ETL processing. Gets the data files, used/free space and temp free space data that is fetch // by doing queries to Oracle -func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMethods) (out *extractedData, err error) { - out = &extractedData{} +func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMethods) (*extractedData, error) { + out := &extractedData{} + var err error if out.dataFiles, err = extractor.dataFilesData(ctx); err != nil { return nil, fmt.Errorf("error getting data_files: %w", err) @@ -31,23 +32,23 @@ func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMeth return nil, fmt.Errorf("error getting free space data: %w", err) } - return + return out, nil } // transform is the T of an ETL (refer to the 'extract' method above if you need to see the origin). Transforms the data // to create a Kibana/Elasticsearch friendly JSON. Data from Oracle is pretty fragmented by design so a lot of data // was necessary. Data is organized by Tablespace entity (Tablespaces might contain one or more data files) -func (m *MetricSet) transform(in *extractedData) (out map[string]mapstr.M) { - out = make(map[string]mapstr.M, 0) +func (m *MetricSet) transform(in *extractedData) map[string]mapstr.M { + out := make(map[string]mapstr.M, 0) - for _, dataFile := range in.dataFiles { - m.addDataFileData(&dataFile, out) + for i := range in.dataFiles { + m.addDataFileData(&in.dataFiles[i], out) } m.addUsedAndFreeSpaceData(in.freeSpace, out) m.addTempFreeSpaceData(in.tempFreeSpace, out) - return + return out } func (m *MetricSet) extractAndTransform(ctx context.Context) ([]mb.Event, error) { @@ -78,7 +79,7 @@ func (m *MetricSet) addTempFreeSpaceData(tempFreeSpaces []tempFreeSpace, out map name := val.(string) if name == "TEMP" { for _, tempFreeSpaceTable := range tempFreeSpaces { - oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.total.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.TablespaceSize}) + oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.total.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.TotalSpaceBytes}) oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.used.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.UsedSpaceBytes}) oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.free.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.FreeSpace}) } @@ -101,6 +102,7 @@ func (m *MetricSet) addUsedAndFreeSpaceData(freeSpaces []usedAndFreeSpace, out m if name == freeSpaceTable.TablespaceName { oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.free.bytes", &oracle.Int64Value{NullInt64: freeSpaceTable.TotalFreeBytes}) oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.used.bytes", &oracle.Int64Value{NullInt64: freeSpaceTable.TotalUsedBytes}) + oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.total.bytes", &oracle.Int64Value{NullInt64: freeSpaceTable.TotalSpaceBytes}) } } } diff --git a/x-pack/metricbeat/module/oracle/tablespace/data_test.go b/x-pack/metricbeat/module/oracle/tablespace/data_test.go index 02e4bdd4528..9c6990a2598 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/data_test.go +++ b/x-pack/metricbeat/module/oracle/tablespace/data_test.go @@ -12,13 +12,13 @@ import ( "github.com/stretchr/testify/assert" ) -var expectedResults = []string{`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux01.dbf","online_status":"ONLINE","size":{"bytes":9999990,"free":{"bytes":99999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`, - `{"data_file":{"id":181,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux02.dbf","online_status":"ONLINE","size":{"bytes":9999991,"free":{"bytes":99999995},"max":{"bytes":9999995}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`, - `{"data_file":{"id":182,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux03.dbf","online_status":"ONLINE","size":{"bytes":9999992,"free":{"bytes":99999996},"max":{"bytes":9999996}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`, - `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/system01.dbf","online_status":"ONLINE","size":{"bytes":999990,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSTEM","space":{"free":{"bytes":9990},"used":{"bytes":9991}}}`, +var expectedResults = []string{`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux01.dbf","online_status":"ONLINE","size":{"bytes":9999990,"free":{"bytes":99999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`, + `{"data_file":{"id":181,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux02.dbf","online_status":"ONLINE","size":{"bytes":9999991,"free":{"bytes":99999995},"max":{"bytes":9999995}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`, + `{"data_file":{"id":182,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux03.dbf","online_status":"ONLINE","size":{"bytes":9999992,"free":{"bytes":99999996},"max":{"bytes":9999996}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`, + `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/system01.dbf","online_status":"ONLINE","size":{"bytes":999990,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSTEM","space":{"free":{"bytes":9990},"total":{"bytes":99999},"used":{"bytes":9991}}}`, `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/temp012017-03-02_07-54-38-075-AM.dbf","online_status":"ONLINE","size":{"bytes":999991,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"TEMP","space":{"free":{"bytes":99999},"total":{"bytes":99999},"used":{"bytes":99999}}}`, - `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/undotbs01.dbf","online_status":"ONLINE","size":{"bytes":999992,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"UNDOTBS1","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`, - `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/users01.dbf","online_status":"ONLINE","size":{"bytes":999993,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"USERS","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`} + `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/undotbs01.dbf","online_status":"ONLINE","size":{"bytes":999992,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"UNDOTBS1","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`, + `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/users01.dbf","online_status":"ONLINE","size":{"bytes":999993,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"USERS","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`} var notExpectedEvents = []string{`{}`, `{"foo":"bar"}`} diff --git a/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go b/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go index 12348236bcc..9f5800166af 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go +++ b/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go @@ -77,16 +77,16 @@ func (h happyDataFiles) dataFilesData(_ context.Context) ([]dataFile, error) { type happyTempFreeSpaceData struct{} func (happyTempFreeSpaceData) tempFreeSpaceData(_ context.Context) ([]tempFreeSpace, error) { - return []tempFreeSpace{{TablespaceName: "TEMP", TablespaceSize: sql.NullInt64{Valid: true, Int64: 99999}, UsedSpaceBytes: sql.NullInt64{Valid: true, Int64: 99999}, FreeSpace: sql.NullInt64{Int64: 99999, Valid: true}}}, nil + return []tempFreeSpace{{TablespaceName: "TEMP", TotalSpaceBytes: sql.NullInt64{Valid: true, Int64: 99999}, UsedSpaceBytes: sql.NullInt64{Valid: true, Int64: 99999}, FreeSpace: sql.NullInt64{Int64: 99999, Valid: true}}}, nil } type happyFreeSpaceData struct{} func (happyFreeSpaceData) usedAndFreeSpaceData(_ context.Context) ([]usedAndFreeSpace, error) { return []usedAndFreeSpace{ - {TablespaceName: "SYSTEM", TotalFreeBytes: sql.NullInt64{Int64: 9990, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}}, - {TablespaceName: "SYSAUX", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}}, - {TablespaceName: "UNDOTBS1", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}}, - {TablespaceName: "USERS", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}}, + {TablespaceName: "SYSTEM", TotalFreeBytes: sql.NullInt64{Int64: 9990, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}}, + {TablespaceName: "SYSAUX", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}}, + {TablespaceName: "UNDOTBS1", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}}, + {TablespaceName: "USERS", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}}, }, nil } diff --git a/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go b/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go index efc8401e2c0..f4b592d2b75 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go +++ b/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go @@ -11,31 +11,24 @@ import ( ) type tempFreeSpace struct { - TablespaceName string - TablespaceSize sql.NullInt64 - UsedSpaceBytes sql.NullInt64 - FreeSpace sql.NullInt64 -} - -func (d *tempFreeSpace) hash() string { - return d.TablespaceName -} - -func (d *tempFreeSpace) eventKey() string { - return d.TablespaceName + TablespaceName string + TotalSpaceBytes sql.NullInt64 + UsedSpaceBytes sql.NullInt64 + FreeSpace sql.NullInt64 } func (e *tablespaceExtractor) tempFreeSpaceData(ctx context.Context) ([]tempFreeSpace, error) { - rows, err := e.db.QueryContext(ctx, "SELECT TABLESPACE_NAME, TABLESPACE_SIZE, ALLOCATED_SPACE, FREE_SPACE FROM DBA_TEMP_FREE_SPACE") + rows, err := e.db.QueryContext(ctx, `SELECT t.TABLESPACE_NAME, (SELECT SUM(BYTES) FROM DBA_DATA_FILES) + (SELECT SUM(BYTES) FROM DBA_TEMP_FILES) AS TOTAL_SUM, t.ALLOCATED_SPACE, t.FREE_SPACE FROM DBA_TEMP_FREE_SPACE t `) if err != nil { return nil, fmt.Errorf("error executing query: %w", err) } + defer rows.Close() results := make([]tempFreeSpace, 0) for rows.Next() { dest := tempFreeSpace{} - if err = rows.Scan(&dest.TablespaceName, &dest.TablespaceSize, &dest.UsedSpaceBytes, &dest.FreeSpace); err != nil { + if err = rows.Scan(&dest.TablespaceName, &dest.TotalSpaceBytes, &dest.UsedSpaceBytes, &dest.FreeSpace); err != nil { return nil, err } results = append(results, dest) diff --git a/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go b/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go index b17b249808d..e327badcf02 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go +++ b/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go @@ -11,30 +11,24 @@ import ( ) type usedAndFreeSpace struct { - TablespaceName string - TotalFreeBytes sql.NullInt64 - TotalUsedBytes sql.NullInt64 -} - -func (d *usedAndFreeSpace) hash() string { - return d.TablespaceName -} - -func (d *usedAndFreeSpace) eventKey() string { - return d.TablespaceName + TablespaceName string + TotalSpaceBytes sql.NullInt64 + TotalFreeBytes sql.NullInt64 + TotalUsedBytes sql.NullInt64 } func (e *tablespaceExtractor) usedAndFreeSpaceData(ctx context.Context) ([]usedAndFreeSpace, error) { - rows, err := e.db.QueryContext(ctx, "SELECT b.tablespace_name, tbs_size used, a.free_space free FROM (SELECT tablespace_name, sum(bytes) AS free_space FROM dba_free_space GROUP BY tablespace_name) a, (SELECT tablespace_name, sum(bytes) AS tbs_size FROM dba_data_files GROUP BY tablespace_name) b WHERE a.tablespace_name(+)=b.tablespace_name") + rows, err := e.db.QueryContext(ctx, `SELECT b.tablespace_name, (b.tbs_size - NVL(a.free_space, 0)) AS used, NVL(a.free_space, 0) AS free, (SELECT SUM(bytes) FROM DBA_DATA_FILES) + (SELECT SUM(bytes) FROM DBA_TEMP_FILES) AS total_sum FROM (SELECT tablespace_name, SUM(bytes) AS free_space FROM DBA_FREE_SPACE GROUP BY tablespace_name) a RIGHT JOIN (SELECT tablespace_name, SUM(bytes) AS tbs_size FROM DBA_DATA_FILES GROUP BY tablespace_name) b ON a.tablespace_name = b.tablespace_name`) if err != nil { return nil, fmt.Errorf("error executing query: %w", err) } + defer rows.Close() results := make([]usedAndFreeSpace, 0) for rows.Next() { dest := usedAndFreeSpace{} - if err = rows.Scan(&dest.TablespaceName, &dest.TotalUsedBytes, &dest.TotalFreeBytes); err != nil { + if err = rows.Scan(&dest.TablespaceName, &dest.TotalUsedBytes, &dest.TotalFreeBytes, &dest.TotalSpaceBytes); err != nil { return nil, err } results = append(results, dest) From 974ace65bda4c9f68e5b002fad0b14b942a8bed2 Mon Sep 17 00:00:00 2001 From: Alex K <8418476+fearful-symmetry@users.noreply.github.com> Date: Thu, 13 Jun 2024 08:52:44 -0700 Subject: [PATCH 17/26] Add section on cgroupns usage (#39889) --- metricbeat/docs/running-on-docker.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metricbeat/docs/running-on-docker.asciidoc b/metricbeat/docs/running-on-docker.asciidoc index 72cfed9757d..26ba27658b8 100644 --- a/metricbeat/docs/running-on-docker.asciidoc +++ b/metricbeat/docs/running-on-docker.asciidoc @@ -21,6 +21,7 @@ docker run \ --mount type=bind,source=/var/run/dbus/system_bus_socket,target=/hostfs/var/run/dbus/system_bus_socket,readonly \ <4> --env DBUS_SYSTEM_BUS_ADDRESS='unix:path=/hostfs/var/run/dbus/system_bus_socket' \ <4> --net=host \ <5> + --cgroupns=host \ <6> {dockerimage} -e -system.hostfs=/hostfs ---- @@ -45,6 +46,7 @@ both require access to dbus. Mount the dbus socket and set the `DBUS_SYSTEM_BUS_ to make this file contain the host's network devices is to use the `--net=host` flag. This is due to Linux namespacing; simply bind mounting the host's `/proc` to `/hostfs/proc` is not sufficient. +<6> Runs the container using the host's cgroup namespace, instead of a private namespace. While this is optional, <> may produce more correct cgroup metrics when running in host mode. NOTE: The special filesystems +/proc+ and +/sys+ are only available if the host system is running Linux. Attempts to bind-mount these filesystems will From 9d2b48da6dbb16b3ed65717669acb06fda7da245 Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Thu, 13 Jun 2024 20:33:14 +0200 Subject: [PATCH 18/26] refactor: replace uber atomic with stdlib atomic types (#39853) * refactor: replace uber atomic with stdlib atomic types Go 1.19 added new atomic types. Migrate usage of go.uber.org/atomic to sync/atomic and drop the dependency. * refactor: store concrete type instead of pointer * Update input.go * lint: fix linting errors --- NOTICE.txt | 58 +++++++++---------- go.mod | 2 +- .../filebeat/input/http_endpoint/handler.go | 8 +-- x-pack/filebeat/input/http_endpoint/input.go | 8 +-- .../input/internal/httplog/roundtripper.go | 23 ++++---- .../manager/aws/event_stack_poller_test.go | 4 +- 6 files changed, 50 insertions(+), 53 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index f6881a796c0..c476eea8226 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -24852,35 +24852,6 @@ Contents of probable licence file $GOMODCACHE/go.mongodb.org/mongo-driver@v1.5.1 limitations under the License. --------------------------------------------------------------------------------- -Dependency : go.uber.org/atomic -Version: v1.11.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/go.uber.org/atomic@v1.11.0/LICENSE.txt: - -Copyright (c) 2016 Uber Technologies, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - -------------------------------------------------------------------------------- Dependency : go.uber.org/multierr Version: v1.11.0 @@ -55030,6 +55001,35 @@ Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/otel/trace@v1. limitations under the License. +-------------------------------------------------------------------------------- +Dependency : go.uber.org/atomic +Version: v1.11.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/go.uber.org/atomic@v1.11.0/LICENSE.txt: + +Copyright (c) 2016 Uber Technologies, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : go.uber.org/goleak Version: v1.3.0 diff --git a/go.mod b/go.mod index 57a93552539..3647392e0c2 100644 --- a/go.mod +++ b/go.mod @@ -150,7 +150,7 @@ require ( go.elastic.co/ecszap v1.0.2 go.elastic.co/go-licence-detector v0.6.0 go.etcd.io/bbolt v1.3.6 - go.uber.org/atomic v1.11.0 + go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.22.0 diff --git a/x-pack/filebeat/input/http_endpoint/handler.go b/x-pack/filebeat/input/http_endpoint/handler.go index b799248a935..4f3fdd550aa 100644 --- a/x-pack/filebeat/input/http_endpoint/handler.go +++ b/x-pack/filebeat/input/http_endpoint/handler.go @@ -18,13 +18,13 @@ import ( "sort" "strconv" "strings" + "sync/atomic" "time" "github.com/google/cel-go/cel" "github.com/google/cel-go/checker/decls" "github.com/google/cel-go/common/types" "github.com/google/cel-go/common/types/ref" - "go.uber.org/atomic" "go.uber.org/zap" "go.uber.org/zap/zapcore" "google.golang.org/protobuf/types/known/structpb" @@ -53,8 +53,8 @@ type handler struct { publish func(beat.Event) log *logp.Logger validator apiValidator - txBaseID string // Random value to make transaction IDs unique. - txIDCounter *atomic.Uint64 // Transaction ID counter that is incremented for each request. + txBaseID string // Random value to make transaction IDs unique. + txIDCounter atomic.Uint64 // Transaction ID counter that is incremented for each request. reqLogger *zap.Logger host, scheme string @@ -290,7 +290,7 @@ func (h *handler) logRequest(txID string, r *http.Request, status int, respBody } func (h *handler) nextTxID() string { - count := h.txIDCounter.Inc() + count := h.txIDCounter.Add(1) return h.formatTxID(count) } diff --git a/x-pack/filebeat/input/http_endpoint/input.go b/x-pack/filebeat/input/http_endpoint/input.go index 6737a9b9aa0..7f0440deb60 100644 --- a/x-pack/filebeat/input/http_endpoint/input.go +++ b/x-pack/filebeat/input/http_endpoint/input.go @@ -24,7 +24,6 @@ import ( "github.com/rcrowley/go-metrics" "go.elastic.co/ecszap" - "go.uber.org/atomic" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -327,10 +326,9 @@ func (s *server) getErr() error { func newHandler(ctx context.Context, c config, prg *program, pub func(beat.Event), log *logp.Logger, metrics *inputMetrics) http.Handler { h := &handler{ - ctx: ctx, - log: log, - txBaseID: newID(), - txIDCounter: atomic.NewUint64(0), + ctx: ctx, + log: log, + txBaseID: newID(), publish: pub, metrics: metrics, diff --git a/x-pack/filebeat/input/internal/httplog/roundtripper.go b/x-pack/filebeat/input/internal/httplog/roundtripper.go index ce68147a2a7..9e60cb60942 100644 --- a/x-pack/filebeat/input/internal/httplog/roundtripper.go +++ b/x-pack/filebeat/input/internal/httplog/roundtripper.go @@ -15,9 +15,9 @@ import ( "net/http" "net/http/httputil" "strconv" + "sync/atomic" "time" - "go.uber.org/atomic" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -36,22 +36,21 @@ type contextKey string // responses to the provided logger. Transaction creation is logged to log. func NewLoggingRoundTripper(next http.RoundTripper, logger *zap.Logger, maxBodyLen int, log *logp.Logger) *LoggingRoundTripper { return &LoggingRoundTripper{ - transport: next, - maxBodyLen: maxBodyLen, - txLog: logger, - txBaseID: newID(), - txIDCounter: atomic.NewUint64(0), - log: log, + transport: next, + maxBodyLen: maxBodyLen, + txLog: logger, + txBaseID: newID(), + log: log, } } // LoggingRoundTripper is an http.RoundTripper that logs requests and responses. type LoggingRoundTripper struct { transport http.RoundTripper - maxBodyLen int // The maximum length of a body. Longer bodies will be truncated. - txLog *zap.Logger // Destination logger. - txBaseID string // Random value to make transaction IDs unique. - txIDCounter *atomic.Uint64 // Transaction ID counter that is incremented for each request. + maxBodyLen int // The maximum length of a body. Longer bodies will be truncated. + txLog *zap.Logger // Destination logger. + txBaseID string // Random value to make transaction IDs unique. + txIDCounter atomic.Uint64 // Transaction ID counter that is incremented for each request. log *logp.Logger } @@ -220,7 +219,7 @@ func (rt *LoggingRoundTripper) TxID() string { // nextTxID returns the next transaction.id value. It increments the internal // request counter. func (rt *LoggingRoundTripper) nextTxID() string { - count := rt.txIDCounter.Inc() + count := rt.txIDCounter.Add(1) return rt.formatTxID(count) } diff --git a/x-pack/functionbeat/manager/aws/event_stack_poller_test.go b/x-pack/functionbeat/manager/aws/event_stack_poller_test.go index fe7473d0f3d..3679f07fb54 100644 --- a/x-pack/functionbeat/manager/aws/event_stack_poller_test.go +++ b/x-pack/functionbeat/manager/aws/event_stack_poller_test.go @@ -7,6 +7,7 @@ package aws import ( "context" "strconv" + "sync/atomic" "testing" "time" @@ -14,7 +15,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/cloudformation" "github.com/stretchr/testify/assert" - "go.uber.org/atomic" "github.com/elastic/elastic-agent-libs/logp" ) @@ -29,7 +29,7 @@ func (m *mockEventHandler) sync(event types.StackEvent) bool { if m.skipCount.Load() >= m.skipEvents { return false } - m.skipCount.Inc() + m.skipCount.Add(1) return true } From cabac6b5838e3e88b8818e454745fdbab000f4b0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Jun 2024 19:08:52 +0000 Subject: [PATCH 19/26] build(deps): bump github.com/elastic/go-licenser from 0.4.1 to 0.4.2 (#39893) * build(deps): bump github.com/elastic/go-licenser from 0.4.1 to 0.4.2 Bumps [github.com/elastic/go-licenser](https://github.com/elastic/go-licenser) from 0.4.1 to 0.4.2. - [Release notes](https://github.com/elastic/go-licenser/releases) - [Changelog](https://github.com/elastic/go-licenser/blob/main/.goreleaser.yml) - [Commits](https://github.com/elastic/go-licenser/compare/v0.4.1...v0.4.2) --- updated-dependencies: - dependency-name: github.com/elastic/go-licenser dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update NOTICE.txt --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: Craig MacKenzie --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index c476eea8226..7e5ec713681 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -14025,11 +14025,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-libaudit/v2@ -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-licenser -Version: v0.4.1 +Version: v0.4.2 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-licenser@v0.4.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-licenser@v0.4.2/LICENSE: Apache License diff --git a/go.mod b/go.mod index 3647392e0c2..5204045ddd9 100644 --- a/go.mod +++ b/go.mod @@ -74,7 +74,7 @@ require ( github.com/elastic/elastic-agent-client/v7 v7.11.0 github.com/elastic/go-concert v0.2.0 github.com/elastic/go-libaudit/v2 v2.5.0 - github.com/elastic/go-licenser v0.4.1 + github.com/elastic/go-licenser v0.4.2 github.com/elastic/go-lookslike v1.0.1 github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595 diff --git a/go.sum b/go.sum index 3f0ce0bf1fc..845774c1f89 100644 --- a/go.sum +++ b/go.sum @@ -568,8 +568,9 @@ github.com/elastic/go-elasticsearch/v8 v8.14.0 h1:1ywU8WFReLLcxE1WJqii3hTtbPUE2h github.com/elastic/go-elasticsearch/v8 v8.14.0/go.mod h1:WRvnlGkSuZyp83M2U8El/LGXpCjYLrvlkSgkAH4O5I4= github.com/elastic/go-libaudit/v2 v2.5.0 h1:5OK919QRnGtcjVBz3n/cs5F42im1mPlVTA9TyIn2K54= github.com/elastic/go-libaudit/v2 v2.5.0/go.mod h1:AjlnhinP+kKQuUJoXLVrqxBM8uyhQmkzoV6jjsCFP4Q= -github.com/elastic/go-licenser v0.4.1 h1:1xDURsc8pL5zYT9R29425J3vkHdt4RT5TNEMeRN48x4= github.com/elastic/go-licenser v0.4.1/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU= +github.com/elastic/go-licenser v0.4.2 h1:bPbGm8bUd8rxzSswFOqvQh1dAkKGkgAmrPxbUi+Y9+A= +github.com/elastic/go-licenser v0.4.2/go.mod h1:W8eH6FaZDR8fQGm+7FnVa7MxI1b/6dAqxz+zPB8nm5c= github.com/elastic/go-lookslike v1.0.1 h1:qVieyn6i/kx4xntar1cEB0qrGHVGNCX5KC8czAaTW/0= github.com/elastic/go-lookslike v1.0.1/go.mod h1:iSXdN6mmM0UVwbup8BY39Tyb51Dd+nX3eBsi5EgBAEo= github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f h1:TsPpU5EAwlt7YZoupKlxZ093qTZYdGou3EhfTF1U0B4= From 823943486f134759b21747e9288d994df9d3d97f Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 13 Jun 2024 16:26:19 -0600 Subject: [PATCH 20/26] Add a note when using `expand_event_list_from_field`, `content_type` is required to set to `application/json` (#39777) * Add a note when using `expand_event_list_from_field` * Update x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc Co-authored-by: Tetiana Kravchenko * Update x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc Co-authored-by: Andrew Gizas --------- Co-authored-by: Tetiana Kravchenko Co-authored-by: Andrew Gizas --- x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc index 794a51de081..c55c80952a4 100644 --- a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc @@ -120,7 +120,7 @@ characters. This only applies to non-JSON logs. See <<_encoding_3>>. ==== `decoding` The file decoding option is used to specify a codec that will be used to -decode the file contents. This can apply to any file stream data. +decode the file contents. This can apply to any file stream data. An example config is shown below: [source,yaml] @@ -131,17 +131,17 @@ An example config is shown below: Currently supported codecs are given below:- 1. <>: This codec decodes parquet compressed data streams. - + [id="attrib-decoding-parquet"] [float] ==== `the parquet codec` The `parquet` codec is used to decode parquet compressed data streams. Only enabling the codec will use the default codec options. The parquet codec supports -two sub attributes which can make parquet decoding more efficient. The `batch_size` attribute and +two sub attributes which can make parquet decoding more efficient. The `batch_size` attribute and the `process_parallel` attribute. The `batch_size` attribute can be used to specify the number of -records to read from the parquet stream at a time. By default the `batch size` is set to `1` and -`process_parallel` is set to `false`. If the `process_parallel` attribute is set to `true` then functions -which read multiple columns will read those columns in parallel from the parquet stream with a +records to read from the parquet stream at a time. By default the `batch size` is set to `1` and +`process_parallel` is set to `false`. If the `process_parallel` attribute is set to `true` then functions +which read multiple columns will read those columns in parallel from the parquet stream with a number of readers equal to the number of columns. Setting `process_parallel` to `true` will greatly increase the rate of processing at the cost of increased memory usage. Having a larger `batch_size` also helps to increase the rate of processing. An example config is shown below: @@ -162,6 +162,8 @@ value can be assigned the name of the field or `.[]`. This setting will be able the messages under the group value into separate events. For example, CloudTrail logs are in JSON format and events are found under the JSON object "Records". +NOTE: When using `expand_event_list_from_field`, `content_type` config parameter has to be set to `application/json`. + ["source","json"] ---- { From e59deb9c69bab9d6163b616d22d59b611c761534 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 14 Jun 2024 17:23:47 +0000 Subject: [PATCH 21/26] [updatecli] update elastic stack version for testing 8.15.0-d4efed09 (#39898) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index a5fb79fb02c..05ba6d0a25c 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.15.0-d2070ed2-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.15.0-d4efed09-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.15.0-d2070ed2-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.15.0-d4efed09-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.15.0-d2070ed2-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.15.0-d4efed09-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From e71bf8220525e6f004f9db5165ff015ad241abaa Mon Sep 17 00:00:00 2001 From: ev1yehor <146825775+ev1yehor@users.noreply.github.com> Date: Mon, 17 Jun 2024 13:03:13 +0300 Subject: [PATCH 22/26] Update pipeline.xpack.osquerybeat.yml (#39888) --- .../x-pack/pipeline.xpack.osquerybeat.yml | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml index cf8dab2de8b..5d3bc1285ff 100644 --- a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml @@ -23,6 +23,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "x-pack/osquerybeat Check/Update" + key: "x-pack-osquerybeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C x-pack/osquerybeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "x-pack/osquerybeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "x-pack-osquerybeat-check-update" + - group: "x-pack/osquerybeat Mandatory Tests" key: "x-pack-osquerybeat-mandatory-tests" From 3b1d7c30362c6a144304f3af903bf11ed1db1025 Mon Sep 17 00:00:00 2001 From: ev1yehor <146825775+ev1yehor@users.noreply.github.com> Date: Mon, 17 Jun 2024 13:06:14 +0300 Subject: [PATCH 23/26] Update pipeline.xpack.dockerlogbeat.yml (#39890) --- .../x-pack/pipeline.xpack.dockerlogbeat.yml | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml index 415c3947874..bdfe4c9dde6 100644 --- a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml @@ -18,6 +18,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "x-pack/dockerlogbeat Check/Update" + key: "x-pack-dockerlogbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C x-pack/dockerlogbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "x-pack/dockerlogbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "x-pack-dockerlogbeat-check-update" + - group: "x-pack/dockerlogbeat Mandatory Tests" key: "xpack-dockerlogbeat-mandatory-tests" steps: From a365feb2756d172c7d856b4d4aaf86f7c902261c Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 17 Jun 2024 13:07:16 +0200 Subject: [PATCH 24/26] [CI] Agentbeat pipeline resource for Pull requests (#39794) * [CI] Agentbeat pipeline resource * Update catalog-info.yaml --------- Co-authored-by: Geoff Rowland <70981735+rowlandgeoff@users.noreply.github.com> --- catalog-info.yaml | 48 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/catalog-info.yaml b/catalog-info.yaml index 4c16ed63739..97e7871f768 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -982,6 +982,54 @@ spec: everyone: access_level: BUILD_AND_READ +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-beats-xpack-agentbeat + description: "Beats x-pack agentbeat pipeline" + links: + - title: Pipeline + url: https://buildkite.com/elastic/beats-xpack-agentbeat + +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: beats-xpack-agentbeat + description: "Beats x-pack agentbeat pipeline" + spec: + # branch_configuration: "main 8.*" #TODO: uncomment after tests + pipeline_file: ".buildkite/x-pack/pipeline.xpack.agentbeat.yml" + maximum_timeout_in_minutes: 120 + provider_settings: + trigger_mode: none # don't trigger jobs from github activity + build_pull_request_forks: false + build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + build_tags: true + filter_enabled: true + filter_condition: >- + build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) + repository: elastic/beats + cancel_intermediate_builds: true + cancel_intermediate_builds_branch_filter: "!main !8.*" + skip_intermediate_builds: true + skip_intermediate_builds_branch_filter: "!main !8.*" + env: + # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved + ELASTIC_PR_COMMENTS_ENABLED: "false" + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: BUILD_AND_READ + + --- # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json apiVersion: backstage.io/v1alpha1 From 7948ba457d23df6ffbaaa4a5de63e4a8f55a7090 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 17 Jun 2024 15:30:24 +0200 Subject: [PATCH 25/26] [CI] Removed obs-ci-cache bucket usage (#39922) --- x-pack/packetbeat/magefile.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/x-pack/packetbeat/magefile.go b/x-pack/packetbeat/magefile.go index 357e5e23585..7e2c70d2f84 100644 --- a/x-pack/packetbeat/magefile.go +++ b/x-pack/packetbeat/magefile.go @@ -173,10 +173,7 @@ func SystemTest(ctx context.Context) error { } func getBucketName() string { - if os.Getenv("BUILDKITE") == "true" { - return "ingest-buildkite-ci" - } - return "obs-ci-cache" + return "ingest-buildkite-ci" } // getNpcapInstaller gets the installer from the Google Cloud Storage service. From b398af8ccc99c8e9ceeb1227afac26cc9cc0fed1 Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Mon, 17 Jun 2024 17:42:05 +0300 Subject: [PATCH 26/26] removed asdf setup (#39907) Updated metricbeat pipeline to match common styling && steps from Jenkins --- .buildkite/metricbeat/pipeline.yml | 100 ++++++++++++++--------------- 1 file changed, 49 insertions(+), 51 deletions(-) diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index ed9fb14f3d4..a8092887386 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -2,7 +2,8 @@ name: "beats-metricbeat" env: - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" @@ -10,7 +11,6 @@ env: IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" @@ -32,10 +32,13 @@ env: steps: - group: "Metricbeat Mandatory Tests" key: "metricbeat-mandatory-tests" + steps: - - label: ":linux: Ubuntu Unit Tests" + - label: ":ubuntu: Metricbeat Unit Tests" key: "mandatory-linux-unit-test" - command: "cd metricbeat && mage build unitTest" + command: | + cd metricbeat + mage build unitTest retry: automatic: - limit: 3 @@ -48,27 +51,20 @@ steps: - "metricbeat/build/*.json" notify: - github_commit_status: - context: "metricbeat: Ubuntu Unit Tests" + context: "metricbeat: Linux x86_64 Unit Tests" - - label: ":go: Go Integration Tests" + - label: ":ubuntu: Metricbeat Go Integration Tests" key: "mandatory-int-test" command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet metricbeat - echo "~~~ Running tests" - - # TODO move this section to base image / pre-command hook - echo "~~~ Installing kind" - asdf plugin add kind - asdf install kind $ASDF_KIND_VERSION - - .buildkite/deploy/kubernetes/scripts/kind-setup.sh echo "~~~ Running tests" export KUBECONFIG="$$PWD/kubecfg" - cd metricbeat && mage goIntegTest + cd metricbeat + mage goIntegTest retry: automatic: - limit: 3 @@ -83,25 +79,18 @@ steps: - github_commit_status: context: "metricbeat: Go Integration Tests" - - label: ":python: Python Integration Tests" + - label: ":ubuntu: Metricbeat Python Integration Tests" key: "mandatory-python-int-test" command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet metricbeat - echo "~~~ Running tests" - - # TODO move this section to base image / pre-command hook - echo "~~~ Installing kind" - asdf plugin add kind - asdf install kind $ASDF_KIND_VERSION - - .buildkite/deploy/kubernetes/scripts/kind-setup.sh echo "~~~ Running tests" export KUBECONFIG="$$PWD/kubecfg" - cd metricbeat && mage pythonIntegTest + cd metricbeat + mage pythonIntegTest retry: automatic: - limit: 3 @@ -116,7 +105,7 @@ steps: - github_commit_status: context: "metricbeat: Python Integration Tests" - - label: ":negative_squared_cross_mark: Cross compile" + - label: ":ubuntu: Metricbeat Crosscompile" key: "mandatory-cross-compile" command: "make -C metricbeat crosscompile" retry: @@ -131,9 +120,9 @@ steps: - "metricbeat/build/*.json" notify: - github_commit_status: - context: "metricbeat: Cross compile" + context: "metricbeat: Crosscompile" - - label: ":windows: Windows 2016 Unit Tests" + - label: ":windows: Metricbeat Win-2016 Unit Tests" command: | Set-Location -Path metricbeat mage build unitTest @@ -152,9 +141,9 @@ steps: - "metricbeat/build/*.json" notify: - github_commit_status: - context: "metricbeat: Windows 2016 Unit Tests" + context: "metricbeat: Win-2016 Unit Tests" - - label: ":windows: Windows 2022 Unit Tests" + - label: ":windows: Metricbeat Win-2022 Unit Tests" command: | Set-Location -Path metricbeat mage build unitTest @@ -173,13 +162,14 @@ steps: - "metricbeat/build/*.json" notify: - github_commit_status: - context: "metricbeat: Windows 2022 Unit Tests" + context: "metricbeat: Win-2022 Unit Tests" - group: "Metricbeat Extended Windows Tests" key: "metricbeat-extended-win-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ + steps: - - label: ":windows: Windows 10 Unit Tests" + - label: ":windows: Metricbeat Win 10 Unit Tests" command: | Set-Location -Path metricbeat mage build unitTest @@ -198,9 +188,9 @@ steps: - "metricbeat/build/*.json" notify: - github_commit_status: - context: "metricbeat: Extended Windows 10 Unit Tests" + context: "metricbeat: Win 10 Unit Tests" - - label: ":windows: Windows 11 Unit Tests" + - label: ":windows: Metricbeat Win 11 Unit Tests" command: | Set-Location -Path metricbeat mage build unitTest @@ -219,9 +209,9 @@ steps: - "metricbeat/build/*.json" notify: - github_commit_status: - context: "metricbeat: Extended Windows 11 Unit Tests" + context: "metricbeat: Win 11 Unit Tests" - - label: ":windows: Windows 2019 Unit Tests" + - label: ":windows: Metricbeat Win-2019 Unit Tests" command: | Set-Location -Path metricbeat mage build unitTest @@ -240,18 +230,20 @@ steps: - "metricbeat/build/*.json" notify: - github_commit_status: - context: "metricbeat: Extended Windows 2019 Unit Tests" + context: "metricbeat: Win-2019 Unit Tests" - - group: "Metricbeat Extended MacOS Tests" + - group: "Metricbeat Extended macOS Tests" key: "metricbeat-extended-macos-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + steps: - - label: ":mac: MacOS x64_64 Unit Tests" + - label: ":mac: Metricbeat macOS x64_64 Unit Tests" key: "extended-macos-x64-64-unit-tests" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh - cd metricbeat && mage build unitTest + cd metricbeat + mage build unitTest retry: automatic: - limit: 3 @@ -263,15 +255,17 @@ steps: - "metricbeat/build/*.json" notify: - github_commit_status: - context: "metricbeat: Extended MacOS x86_64 Unit Tests" + context: "metricbeat: macOS x86_64 Unit Tests" - - label: ":mac: MacOS arm64 Unit Tests" + - label: ":mac: Metricbeat macOS arm64 Unit Tests" key: "extended-macos-arm64-unit-tests" - skip: "due to https://github.com/elastic/beats/issues/33035" + skip: "Skipping due to elastic/beats#33035" + # https://github.com/elastic/beats/issues/33035 command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh - cd metricbeat && mage build unitTest + cd metricbeat + mage build unitTest retry: automatic: - limit: 3 @@ -283,7 +277,7 @@ steps: - "metricbeat/build/*.json" notify: - github_commit_status: - context: "metricbeat: Extended MacOS arm64 Unit Tests" + context: "metricbeat: macOS arm64 Unit Tests" - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed @@ -295,9 +289,11 @@ steps: - group: "Metricbeat Packaging" key: "metricbeat-packaging" steps: - - label: ":linux: Packaging Linux" + - label: ":ubuntu: Metricbeat Packaging Linux" key: "packaging-linux" - command: "cd metricbeat && mage package" + command: | + cd metricbeat + mage package retry: automatic: - limit: 3 @@ -314,20 +310,22 @@ steps: - github_commit_status: context: "metricbeat: Packaging Linux" - - label: ":linux: Packaging ARM" + - label: ":ubuntu: Metricbeat Packaging Linux arm64" key: "packaging-arm" - command: "cd metricbeat && mage package" + command: | + cd metricbeat + mage package retry: automatic: - limit: 3 timeout_in_minutes: 20 agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: PLATFORMS: "linux/arm64" PACKAGES: "docker" notify: - github_commit_status: - context: "metricbeat: Packaging Linux ARM" + context: "metricbeat: Packaging Linux arm64"