From 3f3d352854827e1d3f0bcef56ada8cee720a536c Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Thu, 4 Apr 2024 18:37:50 +0300 Subject: [PATCH] updated auditbeat and filebeat pipelines --- .buildkite/auditbeat/auditbeat-pipeline.yml | 2 +- .buildkite/filebeat/filebeat-pipeline.yml | 2 +- .buildkite/filebeat/scripts/package-step.sh | 51 ----- .../scripts/generate_auditbeat_pipeline.sh | 187 ------------------ .../scripts/generate_filebeat_pipeline.sh | 168 ---------------- 5 files changed, 2 insertions(+), 408 deletions(-) delete mode 100755 .buildkite/filebeat/scripts/package-step.sh delete mode 100644 .buildkite/scripts/generate_auditbeat_pipeline.sh delete mode 100644 .buildkite/scripts/generate_filebeat_pipeline.sh diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index a7223a385b8..a8def81f1fa 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -62,7 +62,7 @@ steps: - label: ":linux: Load dynamic auditbeat pipeline" key: "auditbeat-pipeline" - command: ".buildkite/scripts/generate_auditbeat_pipeline.sh" + command: ".buildkite/auditbeat/generate_auditbeat_pipeline.sh" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index bdf3a56ca24..e08def79385 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -60,7 +60,7 @@ steps: - label: ":linux: Load dynamic filebeat pipeline" key: "filebeat-pipeline" - command: ".buildkite/scripts/generate_filebeat_pipeline.sh" + command: ".buildkite/filebeat/generate_filebeat_pipeline.sh" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/filebeat/scripts/package-step.sh b/.buildkite/filebeat/scripts/package-step.sh deleted file mode 100755 index 4365f9d8d51..00000000000 --- a/.buildkite/filebeat/scripts/package-step.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/env-scripts/util.sh - -changeset="^filebeat/ -^go.mod -^pytest.ini -^dev-tools/ -^libbeat/ -^testing/ -^\.buildkite/filebeat/" - -if are_files_changed "$changeset"; then - bk_pipeline=$(cat <<-YAML - steps: - - label: ":ubuntu: Packaging Linux X86" - key: "package-linux-x86" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" - command: - - ".buildkite/filebeat/scripts/package.sh" - notify: - - github_commit_status: - context: "Filebeat/Packaging: Linux X86" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - - - label: ":linux: Packaging Linux ARM" - key: "package-linux-arm" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" - command: - - ".buildkite/filebeat/scripts/package.sh" - notify: - - github_commit_status: - context: "Filebeat/Packaging: ARM" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.large" -YAML -) - echo "${bk_pipeline}" | buildkite-agent pipeline upload -else - buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning' - exit 0 -fi diff --git a/.buildkite/scripts/generate_auditbeat_pipeline.sh b/.buildkite/scripts/generate_auditbeat_pipeline.sh deleted file mode 100644 index 0ff26cda5d3..00000000000 --- a/.buildkite/scripts/generate_auditbeat_pipeline.sh +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/common.sh - -set -euo pipefail - -pipelineName="pipeline.auditbeat-dynamic.yml" - -# TODO: steps: must be always included -echo "Add the mandatory and extended tests without additional conditions into the pipeline" -if are_conditions_met_mandatory_tests; then - cat > $pipelineName <<- YAML - -steps: - - group: "Auditbeat Mandatory Testing" - key: "mandatory-tests" - - steps: - - label: ":ubuntu: Unit Tests" - command: - - ".buildkite/auditbeat/scripts/unit-tests.sh" - notify: - - github_commit_status: - context: "Auditbeat: linux/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - - - label: ":rhel: Unit Tests" - command: - - ".buildkite/auditbeat/scripts/unit-tests.sh" - notify: - - github_commit_status: - context: "Auditbeat: rhel/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_RHEL9}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - - - label: ":windows:-2016 Unit Tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - notify: - - github_commit_status: - context: "Auditbeat: windows 2016/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - - - label: ":windows:-2022 Unit Tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - notify: - - github_commit_status: - context: "Auditbeat: windows 2022/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - - - label: ":linux: Crosscompile" - command: - - ".buildkite/auditbeat/scripts/crosscompile.sh" - env: - GOX_FLAGS: "-arch amd64" - notify: - - github_commit_status: - context: "Auditbeat: Crosscompile" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" -YAML -else - echo "The conditions don't match to requirements for generating pipeline steps." - exit 0 -fi - -echo "Check and add the Extended Tests into the pipeline" - -if are_conditions_met_arm_tests; then - cat >> $pipelineName <<- YAML - - group: "Extended Tests" - key: "extended-tests-arm" - steps: - - label: ":arm: ARM64 Unit Tests" - key: "extended-arm64-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_ARM64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - notify: - - github_commit_status: - context: "Auditbeat: ARM Unit tests" -YAML -fi - -if are_conditions_met_win_tests; then - cat >> $pipelineName <<- YAML - - group: "Windows Extended Testing" - key: "extended-tests-win" - steps: - - label: ":windows: Windows 2019 Unit Tests" - key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - notify: - - github_commit_status: - context: "Auditbeat: Windows 2019 Unit Tests" - - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - notify: - - github_commit_status: - context: "Auditbeat: Windows 10 Unit Tests" - - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - notify: - - github_commit_status: - context: "Auditbeat: Windows 11 Unit Tests" -YAML -fi - -echo "Check and add the Packaging into the pipeline" -if are_conditions_met_packaging; then -cat >> $pipelineName <<- YAML - - group: "Packaging" - key: "packaging" - depends_on: - - "mandatory-tests" - steps: - - label: Package pipeline - commands: ".buildkite/scripts/packaging/package-step.sh" - notify: - - github_commit_status: - context: "Auditbeat: Packaging" - - -YAML -fi - -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName - -echo "--- Loading dynamic steps" -buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_filebeat_pipeline.sh b/.buildkite/scripts/generate_filebeat_pipeline.sh deleted file mode 100644 index 63108d5aacd..00000000000 --- a/.buildkite/scripts/generate_filebeat_pipeline.sh +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/common.sh - -set -euo pipefail - -pipelineName="pipeline.filebeat-dynamic.yml" - -# TODO: steps: must be always included -echo "Add the mandatory and extended tests without additional conditions into the pipeline" -if are_conditions_met_mandatory_tests; then - cat > $pipelineName <<- YAML - - -steps: - - group: "Filebeat Mandatory Testing" - key: "mandatory-tests" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat" || build.env("BUILDKITE_PULL_REQUEST") != "false" - - steps: - - label: ":ubuntu: Unit Tests" - command: - - ".buildkite/filebeat/scripts/unit-tests.sh" - notify: - - github_commit_status: - context: "Filebeat: linux/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - label: ":ubuntu: Go Integration Tests" - command: - - ".buildkite/filebeat/scripts/integration-gotests.sh" - notify: - - github_commit_status: - context: "Filebeat: Go Integration Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - label: ":ubuntu: Python Integration Tests" - command: - - ".buildkite/filebeat/scripts/integration-pytests.sh" - notify: - - github_commit_status: - context: "Filebeat: Python Integration Tests" - agents: - provider: gcp - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - label: ":windows:-2016 Unit Tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - notify: - - github_commit_status: - context: "Filebeat: windows/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - label: ":windows:-2022 Unit Tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - notify: - - github_commit_status: - context: "Filebeat: windows 2022/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - -YAML -else - echo "The conditions don't match to requirements for generating pipeline steps." - exit 0 -fi - -echo "Check and add the Extended Tests into the pipeline" - -if are_conditions_met_arm_tests; then - cat >> $pipelineName <<- YAML - - group: "Extended Tests: ARM" - key: "extended-tests-arm" - steps: - - label: ":linux: ARM64 Unit Tests" - key: "arm-extended" - command: - - ".buildkite/filebeat/scripts/unit-tests.sh" - notify: - - github_commit_status: - context: "Filebeat/Extended: Unit Tests ARM" - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "filebeat/build/*.xml" - -YAML -fi - -if are_conditions_met_win_tests; then - cat >> $pipelineName <<- YAML - - group: "Windows Extended Testing" - key: "extended-tests-win" - steps: - - label: ":windows: Win 2019 Unit Tests" - key: "win-extended-2019" - command: ".buildkite/scripts/win_unit_tests.ps1" - notify: - - github_commit_status: - context: "Filebeat/Extended: Win-2019 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" -YAML -fi - -echo "Check and add the Packaging into the pipeline" -if are_conditions_met_packaging; then -cat >> $pipelineName <<- YAML - - group: "Packaging" - key: "packaging" - if: build.env("BUILDKITE_PULL_REQUEST") != "false" - depends_on: - - "mandatory-tests" - - steps: - - label: Package pipeline - commands: ".buildkite/scripts/packaging/package-step.sh" - notify: - - github_commit_status: - context: "Filebeat: Packaging" - -YAML -fi - -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName - -echo "--- Loading dynamic steps" -buildkite-agent pipeline upload $pipelineName