From cba30347320f6d949c970c334f7e7bd50a92f0bd Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 5 Apr 2024 13:58:41 +0300 Subject: [PATCH] [8.13](backport #38659) Deploy K8S Pipeline migration (#38713) * Deploy K8S Pipeline migration (#38659) * updated auditbeat & deploy k8s pipelines * added kind & kuve env vars, set higer ulimit * added k8s env setup (cherry picked from commit aaa482983c5423b50b660a7fe534782a2647cf9d) * updated filebeat pipeline --------- Co-authored-by: Olga Naydyonock --- .buildkite/auditbeat/auditbeat-pipeline.yml | 10 +- .../generate_auditbeat_pipeline.sh | 143 +++++++----- .buildkite/auditbeat/scripts/crosscompile.sh | 6 - .buildkite/auditbeat/scripts/unit-tests.sh | 9 - .../deploy/kubernetes/deploy-k8s-pipeline.yml | 64 +++++- .../deploy/kubernetes/scripts/install-kind.sh | 40 ++++ .../kubernetes/scripts/install-kubectl.sh | 42 ++++ .../deploy/kubernetes/scripts/kind-setup.sh | 7 + .buildkite/deploy/kubernetes/scripts/make.sh | 7 + .../kubernetes/scripts/setup-k8s-env.sh | 40 ++++ .buildkite/env-scripts/env.sh | 8 + .buildkite/filebeat/filebeat-pipeline.yml | 8 +- .../filebeat/generate_filebeat_pipeline.sh | 213 ++++++++++++++++++ .../filebeat/scripts/integration-gotests.sh | 10 - .../filebeat/scripts/integration-pytests.sh | 10 - .buildkite/filebeat/scripts/unit-tests.sh | 9 - .buildkite/heartbeat/heartbeat-pipeline.yml | 136 +++++++---- .buildkite/hooks/pre-command | 69 +++--- .buildkite/pull-requests.json | 10 +- .../scripts/generate_filebeat_pipeline.sh | 168 -------------- .buildkite/scripts/setenv.sh | 1 + .../module/file_integrity/fileinfo_test.go | 12 + .../module/file_integrity/metricset_test.go | 18 ++ 23 files changed, 681 insertions(+), 359 deletions(-) rename .buildkite/{scripts => auditbeat}/generate_auditbeat_pipeline.sh (56%) delete mode 100755 .buildkite/auditbeat/scripts/crosscompile.sh delete mode 100755 .buildkite/auditbeat/scripts/unit-tests.sh create mode 100755 .buildkite/deploy/kubernetes/scripts/install-kind.sh create mode 100755 .buildkite/deploy/kubernetes/scripts/install-kubectl.sh create mode 100755 .buildkite/deploy/kubernetes/scripts/kind-setup.sh create mode 100755 .buildkite/deploy/kubernetes/scripts/make.sh create mode 100755 .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh create mode 100644 .buildkite/filebeat/generate_filebeat_pipeline.sh delete mode 100755 .buildkite/filebeat/scripts/integration-gotests.sh delete mode 100755 .buildkite/filebeat/scripts/integration-pytests.sh delete mode 100755 .buildkite/filebeat/scripts/unit-tests.sh delete mode 100644 .buildkite/scripts/generate_filebeat_pipeline.sh diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index df5504f2fc0..1905d63de31 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -2,9 +2,9 @@ name: "beats-auditbeat" env: - AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64" + BEATS_PROJECT_NAME: "auditbeat" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64" IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" @@ -12,11 +12,11 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - BEATS_PROJECT_NAME: "auditbeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" steps: - input: "Input Parameters" @@ -62,7 +62,7 @@ steps: - label: ":linux: Load dynamic auditbeat pipeline" key: "auditbeat-pipeline" - command: ".buildkite/scripts/generate_auditbeat_pipeline.sh" + command: ".buildkite/auditbeat/generate_auditbeat_pipeline.sh" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/scripts/generate_auditbeat_pipeline.sh b/.buildkite/auditbeat/generate_auditbeat_pipeline.sh similarity index 56% rename from .buildkite/scripts/generate_auditbeat_pipeline.sh rename to .buildkite/auditbeat/generate_auditbeat_pipeline.sh index 0ff26cda5d3..56befb51d25 100644 --- a/.buildkite/scripts/generate_auditbeat_pipeline.sh +++ b/.buildkite/auditbeat/generate_auditbeat_pipeline.sh @@ -6,19 +6,18 @@ set -euo pipefail pipelineName="pipeline.auditbeat-dynamic.yml" -# TODO: steps: must be always included +# TODO: steps: must be always included echo "Add the mandatory and extended tests without additional conditions into the pipeline" if are_conditions_met_mandatory_tests; then cat > $pipelineName <<- YAML steps: - group: "Auditbeat Mandatory Testing" - key: "mandatory-tests" + key: "mandatory-tests" steps: - label: ":ubuntu: Unit Tests" - command: - - ".buildkite/auditbeat/scripts/unit-tests.sh" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" notify: - github_commit_status: context: "Auditbeat: linux/Unit Tests" @@ -26,13 +25,10 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":rhel: Unit Tests" - command: - - ".buildkite/auditbeat/scripts/unit-tests.sh" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" notify: - github_commit_status: context: "Auditbeat: rhel/Unit Tests" @@ -40,43 +36,39 @@ steps: provider: "gcp" image: "${IMAGE_RHEL9}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":windows:-2016 Unit Tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + key: "windows-2016" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" notify: - github_commit_status: - context: "Auditbeat: windows 2016/Unit Tests" + context: "Auditbeat: windows-2016/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2016}" machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":windows:-2022 Unit Tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + key: "windows-2022" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" notify: - github_commit_status: - context: "Auditbeat: windows 2022/Unit Tests" + context: "Auditbeat: windows-2022/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2022}" machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":linux: Crosscompile" command: - - ".buildkite/auditbeat/scripts/crosscompile.sh" + - "make -C auditbeat crosscompile" env: GOX_FLAGS: "-arch amd64" notify: @@ -94,22 +86,58 @@ fi echo "Check and add the Extended Tests into the pipeline" -if are_conditions_met_arm_tests; then +if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then cat >> $pipelineName <<- YAML + - group: "Extended Tests" - key: "extended-tests-arm" + key: "extended-tests" steps: - - label: ":arm: ARM64 Unit Tests" - key: "extended-arm64-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" + +YAML +fi + +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - label: ":mac: MacOS Unit Tests" + key: "macos-unit-tests-extended" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + notify: + - github_commit_status: + context: "Auditbeat: MacOS Unit Tests" agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_ARM64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":mac: MacOS ARM Unit Tests" + key: "macos-arm64-unit-tests-extended" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + notify: + - github_commit_status: + context: "Auditbeat: MacOS ARM Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + - label: ":linux: ARM Ubuntu Unit Tests" + key: "extended-arm64-unit-test" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" notify: - github_commit_status: - context: "Auditbeat: ARM Unit tests" + context: "Auditbeat: Unit Tests ARM" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + YAML fi @@ -117,48 +145,49 @@ if are_conditions_met_win_tests; then cat >> $pipelineName <<- YAML - group: "Windows Extended Testing" key: "extended-tests-win" + steps: - - label: ":windows: Windows 2019 Unit Tests" - key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + - label: ":windows:-2019 Unit Tests" + key: "windows-2019-extended" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + notify: + - github_commit_status: + context: "Auditbeat: Win-2019 Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 + disk_size: 200 disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows:-11 Unit Tests" + key: "windows-11-extended" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" notify: - github_commit_status: - context: "Auditbeat: Windows 2019 Unit Tests" - - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + context: "Auditbeat: Win-11 Unit Tests" agents: provider: "gcp" - image: "${IMAGE_WIN_10}" + image: "${IMAGE_WIN_11}" machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 + disk_size: 200 disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows:-10 Unit Tests" + key: "windows-10-extended" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" notify: - github_commit_status: - context: "Auditbeat: Windows 10 Unit Tests" - - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + context: "Auditbeat: Win-10 Unit Tests" agents: provider: "gcp" - image: "${IMAGE_WIN_11}" + image: "${IMAGE_WIN_10}" machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 + disk_size: 200 disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - notify: - - github_commit_status: - context: "Auditbeat: Windows 11 Unit Tests" YAML fi @@ -166,9 +195,10 @@ echo "Check and add the Packaging into the pipeline" if are_conditions_met_packaging; then cat >> $pipelineName <<- YAML - group: "Packaging" - key: "packaging" + key: "packaging" depends_on: - "mandatory-tests" + steps: - label: Package pipeline commands: ".buildkite/scripts/packaging/package-step.sh" @@ -176,7 +206,6 @@ cat >> $pipelineName <<- YAML - github_commit_status: context: "Auditbeat: Packaging" - YAML fi diff --git a/.buildkite/auditbeat/scripts/crosscompile.sh b/.buildkite/auditbeat/scripts/crosscompile.sh deleted file mode 100755 index da8452d5380..00000000000 --- a/.buildkite/auditbeat/scripts/crosscompile.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -echo "--- Executing Crosscompile" -make -C auditbeat crosscompile diff --git a/.buildkite/auditbeat/scripts/unit-tests.sh b/.buildkite/auditbeat/scripts/unit-tests.sh deleted file mode 100755 index 4b8e86243c9..00000000000 --- a/.buildkite/auditbeat/scripts/unit-tests.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -echo "--- Running Unit Tests" -sudo chmod -R go-w auditbeat/ - -umask 0022 -mage -d auditbeat build unitTest diff --git a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml index 34321b61161..246aac9f80a 100644 --- a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml +++ b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml @@ -1,5 +1,65 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +env: + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + steps: - - label: "Example test" - command: echo "Hello!" + - label: "Checks" + command: ".buildkite/deploy/kubernetes/scripts/make.sh" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") == "kubernetes" + notify: + - github_commit_status: + context: "Deploy K8S/Checks" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + + - label: "K8S Test/K8S version: v1.29.0" + key: "k8s-test-129" + env: + K8S_VERSION: "v1.29.0" + commands: + - "MODULE=kubernetes make -C metricbeat integration-tests" + - "make -C deploy/kubernetes test" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + + - label: "K8S Test/K8S version: v1.28.0" + key: "k8s-test-128" + env: + K8S_VERSION: "v1.28.0" + commands: + - "MODULE=kubernetes make -C metricbeat integration-tests" + - "make -C deploy/kubernetes test" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + + - label: "K8S Test/K8S version: v1.27.3" + key: "k8s-test-1273" + env: + K8S_VERSION: "v1.27.3" + commands: + - "MODULE=kubernetes make -C metricbeat integration-tests" + - "make -C deploy/kubernetes test" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + + - label: "K8S Test/K8S version: v1.26.6" + key: "k8s-test-1266" + env: + K8S_VERSION: "v1.26.6" + commands: + - "MODULE=kubernetes make -C metricbeat integration-tests" + - "make -C deploy/kubernetes test" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" diff --git a/.buildkite/deploy/kubernetes/scripts/install-kind.sh b/.buildkite/deploy/kubernetes/scripts/install-kind.sh new file mode 100755 index 00000000000..8c399d2de37 --- /dev/null +++ b/.buildkite/deploy/kubernetes/scripts/install-kind.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -euo pipefail + +MSG="environment variable missing." +KIND_VERSION=${KIND_VERSION:?$MSG} +KIND_BINARY="${BIN}/kind" + +if command -v kind +then + set +e + echo "Found Kind. Checking version.." + FOUND_KIND_VERSION=$(kind --version 2>&1 >/dev/null | awk '{print $3}') + if [ "$FOUND_KIND_VERSION" == "$KIND_VERSION" ] + then + echo "--- Versions match. No need to install Kind. Exiting." + exit 0 + fi + set -e +fi + +echo "UNMET DEP: Installing Kind" + +OS=$(uname -s| tr '[:upper:]' '[:lower:]') +ARCH=$(uname -m| tr '[:upper:]' '[:lower:]') +if [ "${ARCH}" == "aarch64" ] ; then + ARCH_SUFFIX=arm64 +else + ARCH_SUFFIX=amd64 +fi + +if curl -sSLo "${KIND_BINARY}" "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${OS}-${ARCH_SUFFIX}" ; then + chmod +x "${KIND_BINARY}" + echo "Kind installed: ${KIND_VERSION}" +else + echo "Something bad with the download, let's delete the corrupted binary" + if [ -e "${KIND_BINARY}" ] ; then + rm "${KIND_BINARY}" + fi + exit 1 +fi diff --git a/.buildkite/deploy/kubernetes/scripts/install-kubectl.sh b/.buildkite/deploy/kubernetes/scripts/install-kubectl.sh new file mode 100755 index 00000000000..7f6c75bf3b9 --- /dev/null +++ b/.buildkite/deploy/kubernetes/scripts/install-kubectl.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +MSG="parameter missing." +K8S_VERSION=${K8S_VERSION:?$MSG} +KUBECTL_BINARY="${BIN}/kubectl" + +if command -v kubectl +then + set +e + echo "Found kubectl. Checking version.." + FOUND_KUBECTL_VERSION=$(kubectl version --client --short 2>&1 >/dev/null | awk '{print $3}') + if [ "${FOUND_KUBECTL_VERSION}" == "${K8S_VERSION}" ] + then + echo "Kubectl Versions match. No need to install kubectl. Exiting." + exit 0 + fi + set -e +fi + +echo "UNMET DEP: Installing kubectl" + +OS=$(uname -s| tr '[:upper:]' '[:lower:]') +ARCH=$(uname -m| tr '[:upper:]' '[:lower:]') +if [ "${ARCH}" == "aarch64" ] ; then + ARCH_SUFFIX=arm64 +else + ARCH_SUFFIX=amd64 +fi + +if curl -sSLo "${KUBECTL_BINARY}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/${OS}/${ARCH_SUFFIX}/kubectl" ; then + chmod +x "${KUBECTL_BINARY}" + echo "Current K8S Version: ${K8S_VERSION}" + echo "Kubectl installed: ${KUBECTL_BINARY}" +else + echo "--- Something bad with the download, let's delete the corrupted binary" + if [ -e "${KUBECTL_BINARY}" ] ; then + rm "${KUBECTL_BINARY}" + fi + exit 1 +fi + diff --git a/.buildkite/deploy/kubernetes/scripts/kind-setup.sh b/.buildkite/deploy/kubernetes/scripts/kind-setup.sh new file mode 100755 index 00000000000..ca46cdb0fd8 --- /dev/null +++ b/.buildkite/deploy/kubernetes/scripts/kind-setup.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +kind create cluster --image kindest/node:${K8S_VERSION} + +echo "Cluster info: " +kubectl cluster-info diff --git a/.buildkite/deploy/kubernetes/scripts/make.sh b/.buildkite/deploy/kubernetes/scripts/make.sh new file mode 100755 index 00000000000..4c9120a2d90 --- /dev/null +++ b/.buildkite/deploy/kubernetes/scripts/make.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -euo pipefail + +echo "--- Checking K8S" +make -C deploy/kubernetes all +make check-no-changes diff --git a/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh b/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh new file mode 100755 index 00000000000..3656318bd64 --- /dev/null +++ b/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/util.sh + +echo "--- Installing kind & kubectl" +retry_with_count 5 .buildkite/deploy/kubernetes/scripts/install-kind.sh +retry_with_count 5 .buildkite/deploy/kubernetes/scripts/install-kubectl.sh + +echo "--- Setting up kind" +max_retries=3 +timeout=5 +retries=0 + +while true; do + echo "Creating cluster" + script_output=$(.buildkite/deploy/kubernetes/scripts/kind-setup.sh 2>&1) + exit_code=$? + + echo "Script Output: $script_output" + + if [ $exit_code -eq 0 ]; then + break + else + retries=$((retries + 1)) + + if [ $retries -gt $max_retries ]; then + kind delete cluster + echo "Kind setup FAILED: $script_output" + exit 1 + fi + + kind delete cluster + + sleep_time=$((timeout * retries)) + echo "Retry #$retries failed. Retrying after ${sleep_time}s..." + sleep $sleep_time + fi +done diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh index f28658a107d..b30b26c3e8a 100644 --- a/.buildkite/env-scripts/env.sh +++ b/.buildkite/env-scripts/env.sh @@ -30,6 +30,9 @@ ONLY_DOCS=$(changeset_applies "$DOCS_CHANGESET") PACKAGING_CHANGES=$(changeset_applies "$PACKAGING_CHANGESET") GO_MOD_CHANGES=$(changeset_applies "^go.mod") +KIND_VERSION="v0.20.0" +KUBECONFIG="${WORKSPACE}/kubecfg" + export REPO export WORKSPACE export BIN @@ -51,3 +54,8 @@ export DOCKER_REGISTRY export ONLY_DOCS export PACKAGING_CHANGES export GO_MOD_CHANGES + +export KIND_VERSION +export KUBECONFIG + +add_bin_path diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 2d3416cf5aa..9daeef287d9 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -2,20 +2,20 @@ name: "beats-filebeat" env: + BEATS_PROJECT_NAME: "filebeat" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - BEATS_PROJECT_NAME: "filebeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" steps: - input: "Input Parameters" @@ -61,7 +61,7 @@ steps: - label: ":linux: Load dynamic filebeat pipeline" key: "filebeat-pipeline" - command: ".buildkite/scripts/generate_filebeat_pipeline.sh" + command: ".buildkite/filebeat/generate_filebeat_pipeline.sh" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/filebeat/generate_filebeat_pipeline.sh b/.buildkite/filebeat/generate_filebeat_pipeline.sh new file mode 100644 index 00000000000..aa0a03eadc1 --- /dev/null +++ b/.buildkite/filebeat/generate_filebeat_pipeline.sh @@ -0,0 +1,213 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.filebeat-dynamic.yml" + +# TODO: steps: must be always included +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + - group: "Filebeat Mandatory Testing" + key: "mandatory-tests" + + steps: + - label: ":ubuntu: Unit Tests" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + notify: + - github_commit_status: + context: "Filebeat: linux/Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":ubuntu: Go Integration Tests" + command: "cd ${BEATS_PROJECT_NAME} && mage goIntegTest" + notify: + - github_commit_status: + context: "Filebeat: Go Integration Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":ubuntu: Python Integration Tests" + command: "cd ${BEATS_PROJECT_NAME} && mage pythonIntegTest" + notify: + - github_commit_status: + context: "Filebeat: Python Integration Tests" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows:-2016 Unit Tests" + key: "windows-2016" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + notify: + - github_commit_status: + context: "Filebeat: windows-2016/Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows:-2022 Unit Tests" + key: "windows-2022" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + notify: + - github_commit_status: + context: "Filebeat: windows-2022/Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +echo "Check and add the Extended Tests into the pipeline" + +if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: +YAML +fi + +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - label: ":mac: MacOS Unit Tests" + key: "macos-unit-tests-extended" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + notify: + - github_commit_status: + context: "Filebeat: MacOS Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":mac: MacOS ARM Unit Tests" + key: "macos-arm64-unit-tests-extended" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + notify: + - github_commit_status: + context: "Filebeat: MacOS ARM Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" +YAML +fi + +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + + - label: ":linux: ARM Ubuntu Unit Tests" + key: "extended-arm64-unit-test" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + notify: + - github_commit_status: + context: "Filebeat: Unit Tests ARM" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" +YAML +fi + +if are_conditions_met_win_tests; then + cat >> $pipelineName <<- YAML + + - group: "Windows Extended Testing" + key: "extended-tests-win" + steps: + - label: ":windows: Win 2019 Unit Tests" + key: "windows-extended-2019" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + notify: + - github_commit_status: + context: "Filebeat: Win-2019 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows:-11 Unit Tests" + key: "windows-extended-11" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + notify: + - github_commit_status: + context: "Filebeat: Win-11 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows:-10 Unit Tests" + key: "windows-extended-10" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + notify: + - github_commit_status: + context: "Filebeat: Win-10 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" +YAML +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then +cat >> $pipelineName <<- YAML + + - group: "Packaging" + key: "packaging" + depends_on: + - "mandatory-tests" + + steps: + - label: Package pipeline + commands: ".buildkite/scripts/packaging/package-step.sh" + notify: + - github_commit_status: + context: "Filebeat: Packaging" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/filebeat/scripts/integration-gotests.sh b/.buildkite/filebeat/scripts/integration-gotests.sh deleted file mode 100755 index 6de39ff8817..00000000000 --- a/.buildkite/filebeat/scripts/integration-gotests.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -echo "--- Executing Integration Tests" -sudo chmod -R go-w filebeat/ - -cd filebeat -umask 0022 -mage goIntegTest diff --git a/.buildkite/filebeat/scripts/integration-pytests.sh b/.buildkite/filebeat/scripts/integration-pytests.sh deleted file mode 100755 index 9aff8695c35..00000000000 --- a/.buildkite/filebeat/scripts/integration-pytests.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -echo "--- Executing Integration Tests" -sudo chmod -R go-w filebeat/ - -cd filebeat -umask 0022 -mage pythonIntegTest diff --git a/.buildkite/filebeat/scripts/unit-tests.sh b/.buildkite/filebeat/scripts/unit-tests.sh deleted file mode 100755 index 2efb6b1ff8e..00000000000 --- a/.buildkite/filebeat/scripts/unit-tests.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -echo "--- Executing Unit Tests" -sudo chmod -R go-w filebeat/ - -umask 0022 -mage -d filebeat unitTest diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index 2b5f6195f45..e63fb1c60e2 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -3,16 +3,19 @@ env: BEATS_PROJECT_NAME: "heartbeat" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" steps: - group: "Heartbeat Mandatory Testing" @@ -20,48 +23,64 @@ steps: if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat" || build.env("BUILDKITE_PULL_REQUEST") != "false" steps: - - label: ":linux: Unit Tests / {{matrix.image}}" - command: - - ".buildkite/heartbeat/scripts/unit-tests.sh" + - label: ":ubuntu: Unit Tests" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" notify: - github_commit_status: - context: "Heartbeat: linux/Unit Tests" + context: "Heartbeat: ubuntu/Unit Tests" agents: provider: "gcp" - image: "{{matrix.image}}" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "heartbeat/build/*.xml" + - "heartbeat/build/*.json" + + - label: ":rhel:-9 Unit Tests" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + notify: + - github_commit_status: + context: "Heartbeat: rhel-9/Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_RHEL9}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - matrix: - setup: - image: - - "${IMAGE_UBUNTU_X86_64}" - - "${IMAGE_RHEL9}" artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" - - label: ":windows: Unit Tests / {{matrix.image}}" - command: - - ".buildkite/scripts/win_unit_tests.ps1" + - label: ":windows:-2016 Unit Test" + key: "windows-2016" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + notify: + - github_commit_status: + context: "Heartbeat: windows-2016/Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_type: "pd-ssd" + artifact_paths: + - "heartbeat/build/*.xml" + - "heartbeat/build/*.json" + + - label: ":windows:-2022 Unit Test" + key: "windows-2022" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" notify: - github_commit_status: - context: "Heartbeat: windows/Unit Tests" + context: "Heartbeat: windows-2022/Unit Tests" agents: provider: "gcp" - image: "{{matrix.image}}" + image: "${IMAGE_WIN_2022}" machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_type: "pd-ssd" - matrix: - setup: - image: - - "${IMAGE_WIN_2016}" - - "${IMAGE_WIN_2022}" artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" - label: ":ubuntu: Go Integration Tests" - command: - - ".buildkite/heartbeat/scripts/integration-gotests.sh" + command: "cd ${BEATS_PROJECT_NAME} && mage goIntegTest" notify: - github_commit_status: context: "Heartbeat: Go Integration Tests" @@ -74,8 +93,7 @@ steps: - "heartbeat/build/*.json" - label: ":ubuntu: Python Integration Tests" - command: - - ".buildkite/heartbeat/scripts/integration-pytests.sh" + command: "cd ${BEATS_PROJECT_NAME} && mage pythonIntegTest" notify: - github_commit_status: context: "Heartbeat: Python Integration Tests" @@ -95,11 +113,10 @@ steps: - label: ":linux: ARM64 Unit Tests" key: "arm-extended" if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - command: - - ".buildkite/heartbeat/scripts/unit-tests.sh" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" notify: - github_commit_status: - context: "Heartbeat/Extended: Unit Tests ARM" + context: "Heartbeat: Unit Tests ARM" agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -109,27 +126,38 @@ steps: - label: ":mac: MacOS Unit Tests" key: "macos-extended" if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: - - ".buildkite/heartbeat/scripts/unit-tests.sh" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" notify: - github_commit_status: - context: "Heartbeat/Extended: MacOS Unit Tests" + context: "Heartbeat: MacOS Unit Tests" agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" artifact_paths: "heartbeat/build/*.xml" + - label: ":mac: MacOS ARM Unit Tests" + key: "macos-extended-arm" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + notify: + - github_commit_status: + context: "Heartbeat: MacOS ARM Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: "heartbeat/build/*.xml" + - group: "Windows Extended Testing" key: "extended-tests-win" if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ steps: - - label: ":windows: Win 2019 Unit Tests" - key: "win-extended-2019" - command: ".buildkite/scripts/win_unit_tests.ps1" + - label: ":windows:-2019 Unit Tests" + key: "windows-extended-2019" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" notify: - github_commit_status: - context: "Heartbeat/Extended: Win-2019 Unit Tests" + context: "Heartbeat: Win-2019 Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -139,6 +167,36 @@ steps: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" + - label: ":windows:-11 Unit Tests" + key: "windows-extended-11" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + notify: + - github_commit_status: + context: "Heartbeat: Win-11 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_type: "pd-ssd" + artifact_paths: + - "heartbeat/build/*.xml" + - "heartbeat/build/*.json" + + - label: ":windows:-10 Unit Tests" + key: "windows-extended-10" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + notify: + - github_commit_status: + context: "Heartbeat: Win-10 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_type: "pd-ssd" + artifact_paths: + - "heartbeat/build/*.xml" + - "heartbeat/build/*.json" + - group: "Packaging" key: "packaging" if: build.env("BUILDKITE_PULL_REQUEST") != "false" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index e03ade43af1..73aec305150 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,44 +2,29 @@ set -euo pipefail +source .buildkite/env-scripts/util.sh + # Secrets must be redacted # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth" PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account" DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" -PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/private_ci_artifacts_gcs_credentials" GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token" -retry() { - local retries=$1 - shift - local count=0 - until "$@"; do - exit=$? - wait=$((2 ** count)) - count=$((count + 1)) - if [ $count -lt "$retries" ]; then - >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." - sleep $wait - else - >&2 echo "Retry $count/$retries exited $exit, no more retries left." - return $exit - fi - done - return 0 -} - - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then source .buildkite/env-scripts/env.sh - source .buildkite/env-scripts/util.sh - - if [[ -z "${GO_VERSION-""}" ]]; then - export GO_VERSION=$(cat "${WORKSPACE}/.go-version") - fi + # Images with prefix "platform-ingest-beats-*" has Go pre-setup. + # Image itself takes care of Go version download/installation based on .go-version file if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then - ulimit -Sn 30000 + if [[ -z "${GO_VERSION-""}" ]]; then + GO_VERSION=$(cat "${WORKSPACE}/.go-version") + export GO_VERSION + fi + + # To prevent "OSError: [Errno 24] Too many open files" + ulimit -Sn 150000 + echo "--- Ulimit: $(ulimit)" echo "--- Setting up environment" add_bin_path @@ -47,10 +32,20 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "fi with_mage fi + if [[ "$BUILDKITE_STEP_KEY" == windows* ]]; then + git config core.autocrlf false + git rm --quiet --cached -r . + git reset --quiet --hard + fi + if [[ "$BUILDKITE_STEP_KEY" == package* ]]; then - export DOCKER_USERNAME_SECRET=$(retry_with_count 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}") - export DOCKER_PASSWORD_SECRET=$(retry_with_count 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}") - export GITHUB_TOKEN_SECRET=$(retry_with_count 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH}) + DOCKER_USERNAME_SECRET=$(retry_with_count 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}") + DOCKER_PASSWORD_SECRET=$(retry_with_count 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}") + GITHUB_TOKEN_SECRET=$(retry_with_count 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH}) + + export DOCKER_USERNAME_SECRET + export DOCKER_PASSWORD_SECRET + export GITHUB_TOKEN_SECRET docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null @@ -60,6 +55,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "fi git config user.name "$github_username" git config user.email "$github_email" fi + + if [[ "$BUILDKITE_STEP_KEY" == k8s-test* ]]; then + .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh + fi fi ENABLED_BEATS_PIPELINES_SLUGS=( @@ -76,8 +75,8 @@ ENABLED_BEATS_PIPELINES_SLUGS=( "beats-xpack-winlogbeat" "beats-xpack-dockerlogbeat" "beats-xpack-auditbeat" - "beats-xpack-filebeat" - "beats-xpack-heartbeat" + "beats-xpack-filebeat" + "beats-xpack-heartbeat" "beats-xpack-osquerybeat" ) @@ -90,16 +89,16 @@ done if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then if [[ "$BUILDKITE_STEP_KEY" == "extended-cloud-test" ]]; then - BEATS_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) + BEATS_AWS_SECRET_KEY=$(retry_with_count 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) export BEATS_AWS_SECRET_KEY - BEATS_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) + BEATS_AWS_ACCESS_KEY=$(retry_with_count 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) export BEATS_AWS_ACCESS_KEY fi fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then if [[ "$BUILDKITE_STEP_KEY" == "extended-win-10-system-tests" || "$BUILDKITE_STEP_KEY" == "mandatory-win-2022-system-tests" ]]; then - PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) + PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry_with_count 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) export PRIVATE_CI_GCS_CREDENTIALS_SECRET fi fi diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 8219bb2c9e9..abf9580c208 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -26,7 +26,7 @@ "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^/test filebeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$", - "always_trigger_comment_regex": "^/test filebeat(for (arm|macos|windows|extended support))?$|^/package filebeat$", + "always_trigger_comment_regex": "^/test filebeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$", "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], @@ -57,8 +57,8 @@ "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, - "trigger_comment_regex": "^/test auditbeat(for (arm|macos|windows|extended support))?$", - "always_trigger_comment_regex": "^/test auditbeat(for (arm|macos|windows|extended support))?$", + "trigger_comment_regex": "^/test auditbeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$", + "always_trigger_comment_regex": "^/test auditbeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$", "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], @@ -73,8 +73,8 @@ "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, - "trigger_comment_regex": "^/test heartbeat(for (arm|macos|windows|extended support))?$|^/package heartbeat$", - "always_trigger_comment_regex": "^/test heartbeat(for (arm|macos|windows|extended support))?$|^/package heartbeat$", + "trigger_comment_regex": "^/test heartbeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$", + "always_trigger_comment_regex": "^/test heartbeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$", "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], diff --git a/.buildkite/scripts/generate_filebeat_pipeline.sh b/.buildkite/scripts/generate_filebeat_pipeline.sh deleted file mode 100644 index 63108d5aacd..00000000000 --- a/.buildkite/scripts/generate_filebeat_pipeline.sh +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/common.sh - -set -euo pipefail - -pipelineName="pipeline.filebeat-dynamic.yml" - -# TODO: steps: must be always included -echo "Add the mandatory and extended tests without additional conditions into the pipeline" -if are_conditions_met_mandatory_tests; then - cat > $pipelineName <<- YAML - - -steps: - - group: "Filebeat Mandatory Testing" - key: "mandatory-tests" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat" || build.env("BUILDKITE_PULL_REQUEST") != "false" - - steps: - - label: ":ubuntu: Unit Tests" - command: - - ".buildkite/filebeat/scripts/unit-tests.sh" - notify: - - github_commit_status: - context: "Filebeat: linux/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - label: ":ubuntu: Go Integration Tests" - command: - - ".buildkite/filebeat/scripts/integration-gotests.sh" - notify: - - github_commit_status: - context: "Filebeat: Go Integration Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - label: ":ubuntu: Python Integration Tests" - command: - - ".buildkite/filebeat/scripts/integration-pytests.sh" - notify: - - github_commit_status: - context: "Filebeat: Python Integration Tests" - agents: - provider: gcp - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - label: ":windows:-2016 Unit Tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - notify: - - github_commit_status: - context: "Filebeat: windows/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - label: ":windows:-2022 Unit Tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - notify: - - github_commit_status: - context: "Filebeat: windows 2022/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - -YAML -else - echo "The conditions don't match to requirements for generating pipeline steps." - exit 0 -fi - -echo "Check and add the Extended Tests into the pipeline" - -if are_conditions_met_arm_tests; then - cat >> $pipelineName <<- YAML - - group: "Extended Tests: ARM" - key: "extended-tests-arm" - steps: - - label: ":linux: ARM64 Unit Tests" - key: "arm-extended" - command: - - ".buildkite/filebeat/scripts/unit-tests.sh" - notify: - - github_commit_status: - context: "Filebeat/Extended: Unit Tests ARM" - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "filebeat/build/*.xml" - -YAML -fi - -if are_conditions_met_win_tests; then - cat >> $pipelineName <<- YAML - - group: "Windows Extended Testing" - key: "extended-tests-win" - steps: - - label: ":windows: Win 2019 Unit Tests" - key: "win-extended-2019" - command: ".buildkite/scripts/win_unit_tests.ps1" - notify: - - github_commit_status: - context: "Filebeat/Extended: Win-2019 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" -YAML -fi - -echo "Check and add the Packaging into the pipeline" -if are_conditions_met_packaging; then -cat >> $pipelineName <<- YAML - - group: "Packaging" - key: "packaging" - if: build.env("BUILDKITE_PULL_REQUEST") != "false" - depends_on: - - "mandatory-tests" - - steps: - - label: Package pipeline - commands: ".buildkite/scripts/packaging/package-step.sh" - notify: - - github_commit_status: - context: "Filebeat: Packaging" - -YAML -fi - -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName - -echo "--- Loading dynamic steps" -buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 8da3796aa1e..f0116c6b308 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash set -euo pipefail + REPO="beats" TMP_FOLDER="tmp.${REPO}" DOCKER_REGISTRY="docker.elastic.co" diff --git a/auditbeat/module/file_integrity/fileinfo_test.go b/auditbeat/module/file_integrity/fileinfo_test.go index d2ac75821fc..4aa63d0a400 100644 --- a/auditbeat/module/file_integrity/fileinfo_test.go +++ b/auditbeat/module/file_integrity/fileinfo_test.go @@ -30,6 +30,9 @@ import ( ) func TestNewMetadata(t *testing.T) { + // Can be removed after https://github.com/elastic/beats/issues/37701 is solved + skipOnBuildkiteDarwin(t, "Group check") + f, err := os.CreateTemp(t.TempDir(), "metadata") if err != nil { t.Fatal(err) @@ -95,6 +98,9 @@ func TestNewMetadata(t *testing.T) { } func TestSetUIDSetGIDBits(t *testing.T) { + // Can be removed after https://github.com/elastic/beats/issues/37701 is solved + skipOnBuildkiteDarwin(t, "Wheel permission issue") + f, err := os.CreateTemp(t.TempDir(), "setuid") if err != nil { t.Fatal(err) @@ -148,3 +154,9 @@ func TestSetUIDSetGIDBits(t *testing.T) { assert.Equal(t, flags&os.ModeSetgid != 0, meta.SetGID) } } + +func skipOnBuildkiteDarwin(t testing.TB, reason string) { + if os.Getenv("BUILDKITE") == "true" && runtime.GOOS == "darwin" { + t.Skip("Skip test on Buildkite MacOS: Wheel permission while expected staff") + } +} diff --git a/auditbeat/module/file_integrity/metricset_test.go b/auditbeat/module/file_integrity/metricset_test.go index 2a6c33e1798..a7f0b922e15 100644 --- a/auditbeat/module/file_integrity/metricset_test.go +++ b/auditbeat/module/file_integrity/metricset_test.go @@ -62,7 +62,11 @@ func TestData(t *testing.T) { func TestActions(t *testing.T) { skipOnCIForDarwinAMD64(t) + + // Can be removed after https://github.com/elastic/ingest-dev/issues/3016 is solved skipOnBuildkiteWindows(t) + // Can be removed after https://github.com/elastic/ingest-dev/issues/3076 is solved + skipOnBuildkiteDarwinArm(t) defer abtest.SetupDataDir(t)() @@ -155,7 +159,11 @@ func TestActions(t *testing.T) { func TestExcludedFiles(t *testing.T) { skipOnCIForDarwinAMD64(t) + + // Can be removed after https://github.com/elastic/ingest-dev/issues/3016 is solved skipOnBuildkiteWindows(t) + // Can be removed after https://github.com/elastic/ingest-dev/issues/3076 is solved + skipOnBuildkiteDarwinArm(t) defer abtest.SetupDataDir(t)() @@ -203,7 +211,11 @@ func TestExcludedFiles(t *testing.T) { func TestIncludedExcludedFiles(t *testing.T) { skipOnCIForDarwinAMD64(t) + + // Can be removed after https://github.com/elastic/ingest-dev/issues/3016 is solved skipOnBuildkiteWindows(t) + // Can be removed after https://github.com/elastic/ingest-dev/issues/3076 is solved + skipOnBuildkiteDarwinArm(t) defer abtest.SetupDataDir(t)() @@ -958,3 +970,9 @@ func skipOnBuildkiteWindows(t testing.TB) { t.Skip("Skip on Buildkite Windows: Shortened TMP problem") } } + +func skipOnBuildkiteDarwinArm(t testing.TB) { + if os.Getenv("BUILDKITE") == "true" && runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + t.Skip("Skip test on Buldkite: unexpected path error") + } +}