diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index d4b33be7690..21e4f04eb13 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -12,7 +12,8 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then source .buildkite/scripts/setenv.sh if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then echo "Skipped pre-command when running the Upload pipeline" diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index 1fb185b59d1..cfaf7818360 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -4,7 +4,7 @@ env: IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" - GCP_HI_PERF_MASHINE_TYPE: "c2d-highcpu-16" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" BEATS_PROJECT_NAME: "libbeat" diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index c42f17d2a36..d882edaaa0a 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -11,7 +11,7 @@ env: GO_AGENT_IMAGE: "golang:${GO_VERSION}" BEATS_PROJECT_NAME: "metricbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" - GCP_HI_PERF_MASHINE_TYPE: "c2d-highcpu-16" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index 34321b61161..0dbb87fc1d1 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -1,5 +1,58 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +env: + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + IMAGE_RHEL9_X86_64: "family/core-rhel-9" + IMAGE_WIN_10: "family/general-windows-10" + IMAGE_WIN_11: "family/general-windows-11" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "packetbeat" steps: - - label: "Example test" - command: echo "Hello!" + + - input: "Input Parameters" + key: "input-run-all-stages" + fields: + - select: "Packetbeat - runPacketbeat" + key: "runPacketbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Packetbeat - runPacketbeatArmTest" + key: "runPacketbeatArmTest" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Packetbeat - runPacketbeatMacOsTests" + key: "runPacketbeatMacOsTests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic packetbeat pipeline" + key: "packetbeat-pipeline" + command: ".buildkite/scripts/generate_packetbeat_pipeline.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index e3dd2ec4ac4..f51097091f4 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -8,19 +8,27 @@ platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]') arch_type="$(uname -m)" GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} ONLY_DOCS=${ONLY_DOCS:-"true"} -runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" -runMetricbeat="$(buildkite-agent meta-data get runMetricbeat --default ${runMetricbeat:-"false"})" -runLibBeatArmTest="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" -runMetricbeatMacOsTests="$(buildkite-agent meta-data get runMetricbeatMacOsTests --default ${runMetricbeatMacOsTests:-"false"})" + +[ -z "${runLibbeat+x}" ] && runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" +[ -z "${runMetricbeat+x}" ] && runMetricbeat="$(buildkite-agent meta-data get runMetricbeat --default ${runMetricbeat:-"false"})" +[ -z "${runPacketbeat+x}" ] && runPacketbeat="$(buildkite-agent meta-data get runPacketbeat --default ${runPacketbeat:-"false"})" +[ -z "${runLibBeatArmTest+x}" ] && runLibBeatArmTest="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" +[ -z "${runPacketbeatArmTest+x}" ] && runPacketbeatArmTest="$(buildkite-agent meta-data get runPacketbeatArmTest --default ${runPacketbeatArmTest:-"false"})" +[ -z "${runMetricbeatMacOsTests+x}" ] && runMetricbeatMacOsTests="$(buildkite-agent meta-data get runMetricbeatMacOsTests --default ${runMetricbeatMacOsTests:-"false"})" +[ -z "${runPacketbeatMacOsTests+x}" ] && runPacketbeatMacOsTests="$(buildkite-agent meta-data get runPacketbeatMacOsTests --default ${runPacketbeatMacOsTests:-"false"})" metricbeat_changeset=( "^metricbeat/.*" - "^go.mod" - "^pytest.ini" - "^dev-tools/.*" + ) + +libbeat_changeset=( "^libbeat/.*" - "^testing/.*" ) + +packetbeat_changeset=( + "^packetbeat/.*" + ) + oss_changeset=( "^go.mod" "^pytest.ini" @@ -28,16 +36,20 @@ oss_changeset=( "^libbeat/.*" "^testing/.*" ) + ci_changeset=( "^.buildkite/.*" ) + go_mod_changeset=( "^go.mod" ) + docs_changeset=( ".*\\.(asciidoc|md)" "deploy/kubernetes/.*-kubernetes\\.yaml" ) + packaging_changeset=( "^dev-tools/packaging/.*" ".go-version" @@ -77,7 +89,7 @@ check_platform_architeture() { go_arch_type="arm64" ;; *) - echo "The current platform/OS type is unsupported yet" + echo "The current platform or OS type is unsupported yet" ;; esac } @@ -109,11 +121,32 @@ with_go() { export PATH="${go_path}:${PATH}" } +checkLinuxType() { + if [ "${platform_type}" == "Linux" ]; then + if grep -q 'ubuntu' /etc/os-release; then + echo "ubuntu" + elif grep -q 'rhel' /etc/os-release; then + echo "rhel" + else + echo "Unsupported Linux" + fi + else + echo "This is not a Linux" + fi +} + with_python() { + local linuxType="$(checkLinuxType)" + echo "${linuxType}" if [ "${platform_type}" == "Linux" ]; then - #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) - sudo apt-get update - sudo apt-get install -y python3-pip python3-venv + if [ "${linuxType}" = "ubuntu" ]; then + sudo apt-get update + sudo apt-get install -y python3-pip python3-venv + elif [ "${linuxType}" = "rhel" ]; then + sudo dnf update -y + sudo dnf install -y python3 python3-pip + pip3 install virtualenv + fi elif [ "${platform_type}" == "Darwin" ]; then brew update pip3 install virtualenv @@ -122,15 +155,30 @@ with_python() { } with_dependencies() { + local linuxType="$(checkLinuxType)" + echo "${linuxType}" if [ "${platform_type}" == "Linux" ]; then - #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) - sudo apt-get update - sudo apt-get install -y libsystemd-dev libpcap-dev + if [ "${linuxType}" = "ubuntu" ]; then + sudo apt-get update + sudo apt-get install -y libsystemd-dev libpcap-dev + elif [ "${linuxType}" = "rhel" ]; then + # sudo dnf update -y + sudo dnf install -y systemd-devel + wget https://mirror.stream.centos.org/9-stream/CRB/${arch_type}/os/Packages/libpcap-devel-1.10.0-4.el9.${arch_type}.rpm #TODO: move this step to our own image + sudo dnf install -y libpcap-devel-1.10.0-4.el9.${arch_type}.rpm #TODO: move this step to our own image + fi elif [ "${platform_type}" == "Darwin" ]; then pip3 install libpcap fi } +config_git() { + if [ -z "$(git config --get user.email)" ]; then + git config --global user.email "beatsmachine@users.noreply.github.com" + git config --global user.name "beatsmachine" + fi +} + retry() { local retries=$1 shift @@ -153,7 +201,6 @@ retry() { are_paths_changed() { local patterns=("${@}") local changelist=() - for pattern in "${patterns[@]}"; do changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) if [ "${#changed_files[@]}" -gt 0 ]; then @@ -183,13 +230,17 @@ are_changed_only_paths() { } are_conditions_met_mandatory_tests() { - if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + if are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" || "${GITHUB_PR_LABELS}" =~ Metricbeat || "${runMetricbeat}" == "true" ]]; then + if are_paths_changed "${metricbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" || "${GITHUB_PR_LABELS}" =~ Metricbeat || "${runMetricbeat}" == "true" ]]; then return 0 fi elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then + if are_paths_changed "${libbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then + return 0 + fi + elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then + if are_paths_changed "${packetbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat" || "${GITHUB_PR_LABELS}" =~ Packetbeat || "${runPacketbeat}" == "true" ]]; then return 0 fi fi @@ -197,27 +248,47 @@ are_conditions_met_mandatory_tests() { return 1 } -are_conditions_met_libbeat_arm_tests() { +are_conditions_met_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat for arm" || "${GITHUB_PR_LABELS}" =~ arm || "${runLibBeatArmTest}" == "true" ]]; then return 0 fi + elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat for arm" || "${GITHUB_PR_LABELS}" =~ arm || "${runPacketbeatArmTest}" == "true" ]]; then + return 0 + fi fi fi return 1 } -are_conditions_met_metricbeat_macos_tests() { - if [[ "${runMetricbeatMacOsTests}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 - return 0 +are_conditions_met_macos_tests() { + if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then + if [[ "${runMetricbeatMacOsTests}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + return 0 + fi + elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then + if [[ "${runPacketbeatMacOsTests}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then + return 0 + fi + fi fi return 1 } are_conditions_met_packaging() { - if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || [[ "${BUILDKITE_TAG}" == "" ]] || [[ "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103 - return 0 + if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then + if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103 + return 0 + fi + elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then + if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then + return 0 + fi + fi fi return 1 } diff --git a/.buildkite/scripts/generate_libbeat_pipeline.sh b/.buildkite/scripts/generate_libbeat_pipeline.sh index 6da1bef711d..0674d0b186b 100755 --- a/.buildkite/scripts/generate_libbeat_pipeline.sh +++ b/.buildkite/scripts/generate_libbeat_pipeline.sh @@ -30,7 +30,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MASHINE_TYPE}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":python: Python Integration Tests" @@ -39,7 +39,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MASHINE_TYPE}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":negative_squared_cross_mark: Cross compile" @@ -48,7 +48,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MASHINE_TYPE}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: " ${BEATS_PROJECT_NAME}/build/*.xml" - label: ":testengine: Stress Tests" @@ -61,10 +61,13 @@ steps: artifact_paths: "${BEATS_PROJECT_NAME}/libbeat-stress-test.xml" YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 fi echo "Check and add the Extended Tests into the pipeline" -if are_conditions_met_libbeat_arm_tests; then +if are_conditions_met_arm_tests; then cat >> $pipelineName <<- YAML - group: "Extended Tests" diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index 0ea19734c4f..850ef14f9bd 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -30,7 +30,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MASHINE_TYPE}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":python: Python Integration Tests" @@ -39,7 +39,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MASHINE_TYPE}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":negative_squared_cross_mark: Cross compile" @@ -107,10 +107,13 @@ steps: disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 fi echo "Check and add the Extended Tests into the pipeline" -if are_conditions_met_metricbeat_macos_tests; then +if are_conditions_met_macos_tests; then cat >> $pipelineName <<- YAML - group: "Extended Tests" @@ -128,7 +131,7 @@ YAML fi echo "Check and add the Packaging into the pipeline" -if are_conditions_met_mandatory_tests && are_conditions_met_packaging; then +if are_conditions_met_packaging; then cat >> $pipelineName <<- YAML - wait: ~ @@ -145,7 +148,7 @@ if are_conditions_met_mandatory_tests && are_conditions_met_packaging; then agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MASHINE_TYPE}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh new file mode 100755 index 00000000000..89ea7a33e20 --- /dev/null +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -0,0 +1,173 @@ +#!/bin/bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.packetbeat-dynamic.yml" + +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":rhel: RHEL-9 Unit Tests" + key: "mandatory-rhel9-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_RHEL9_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - group: "Extended Windowds Tests" + key: "extended-win-tests" + steps: + - label: ":windows: Win 2019 Unit Tests" + key: "extended-win-2019-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 10 Unit Tests" + key: "extended-win-10-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 11 Unit Tests" + key: "extended-win-11-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +if are_conditions_met_arm_tests && are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + +YAML +fi + +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - label: ":mac: MacOS Unit Tests" + key: "extended-macos-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + - label: ":linux: ARM Ubuntu Unit Tests" + key: "extended-arm64-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + + +if are_conditions_met_packaging; then + cat >> $pipelineName <<- YAML + + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: ".buildkite/scripts/packaging.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: ".buildkite/scripts/packaging.sh" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/install_tools.sh b/.buildkite/scripts/install_tools.sh old mode 100644 new mode 100755 index 1a1e3a29f54..80e70ae96c5 --- a/.buildkite/scripts/install_tools.sh +++ b/.buildkite/scripts/install_tools.sh @@ -9,10 +9,14 @@ echo "--- Env preparation" # Temporary solution to fix the issues with "sudo apt get...." https://elastic.slack.com/archives/C0522G6FBNE/p1706003603442859?thread_ts=1706003209.424539&cid=C0522G6FBNE # It could be removed when we use our own image for the BK agent. if [ "${platform_type}" == "Linux" ]; then - DEBIAN_FRONTEND="noninteractive" - #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) - sudo mkdir -p /etc/needrestart - echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null + if [ "${platform_type}" == "Linux" ]; then + if [ $(checkLinuxType) = "ubuntu" ]; then + DEBIAN_FRONTEND="noninteractive" + #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) + sudo mkdir -p /etc/needrestart + echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null + fi + fi fi add_bin_path diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 901ba9891c2..dbf23c3e5ef 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -8,8 +8,42 @@ DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" SETUP_WIN_PYTHON_VERSION="3.11.0" GO_VERSION=$(cat .go-version) + export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION export DOCKER_COMPOSE_VERSION_AARCH64 export SETUP_WIN_PYTHON_VERSION export GO_VERSION + +exportVars() { + local platform_type="$(uname)" + local arch_type="$(uname -m)" + if [ "${arch_type}" == "x86_64" ]; then + case "${platform_type}" in + Linux|Darwin) + export GOX_FLAGS="-arch amd64" + export testResults="**/build/TEST*.xml" + export artifacts="**/build/TEST*.out" + ;; + MINGW*) + export GOX_FLAGS="-arch 386" + export testResults="**\\build\\TEST*.xml" + export artifacts="**\\build\\TEST*.out" + ;; + esac + elif [[ "${arch_type}" == "aarch64" || "${arch_type}" == "arm64" ]]; then + export GOX_FLAGS="-arch arm" + export testResults="**/build/TEST*.xml" + export artifacts="**/build/TEST*.out" + else + echo "Unsupported OS" + fi +} + + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then + exportVars + export RACE_DETECTOR="true" + export TEST_COVERAGE="true" + export DOCKER_PULL="0" +fi