From f7f41a2f8f79f00e41b3d42bf4d69bab1860b84d Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 24 Apr 2024 17:48:45 +0200 Subject: [PATCH 01/11] Beats DRA pipeline (#39126) * Packaging pipieline resource * Revetred agentbeat pipeline deletion * Cleanup * Test auditbeat packaging pipeline * Fix steps keys * Fix steps keys * Fix env vars * Fix env vars * Fix env vars * Unified artifacts dir * Implemented DRA steps * Test filebeat * Test matrix * Aligned artifacts directories * Aligned artifacts directories * Aligned artifacts directories * Aligned artifacts directories * Debug * Debug * Package all beats * Test DRA snapshot * Test DRA snapshot * Rename artifacts * fix dashboards artifacts * Cleanup * cleanup * No need to install msi tools * Apply suggestions from code review Co-authored-by: Dimitrios Liappis * Extraced platforms variables * Cleanup * Added RUN_SNAPSHOT condition * Cleanup * Boolean dry-run * Boolean dry-run --------- Co-authored-by: Dimitrios Liappis (cherry picked from commit c6444db54474441450921fc845a97b148bfaba71) # Conflicts: # .buildkite/packaging.pipeline.yml --- .buildkite/packaging.pipeline.md | 23 ++ .buildkite/packaging.pipeline.yml | 231 ++++++++++++++++++ .buildkite/scripts/dra.sh | 46 ++++ .buildkite/scripts/packaging/package-dra.sh | 21 ++ .../packaging/prepare-release-manager.sh | 41 ++++ 5 files changed, 362 insertions(+) create mode 100644 .buildkite/packaging.pipeline.md create mode 100644 .buildkite/packaging.pipeline.yml create mode 100755 .buildkite/scripts/dra.sh create mode 100755 .buildkite/scripts/packaging/package-dra.sh create mode 100755 .buildkite/scripts/packaging/prepare-release-manager.sh diff --git a/.buildkite/packaging.pipeline.md b/.buildkite/packaging.pipeline.md new file mode 100644 index 00000000000..577310d0104 --- /dev/null +++ b/.buildkite/packaging.pipeline.md @@ -0,0 +1,23 @@ +### Beats Packaging pipeline +[Buildkite packaging pipeline](https://buildkite.com/elastic/beats-packaging-pipeline) is used to build and publish the packages for the Beats. The pipeline is triggered by a commit to the `main` or release branches. +The pipeline definition is located in the `.buildkite/packaging.pipeline.yml` + +### Triggers +Staging packaging DRA is triggered for release branches only. +Snapshot is triggered for `main` and release branches. + +### Pipeline steps + +#### Beats dashboard +Generates `build/distributions/dependencies.csv` and `tar.gz` and adds them to the `beats-dashboards` artifact. This is required by the release-manager configuration. + +#### Packaging snapshot/staging + +- Builds the Beats packages for all supported platforms and architectures (`mage package, mage ironbank`) +- Copies artifacts `build/distributions//` directory and adds it as an artifact, where `` is the corresponding beat name. +- x-pack artifacts are also copied to `build/distributions//` directory, where `` is the name of the beat. For example, `auditbeat`, not `x-pack/auditbeat`. It's required for the DRA publish step by [release-manager configuration](https://github.com/elastic/infra/blob/master/cd/release/release-manager/project-configs/master/beats.gradle). + +#### DRA publish +Downloads the artifacts from the `packaging snapshot/staging` step and publishes them to the Elastic DRA registry. + + diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml new file mode 100644 index 00000000000..fb396328d8e --- /dev/null +++ b/.buildkite/packaging.pipeline.yml @@ -0,0 +1,231 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +# TODO: Pre-cache beats-dev/golang-crossbuild container image + +env: + ASDF_MAGE_VERSION: 1.15.0 + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + + PLATFORMS_AMD: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS_ARM: "linux/arm64" + +steps: + - group: Beats dashboards + key: dashboards + steps: + - label: Snapshot dashboards + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" + key: dashboards-snapshot + # TODO: container with go and make + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + commands: + - make build/distributions/dependencies.csv + - make beats-dashboards + env: + SNAPSHOT: true + artifact_paths: + - build/distributions/**/* + + - label: Staging dashboards + if: "build.branch =~ /\\d+\\.\\d+/" + key: dashboards-staging + # TODO: container with go and make + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + commands: + - make build/distributions/dependencies.csv + - make beats-dashboards + artifact_paths: + - build/distributions/**/* + + - group: Packaging snapshot + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" + key: packaging-snapshot + steps: + # x86 + - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + env: + PLATFORMS: "${PLATFORMS_AMD}" + SNAPSHOT: true + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - winlogbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/functionbeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/osquerybeat + - x-pack/packetbeat + - x-pack/winlogbeat + + ## ARM + - label: ":linux: {{matrix}}/Packaging Linux arm64 Snapshot" + env: + PLATFORMS: "${PLATFORMS_ARM}" + PACKAGES: "docker" + SNAPSHOT: true + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/packetbeat + + ## Agentbeat needs more CPUs because it builds many other beats + - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + env: + PLATFORMS: "${PLATFORMS_AMD}" + SNAPSHOT: true + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: + - build/distributions/**/* + matrix: + - x-pack/agentbeat + + - group: Packaging staging + + key: packaging-staging + ## Only for release + if: "build.branch =~ /\\d+\\.\\d+/" + steps: + # x86 + - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" + env: + PLATFORMS: "${PLATFORMS_AMD}" + SNAPSHOT: false + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - winlogbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/functionbeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/osquerybeat + - x-pack/packetbeat + - x-pack/winlogbeat + + ## ARM + - label: ":linux: {{matrix}}/Packaging Linux arm64 Staging" + env: + PLATFORMS: "${PLATFORMS_ARM}" + PACKAGES: "docker" + SNAPSHOT: false + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - build/distributions/** + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/packetbeat + + ## Agentbeat needs more CPUs because it builds many other beats + - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + env: + PLATFORMS: "${PLATFORMS_AMD}" + SNAPSHOT: true + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: + - build/distributions/**/* + matrix: + - x-pack/agentbeat + + - group: DRA publish + key: dra + steps: + - label: DRA Snapshot + ## Only for release branches and main + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" + key: dra-snapshot + env: + DRA_WORKFLOW: snapshot + depends_on: + - packaging-snapshot + - dashboards-snapshot + command: | + buildkite-agent artifact download "build/**/*" . + .buildkite/scripts/packaging/prepare-release-manager.sh + .buildkite/scripts/dra.sh + agents: + provider: "gcp" + + - label: DRA Staging + ## Only for release branches + if: "build.branch =~ /\\d+\\.\\d+/" + key: dra-staging + env: + DRA_WORKFLOW: staging + depends_on: + - packaging-staging + - dashboards-staging + command: | + buildkite-agent artifact download "build/**" . + .buildkite/scripts/packaging/prepare-release-manager.sh + .buildkite/scripts/dra.sh + agents: + provider: "gcp" diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh new file mode 100755 index 00000000000..ee0f4b286f0 --- /dev/null +++ b/.buildkite/scripts/dra.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +## TODO: Set to empty string when Jenkins is disabled +if [[ "$DRY_RUN" == "false" ]]; then echo "--- Running in publish mode"; DRY_RUN=""; else echo "--- Running in dry-run mode"; DRY_RUN="--dry-run"; fi +set -euo pipefail +BRANCH="${BUILDKITE_BRANCH}" + +if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ]]; then + BRANCH=main + DRY_RUN="--dry-run" + echo "+++ Running in PR and setting branch main and --dry-run" +fi + +BEAT_VERSION=$(make get-version) + +CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role" + +function release_manager_login { + DRA_CREDS_SECRET=$(retry -t 5 -- vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH}) + VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr') + VAULT_ROLE_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id') + VAULT_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id') + export VAULT_ADDR_SECRET VAULT_ROLE_ID_SECRET VAULT_SECRET +} + +release_manager_login + +echo "+++ Changing permissions for the BK API commands" +sudo chown -R :1000 build/distributions/ + +echo "+++ :hammer_and_pick: Publishing $BRANCH $DRA_WORKFLOW DRA artifacts..." +docker run --rm \ + --name release-manager \ + -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ + -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \ + -e VAULT_SECRET_ID="${VAULT_SECRET}" \ + --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \ + docker.elastic.co/infra/release-manager:latest \ + cli collect \ + --project "beats" \ + --branch "${BRANCH}" \ + --commit "${BUILDKITE_COMMIT}" \ + --workflow "${DRA_WORKFLOW}" \ + --version "${BEAT_VERSION}" \ + --artifact-set "main" \ + ${DRY_RUN} \ No newline at end of file diff --git a/.buildkite/scripts/packaging/package-dra.sh b/.buildkite/scripts/packaging/package-dra.sh new file mode 100755 index 00000000000..a70bd455278 --- /dev/null +++ b/.buildkite/scripts/packaging/package-dra.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -ueo pipefail +BEAT_DIR=${1:-""} + +if [ -z "$BEAT_DIR" ]; then + echo "Error: Beat directory must be specified." + exit 1 +fi + +echo "~~~Packaging : $BEAT_DIR" + +WORKSPACE=$(pwd) +BEAT_NAME_SLUG=$(echo "$BEAT_DIR" | sed 's/x-pack\///g') + +cd $BEAT_DIR +mage package +mage ironbank + +mkdir -p $WORKSPACE/build/distributions/$BEAT_NAME_SLUG +cp build/distributions/* $WORKSPACE/build/distributions/$BEAT_NAME_SLUG/ +cd $WORKSPACE \ No newline at end of file diff --git a/.buildkite/scripts/packaging/prepare-release-manager.sh b/.buildkite/scripts/packaging/prepare-release-manager.sh new file mode 100755 index 00000000000..2727fd986cc --- /dev/null +++ b/.buildkite/scripts/packaging/prepare-release-manager.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# +# This script is executed by the DRA stage. +# It prepares the required files to be consumed by the release-manager +# It can be published as snapshot or staging, for such you use +# the paramater $0 "snapshot" or $0 "staging" +# +set -ueo pipefail + +readonly TYPE=${1:-snapshot} + +# rename dependencies.csv to the name expected by release-manager. +VERSION=$(make get-version) +FINAL_VERSION=$VERSION-SNAPSHOT +if [ "$TYPE" != "snapshot" ] ; then + FINAL_VERSION=$VERSION +fi +echo "Rename dependencies to $FINAL_VERSION" +mv build/distributions/dependencies.csv \ + build/distributions/dependencies-"$FINAL_VERSION".csv + +# rename docker files to support the unified release format. +# TODO: this could be supported by the package system itself +# or the unified release process the one to do the transformation +# See https://github.com/elastic/beats/pull/30895 +find build/distributions -name '*linux-arm64.docker.tar.gz*' -print0 | + while IFS= read -r -d '' file + do + echo "Rename file $file" + mv "$file" "${file/linux-arm64.docker.tar.gz/docker-image-linux-arm64.tar.gz}" + done + +find build/distributions -name '*linux-amd64.docker.tar.gz*' -print0 | + while IFS= read -r -d '' file + do + echo "Rename file $file" + mv "$file" "${file/linux-amd64.docker.tar.gz/docker-image-linux-amd64.tar.gz}" + done + +echo 'List all the files' +find build/distributions -type f -ls || true From fe57c6c16834c6625f8ac65f018ae57e4429c8ca Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Thu, 25 Apr 2024 15:57:00 +0200 Subject: [PATCH 02/11] removed agentbeat, changed required platforms --- .buildkite/packaging.pipeline.yml | 32 +------------------------------ 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index fb396328d8e..bcdda0b0fd2 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -8,7 +8,7 @@ env: GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - PLATFORMS_AMD: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS_AMD: "+all linux/amd64 linux/386 linux/arm64 windows/amd64 windows/386 darwin/amd64" PLATFORMS_ARM: "linux/arm64" steps: @@ -104,21 +104,6 @@ steps: - x-pack/metricbeat - x-pack/packetbeat - ## Agentbeat needs more CPUs because it builds many other beats - - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" - env: - PLATFORMS: "${PLATFORMS_AMD}" - SNAPSHOT: true - command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" - agents: - provider: gcp - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: - - build/distributions/**/* - matrix: - - x-pack/agentbeat - - group: Packaging staging key: packaging-staging @@ -180,21 +165,6 @@ steps: - x-pack/metricbeat - x-pack/packetbeat - ## Agentbeat needs more CPUs because it builds many other beats - - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" - env: - PLATFORMS: "${PLATFORMS_AMD}" - SNAPSHOT: true - command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" - agents: - provider: gcp - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: - - build/distributions/**/* - matrix: - - x-pack/agentbeat - - group: DRA publish key: dra steps: From a7192de41f9dfd2b41d3ded0c13d1a0744418ea3 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Thu, 25 Apr 2024 16:15:45 +0300 Subject: [PATCH 03/11] Fixes for Buildkite packaging pipeline (#39207) This PR fixes a number of bugs from #39126 --- .buildkite/packaging.pipeline.yml | 46 ++++++++++----------- .buildkite/scripts/dra.sh | 18 +++++++- .buildkite/scripts/packaging/package-dra.sh | 2 +- 3 files changed, 41 insertions(+), 25 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index bcdda0b0fd2..4372658d2a1 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -8,7 +8,7 @@ env: GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - PLATFORMS_AMD: "+all linux/amd64 linux/386 linux/arm64 windows/amd64 windows/386 darwin/amd64" + PLATFORMS: "+all linux/amd64 linux/386 linux/arm64 windows/amd64 windows/386 darwin/amd64" PLATFORMS_ARM: "linux/arm64" steps: @@ -16,13 +16,13 @@ steps: key: dashboards steps: - label: Snapshot dashboards - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" + if: build.branch =~ /^\d+\.\d+$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: dashboards-snapshot # TODO: container with go and make agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" commands: - make build/distributions/dependencies.csv - make beats-dashboards @@ -32,13 +32,13 @@ steps: - build/distributions/**/* - label: Staging dashboards - if: "build.branch =~ /\\d+\\.\\d+/" + if: build.branch =~ /^\d+\.\d+$/ key: dashboards-staging # TODO: container with go and make agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" commands: - make build/distributions/dependencies.csv - make beats-dashboards @@ -46,19 +46,18 @@ steps: - build/distributions/**/* - group: Packaging snapshot - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" + if: build.branch =~ /^\d+\.\d+$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: packaging-snapshot steps: - # x86 - - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + - label: "SNAPSHOT: {{matrix}}" env: - PLATFORMS: "${PLATFORMS_AMD}" + PLATFORMS: "${PLATFORMS}" SNAPSHOT: true command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: - build/distributions/**/* matrix: @@ -78,8 +77,7 @@ steps: - x-pack/packetbeat - x-pack/winlogbeat - ## ARM - - label: ":linux: {{matrix}}/Packaging Linux arm64 Snapshot" + - label: "SNAPSHOT: {{matrix}} docker Linux/arm64" env: PLATFORMS: "${PLATFORMS_ARM}" PACKAGES: "docker" @@ -108,18 +106,17 @@ steps: key: packaging-staging ## Only for release - if: "build.branch =~ /\\d+\\.\\d+/" + if: build.branch =~ /^\d+\.\d+$/ steps: - # x86 - - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" + - label: "STAGING: {{matrix}}" env: - PLATFORMS: "${PLATFORMS_AMD}" + PLATFORMS: "${PLATFORMS}" SNAPSHOT: false command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: - build/distributions/**/* matrix: @@ -139,8 +136,7 @@ steps: - x-pack/packetbeat - x-pack/winlogbeat - ## ARM - - label: ":linux: {{matrix}}/Packaging Linux arm64 Staging" + - label: "STAGING: {{matrix}} docker Linux/arm64" env: PLATFORMS: "${PLATFORMS_ARM}" PACKAGES: "docker" @@ -170,7 +166,7 @@ steps: steps: - label: DRA Snapshot ## Only for release branches and main - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" + if: build.branch =~ /^\d+\.\d+$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: dra-snapshot env: DRA_WORKFLOW: snapshot @@ -182,11 +178,13 @@ steps: .buildkite/scripts/packaging/prepare-release-manager.sh .buildkite/scripts/dra.sh agents: - provider: "gcp" + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - label: DRA Staging ## Only for release branches - if: "build.branch =~ /\\d+\\.\\d+/" + if: build.branch =~ /^\d+\.\d+$/ key: dra-staging env: DRA_WORKFLOW: staging @@ -198,4 +196,6 @@ steps: .buildkite/scripts/packaging/prepare-release-manager.sh .buildkite/scripts/dra.sh agents: - provider: "gcp" + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index ee0f4b286f0..4957ef36199 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -28,6 +28,22 @@ release_manager_login echo "+++ Changing permissions for the BK API commands" sudo chown -R :1000 build/distributions/ +echo "+++ :hammer_and_pick: Listing $BRANCH $DRA_WORKFLOW DRA artifacts..." +docker run --rm \ + --name release-manager \ + -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ + -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \ + -e VAULT_SECRET_ID="${VAULT_SECRET}" \ + --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \ + docker.elastic.co/infra/release-manager:latest \ + cli list \ + --project "beats" \ + --branch "${BRANCH}" \ + --commit "${BUILDKITE_COMMIT}" \ + --workflow "${DRA_WORKFLOW}" \ + --version "${BEAT_VERSION}" \ + --artifact-set "main" + echo "+++ :hammer_and_pick: Publishing $BRANCH $DRA_WORKFLOW DRA artifacts..." docker run --rm \ --name release-manager \ @@ -43,4 +59,4 @@ docker run --rm \ --workflow "${DRA_WORKFLOW}" \ --version "${BEAT_VERSION}" \ --artifact-set "main" \ - ${DRY_RUN} \ No newline at end of file + ${DRY_RUN} diff --git a/.buildkite/scripts/packaging/package-dra.sh b/.buildkite/scripts/packaging/package-dra.sh index a70bd455278..10d5425da36 100755 --- a/.buildkite/scripts/packaging/package-dra.sh +++ b/.buildkite/scripts/packaging/package-dra.sh @@ -7,7 +7,7 @@ if [ -z "$BEAT_DIR" ]; then exit 1 fi -echo "~~~Packaging : $BEAT_DIR" +echo "~~~ Packaging : $BEAT_DIR" WORKSPACE=$(pwd) BEAT_NAME_SLUG=$(echo "$BEAT_DIR" | sed 's/x-pack\///g') From 684bf9d0749b7d49dce501595ffe5f74c251cacc Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Thu, 25 Apr 2024 16:38:05 +0300 Subject: [PATCH 04/11] Fix BK DRA pipeline (#39212) This commit fixes the error "interpolating remaining fields: Expected identifier to start with a letter, got /" --- .buildkite/packaging.pipeline.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 4372658d2a1..82a5593cb0c 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -16,7 +16,7 @@ steps: key: dashboards steps: - label: Snapshot dashboards - if: build.branch =~ /^\d+\.\d+$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: dashboards-snapshot # TODO: container with go and make agents: @@ -32,7 +32,7 @@ steps: - build/distributions/**/* - label: Staging dashboards - if: build.branch =~ /^\d+\.\d+$/ + if: build.branch =~ /^\d+\.\d+$$/ key: dashboards-staging # TODO: container with go and make agents: @@ -46,7 +46,7 @@ steps: - build/distributions/**/* - group: Packaging snapshot - if: build.branch =~ /^\d+\.\d+$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: packaging-snapshot steps: - label: "SNAPSHOT: {{matrix}}" @@ -106,7 +106,7 @@ steps: key: packaging-staging ## Only for release - if: build.branch =~ /^\d+\.\d+$/ + if: build.branch =~ /^\d+\.\d+$$/ steps: - label: "STAGING: {{matrix}}" env: @@ -166,7 +166,7 @@ steps: steps: - label: DRA Snapshot ## Only for release branches and main - if: build.branch =~ /^\d+\.\d+$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: dra-snapshot env: DRA_WORKFLOW: snapshot @@ -184,7 +184,7 @@ steps: - label: DRA Staging ## Only for release branches - if: build.branch =~ /^\d+\.\d+$/ + if: build.branch =~ /^\d+\.\d+$$/ key: dra-staging env: DRA_WORKFLOW: staging From 75baa69c1cc94063fdd1fab5941983cd088915ac Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Fri, 26 Apr 2024 12:10:43 +0300 Subject: [PATCH 05/11] Fix permissions for packaging (#39218) This commit fixes the release-manager error > # > java.io.FileNotFoundException: /artifacts/build/distributions/... (Permission denied) by adjusting the permissions of parent directories. Additionally there are a few logging enhancements to improve debugging, as well as the introduction of a new optional `DRA_BRANCH` environment variable, to help running and debugging via PRs. Relates https://github.com/elastic/ingest-dev/issues/3095 --------- Signed-off-by: Alexandros Sapranidis Co-authored-by: Dimitrios Liappis --- .buildkite/scripts/dra.sh | 52 ++++++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index 4957ef36199..b20d6c6a696 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -1,16 +1,29 @@ #!/usr/bin/env bash -## TODO: Set to empty string when Jenkins is disabled -if [[ "$DRY_RUN" == "false" ]]; then echo "--- Running in publish mode"; DRY_RUN=""; else echo "--- Running in dry-run mode"; DRY_RUN="--dry-run"; fi -set -euo pipefail -BRANCH="${BUILDKITE_BRANCH}" +# TODO: uncomment out below when Jenkins packaging has been stopped +# if [[ "$DRY_RUN" == "true" ]]; then +# echo "~~~ Running in dry-run mode -- will NOT publish artifacts" +# DRY_RUN="--dry-run" +# else +# echo "~~~ Running in publish mode" +# DRY_RUN="" +# fi -if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ]]; then - BRANCH=main +# TODO: delete the conditional below (and replace it with the above, uncommented out, section) after Jenkins packaging has been stopped +if [[ "$DRY_RUN" == "false" ]]; then + echo "~~~ Running in publish mode" + DRY_RUN="" +else + echo "~~~ Running in dry-run mode -- will NOT publish artifacts" DRY_RUN="--dry-run" - echo "+++ Running in PR and setting branch main and --dry-run" fi +set -euo pipefail + +# DRA_BRANCH can be used for manually testing packaging with PRs +# e.g. define `DRA_BRANCH="main"` and `RUN_SNAPSHOT="true"` under Options/Environment Variables in the Buildkite UI after clicking new Build +BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}" + BEAT_VERSION=$(make get-version) CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role" @@ -23,12 +36,16 @@ function release_manager_login { export VAULT_ADDR_SECRET VAULT_ROLE_ID_SECRET VAULT_SECRET } +set +x release_manager_login -echo "+++ Changing permissions for the BK API commands" -sudo chown -R :1000 build/distributions/ +# required by the release-manager docker image, otherwise we hit: +# > java.io.FileNotFoundException: /artifacts/build/distributions/agentbeat/agentbeat-8.15.0-SNAPSHOT-darwin-x86_64.tar.gz.sha512 (Permission denied) +chmod -R a+r build/* +chmod -R a+w build -echo "+++ :hammer_and_pick: Listing $BRANCH $DRA_WORKFLOW DRA artifacts..." +echo "+++ :clipboard: Listing DRA artifacts for version [$BEAT_VERSION], branch [$BRANCH] and workflow [$DRA_WORKFLOW]" +set +x docker run --rm \ --name release-manager \ -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ @@ -44,7 +61,9 @@ docker run --rm \ --version "${BEAT_VERSION}" \ --artifact-set "main" -echo "+++ :hammer_and_pick: Publishing $BRANCH $DRA_WORKFLOW DRA artifacts..." +echo "+++ :hammer_and_pick: Publishing DRA artifacts for version [$BEAT_VERSION], branch [$BRANCH], workflow [$DRA_WORKFLOW] and DRY_RUN: [$DRY_RUN]" + +set +x docker run --rm \ --name release-manager \ -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ @@ -59,4 +78,13 @@ docker run --rm \ --workflow "${DRA_WORKFLOW}" \ --version "${BEAT_VERSION}" \ --artifact-set "main" \ - ${DRY_RUN} + ${DRY_RUN} | tee rm-output.txt + +# extract the summary URL from a release manager output line like: +# Report summary-18.22.0.html can be found at https://artifacts-staging.elastic.co/beats/18.22.0-ABCDEFGH/summary-18.22.0.html + +SUMMARY_URL=$(grep -E '^Report summary-.* can be found at ' rm-output.txt | grep -oP 'https://\S+' | awk '{print $1}') +rm rm-output.txt + +# and make it easily clickable as a Builkite annotation +printf "**Summary link:** [${SUMMARY_URL}](${SUMMARY_URL})\n" | buildkite-agent annotate --style=success From d67ccbfabc5f5d7a4ebfb6d3f65f02f9db016199 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 26 Apr 2024 11:24:15 +0200 Subject: [PATCH 06/11] Added DEV flag to snapshot. prepare-release-manager.sh staging param (#39217) Added DEV flag to snapshot builds. Specified the workflow for prepare-release-manager.sh Relates: https://github.com/elastic/ingest-dev/issues/3095 --- .buildkite/packaging.pipeline.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 82a5593cb0c..fae197eb838 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -28,6 +28,7 @@ steps: - make beats-dashboards env: SNAPSHOT: true + DEV: true artifact_paths: - build/distributions/**/* @@ -42,6 +43,9 @@ steps: commands: - make build/distributions/dependencies.csv - make beats-dashboards + env: + SNAPSHOT: false + DEV: false artifact_paths: - build/distributions/**/* @@ -53,6 +57,7 @@ steps: env: PLATFORMS: "${PLATFORMS}" SNAPSHOT: true + DEV: true command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: gcp @@ -82,6 +87,7 @@ steps: PLATFORMS: "${PLATFORMS_ARM}" PACKAGES: "docker" SNAPSHOT: true + DEV: true command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: "aws" @@ -103,7 +109,6 @@ steps: - x-pack/packetbeat - group: Packaging staging - key: packaging-staging ## Only for release if: build.branch =~ /^\d+\.\d+$$/ @@ -112,6 +117,7 @@ steps: env: PLATFORMS: "${PLATFORMS}" SNAPSHOT: false + DEV: false command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: gcp @@ -141,6 +147,7 @@ steps: PLATFORMS: "${PLATFORMS_ARM}" PACKAGES: "docker" SNAPSHOT: false + DEV: false command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: "aws" @@ -175,7 +182,7 @@ steps: - dashboards-snapshot command: | buildkite-agent artifact download "build/**/*" . - .buildkite/scripts/packaging/prepare-release-manager.sh + .buildkite/scripts/packaging/prepare-release-manager.sh snapshot .buildkite/scripts/dra.sh agents: provider: gcp @@ -193,7 +200,7 @@ steps: - dashboards-staging command: | buildkite-agent artifact download "build/**" . - .buildkite/scripts/packaging/prepare-release-manager.sh + .buildkite/scripts/packaging/prepare-release-manager.sh staging .buildkite/scripts/dra.sh agents: provider: gcp From 7173d85bf4e04488c78efcb627ca7ae4031d7ed2 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 30 Apr 2024 10:27:44 +0200 Subject: [PATCH 07/11] DRA: Disable summary reports for dry runs (#39240) --- .buildkite/scripts/dra.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index b20d6c6a696..e3bc62202fc 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -80,11 +80,13 @@ docker run --rm \ --artifact-set "main" \ ${DRY_RUN} | tee rm-output.txt -# extract the summary URL from a release manager output line like: -# Report summary-18.22.0.html can be found at https://artifacts-staging.elastic.co/beats/18.22.0-ABCDEFGH/summary-18.22.0.html -SUMMARY_URL=$(grep -E '^Report summary-.* can be found at ' rm-output.txt | grep -oP 'https://\S+' | awk '{print $1}') -rm rm-output.txt +if [[ "$DRY_RUN" != "--dry-run" ]]; then + # extract the summary URL from a release manager output line like: + # Report summary-18.22.0.html can be found at https://artifacts-staging.elastic.co/beats/18.22.0-ABCDEFGH/summary-18.22.0.html + SUMMARY_URL=$(grep -E '^Report summary-.* can be found at ' rm-output.txt | grep -oP 'https://\S+' | awk '{print $1}') + rm rm-output.txt -# and make it easily clickable as a Builkite annotation -printf "**Summary link:** [${SUMMARY_URL}](${SUMMARY_URL})\n" | buildkite-agent annotate --style=success + # and make it easily clickable as a Builkite annotation + printf "**Summary link:** [${SUMMARY_URL}](${SUMMARY_URL})\n" | buildkite-agent annotate --style=success +fi From 0fdba87cf8382266aba103223416047e159e349d Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Tue, 30 Apr 2024 11:36:42 +0300 Subject: [PATCH 08/11] Ensure ordered DRA artifacts (#39270) As things are now we allow parallel builds on the packaging pipeline, which could result in out of order artifacts (depending on which one takes longer to finish). This commit implements two "queues" (snapshot/staging) to ensure ordered builds of DRA artifacts. Relates https://github.com/elastic/ingest-dev/issues/3095 --- .buildkite/packaging.pipeline.yml | 40 +++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index fae197eb838..6d596fa568a 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -12,11 +12,32 @@ env: PLATFORMS_ARM: "linux/arm64" steps: + # we use concurrency gates (https://buildkite.com/blog/concurrency-gates) + # to implement two FIFO queues for DRA-snapshot and DRA-staging + # this prevents parallel builds and possibility of publishing out of order DRA artifacts if the first job takes longer than the second + + - name: Start of concurrency group for DRA Snapshot + if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + command: echo "--> Start of concurrency gate dra-snapshot" + concurrency_group: "dra-gate-snapshot" + concurrency: 1 + key: start-gate-snapshot + + - name: Start of concurrency group for DRA Staging + if: build.branch =~ /^\d+\.\d+$$/ + command: echo "--> Start of concurrency gate dra-staging" + concurrency_group: "dra-gate-staging" + concurrency: 1 + key: start-gate-staging + + - wait + - group: Beats dashboards key: dashboards steps: - label: Snapshot dashboards if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + depends_on: start-gate-snapshot key: dashboards-snapshot # TODO: container with go and make agents: @@ -34,6 +55,7 @@ steps: - label: Staging dashboards if: build.branch =~ /^\d+\.\d+$$/ + depends_on: start-gate-staging key: dashboards-staging # TODO: container with go and make agents: @@ -52,6 +74,7 @@ steps: - group: Packaging snapshot if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: packaging-snapshot + depends_on: start-gate-snapshot steps: - label: "SNAPSHOT: {{matrix}}" env: @@ -110,6 +133,7 @@ steps: - group: Packaging staging key: packaging-staging + depends_on: start-gate-staging ## Only for release if: build.branch =~ /^\d+\.\d+$$/ steps: @@ -178,6 +202,7 @@ steps: env: DRA_WORKFLOW: snapshot depends_on: + - start-gate-snapshot - packaging-snapshot - dashboards-snapshot command: | @@ -196,6 +221,7 @@ steps: env: DRA_WORKFLOW: staging depends_on: + - start-gate-staging - packaging-staging - dashboards-staging command: | @@ -206,3 +232,17 @@ steps: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + + - wait + + - command: echo "End of concurrency gate dra-snapshot <--" + if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + concurrency_group: "dra-gate-snapshot" + concurrency: 1 + key: end-gate-snapshot + + - command: echo "End of concurrency gate dra-staging <--" + if: build.branch =~ /^\d+\.\d+$$/ + concurrency_group: "dra-gate-staging" + concurrency: 1 + key: end-gate-staging From b774278504a7534ff3aca843a236184eaada5764 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Tue, 30 Apr 2024 14:41:35 +0300 Subject: [PATCH 09/11] Fix missing docker staging DRA artifacts (#39297) The DRA staging release is failing because the Buildkite step isn't capturing the right artifacts. This commit fixes the issue by adjusting the artifact_paths to match the other steps. --- .buildkite/packaging.pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 6d596fa568a..76b42b65e47 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -178,7 +178,7 @@ steps: imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: - - build/distributions/** + - build/distributions/**/* matrix: - auditbeat - filebeat From b6f7a902eecb0320311ad0fc1cddcd3c3e1246bc Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 30 Apr 2024 14:58:10 +0300 Subject: [PATCH 10/11] Fix the annotating for snapshot and staging (#39299) * Fix the annotating for snapshot and staging This commit adds the `--append` flag to the buildkite-annotate so that when it is called by the snapshot and staging steps, it will not overwrite the other annotation. Signed-off-by: Alexandros Sapranidis --- .buildkite/scripts/dra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index e3bc62202fc..aeee80908a7 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -88,5 +88,5 @@ if [[ "$DRY_RUN" != "--dry-run" ]]; then rm rm-output.txt # and make it easily clickable as a Builkite annotation - printf "**Summary link:** [${SUMMARY_URL}](${SUMMARY_URL})\n" | buildkite-agent annotate --style=success + printf "**${DRA_WORKFLOW} summary link:** [${SUMMARY_URL}](${SUMMARY_URL})\n" | buildkite-agent annotate --style=success --append fi From a4c31fc3b8b69cc900d203ff3d1ad8b7643a5eef Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Tue, 30 Apr 2024 14:53:42 +0300 Subject: [PATCH 11/11] Branch specific concurrency gates (#39298) PR #39293 introduced one concurrency queue per staging/snapshot but this slows down unnecessarily concurrent DRA builds for main and other release branches. This commit makes the concurrency gates (additionally) specific per branch. --- .buildkite/packaging.pipeline.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 76b42b65e47..fc5197c4314 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -19,14 +19,14 @@ steps: - name: Start of concurrency group for DRA Snapshot if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" command: echo "--> Start of concurrency gate dra-snapshot" - concurrency_group: "dra-gate-snapshot" + concurrency_group: "dra-gate-snapshot-$BUILDKITE_BRANCH" concurrency: 1 key: start-gate-snapshot - name: Start of concurrency group for DRA Staging if: build.branch =~ /^\d+\.\d+$$/ command: echo "--> Start of concurrency gate dra-staging" - concurrency_group: "dra-gate-staging" + concurrency_group: "dra-gate-staging-$BUILDKITE_BRANCH" concurrency: 1 key: start-gate-staging @@ -237,12 +237,12 @@ steps: - command: echo "End of concurrency gate dra-snapshot <--" if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" - concurrency_group: "dra-gate-snapshot" + concurrency_group: "dra-gate-snapshot-$BUILDKITE_BRANCH" concurrency: 1 key: end-gate-snapshot - command: echo "End of concurrency gate dra-staging <--" if: build.branch =~ /^\d+\.\d+$$/ - concurrency_group: "dra-gate-staging" + concurrency_group: "dra-gate-staging-$BUILDKITE_BRANCH" concurrency: 1 key: end-gate-staging