From a0d3f9fa63296cdd3ec1ff1ea4d1df0de60b0376 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 24 Apr 2024 17:48:45 +0200 Subject: [PATCH 01/12] Beats DRA pipeline (#39126) * Packaging pipieline resource * Revetred agentbeat pipeline deletion * Cleanup * Test auditbeat packaging pipeline * Fix steps keys * Fix steps keys * Fix env vars * Fix env vars * Fix env vars * Unified artifacts dir * Implemented DRA steps * Test filebeat * Test matrix * Aligned artifacts directories * Aligned artifacts directories * Aligned artifacts directories * Aligned artifacts directories * Debug * Debug * Package all beats * Test DRA snapshot * Test DRA snapshot * Rename artifacts * fix dashboards artifacts * Cleanup * cleanup * No need to install msi tools * Apply suggestions from code review Co-authored-by: Dimitrios Liappis * Extraced platforms variables * Cleanup * Added RUN_SNAPSHOT condition * Cleanup * Boolean dry-run * Boolean dry-run --------- Co-authored-by: Dimitrios Liappis (cherry picked from commit c6444db54474441450921fc845a97b148bfaba71) # Conflicts: # .buildkite/packaging.pipeline.yml --- .buildkite/packaging.pipeline.md | 23 ++ .buildkite/packaging.pipeline.yml | 231 ++++++++++++++++++ .buildkite/scripts/dra.sh | 46 ++++ .buildkite/scripts/packaging/package-dra.sh | 21 ++ .../packaging/prepare-release-manager.sh | 41 ++++ 5 files changed, 362 insertions(+) create mode 100644 .buildkite/packaging.pipeline.md create mode 100644 .buildkite/packaging.pipeline.yml create mode 100755 .buildkite/scripts/dra.sh create mode 100755 .buildkite/scripts/packaging/package-dra.sh create mode 100755 .buildkite/scripts/packaging/prepare-release-manager.sh diff --git a/.buildkite/packaging.pipeline.md b/.buildkite/packaging.pipeline.md new file mode 100644 index 00000000000..577310d0104 --- /dev/null +++ b/.buildkite/packaging.pipeline.md @@ -0,0 +1,23 @@ +### Beats Packaging pipeline +[Buildkite packaging pipeline](https://buildkite.com/elastic/beats-packaging-pipeline) is used to build and publish the packages for the Beats. The pipeline is triggered by a commit to the `main` or release branches. +The pipeline definition is located in the `.buildkite/packaging.pipeline.yml` + +### Triggers +Staging packaging DRA is triggered for release branches only. +Snapshot is triggered for `main` and release branches. + +### Pipeline steps + +#### Beats dashboard +Generates `build/distributions/dependencies.csv` and `tar.gz` and adds them to the `beats-dashboards` artifact. This is required by the release-manager configuration. + +#### Packaging snapshot/staging + +- Builds the Beats packages for all supported platforms and architectures (`mage package, mage ironbank`) +- Copies artifacts `build/distributions//` directory and adds it as an artifact, where `` is the corresponding beat name. +- x-pack artifacts are also copied to `build/distributions//` directory, where `` is the name of the beat. For example, `auditbeat`, not `x-pack/auditbeat`. It's required for the DRA publish step by [release-manager configuration](https://github.com/elastic/infra/blob/master/cd/release/release-manager/project-configs/master/beats.gradle). + +#### DRA publish +Downloads the artifacts from the `packaging snapshot/staging` step and publishes them to the Elastic DRA registry. + + diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml new file mode 100644 index 00000000000..fb396328d8e --- /dev/null +++ b/.buildkite/packaging.pipeline.yml @@ -0,0 +1,231 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +# TODO: Pre-cache beats-dev/golang-crossbuild container image + +env: + ASDF_MAGE_VERSION: 1.15.0 + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + + PLATFORMS_AMD: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS_ARM: "linux/arm64" + +steps: + - group: Beats dashboards + key: dashboards + steps: + - label: Snapshot dashboards + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" + key: dashboards-snapshot + # TODO: container with go and make + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + commands: + - make build/distributions/dependencies.csv + - make beats-dashboards + env: + SNAPSHOT: true + artifact_paths: + - build/distributions/**/* + + - label: Staging dashboards + if: "build.branch =~ /\\d+\\.\\d+/" + key: dashboards-staging + # TODO: container with go and make + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + commands: + - make build/distributions/dependencies.csv + - make beats-dashboards + artifact_paths: + - build/distributions/**/* + + - group: Packaging snapshot + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" + key: packaging-snapshot + steps: + # x86 + - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + env: + PLATFORMS: "${PLATFORMS_AMD}" + SNAPSHOT: true + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - winlogbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/functionbeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/osquerybeat + - x-pack/packetbeat + - x-pack/winlogbeat + + ## ARM + - label: ":linux: {{matrix}}/Packaging Linux arm64 Snapshot" + env: + PLATFORMS: "${PLATFORMS_ARM}" + PACKAGES: "docker" + SNAPSHOT: true + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/packetbeat + + ## Agentbeat needs more CPUs because it builds many other beats + - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + env: + PLATFORMS: "${PLATFORMS_AMD}" + SNAPSHOT: true + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: + - build/distributions/**/* + matrix: + - x-pack/agentbeat + + - group: Packaging staging + + key: packaging-staging + ## Only for release + if: "build.branch =~ /\\d+\\.\\d+/" + steps: + # x86 + - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" + env: + PLATFORMS: "${PLATFORMS_AMD}" + SNAPSHOT: false + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - winlogbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/functionbeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/osquerybeat + - x-pack/packetbeat + - x-pack/winlogbeat + + ## ARM + - label: ":linux: {{matrix}}/Packaging Linux arm64 Staging" + env: + PLATFORMS: "${PLATFORMS_ARM}" + PACKAGES: "docker" + SNAPSHOT: false + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - build/distributions/** + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/packetbeat + + ## Agentbeat needs more CPUs because it builds many other beats + - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + env: + PLATFORMS: "${PLATFORMS_AMD}" + SNAPSHOT: true + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: + - build/distributions/**/* + matrix: + - x-pack/agentbeat + + - group: DRA publish + key: dra + steps: + - label: DRA Snapshot + ## Only for release branches and main + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" + key: dra-snapshot + env: + DRA_WORKFLOW: snapshot + depends_on: + - packaging-snapshot + - dashboards-snapshot + command: | + buildkite-agent artifact download "build/**/*" . + .buildkite/scripts/packaging/prepare-release-manager.sh + .buildkite/scripts/dra.sh + agents: + provider: "gcp" + + - label: DRA Staging + ## Only for release branches + if: "build.branch =~ /\\d+\\.\\d+/" + key: dra-staging + env: + DRA_WORKFLOW: staging + depends_on: + - packaging-staging + - dashboards-staging + command: | + buildkite-agent artifact download "build/**" . + .buildkite/scripts/packaging/prepare-release-manager.sh + .buildkite/scripts/dra.sh + agents: + provider: "gcp" diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh new file mode 100755 index 00000000000..ee0f4b286f0 --- /dev/null +++ b/.buildkite/scripts/dra.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +## TODO: Set to empty string when Jenkins is disabled +if [[ "$DRY_RUN" == "false" ]]; then echo "--- Running in publish mode"; DRY_RUN=""; else echo "--- Running in dry-run mode"; DRY_RUN="--dry-run"; fi +set -euo pipefail +BRANCH="${BUILDKITE_BRANCH}" + +if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ]]; then + BRANCH=main + DRY_RUN="--dry-run" + echo "+++ Running in PR and setting branch main and --dry-run" +fi + +BEAT_VERSION=$(make get-version) + +CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role" + +function release_manager_login { + DRA_CREDS_SECRET=$(retry -t 5 -- vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH}) + VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr') + VAULT_ROLE_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id') + VAULT_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id') + export VAULT_ADDR_SECRET VAULT_ROLE_ID_SECRET VAULT_SECRET +} + +release_manager_login + +echo "+++ Changing permissions for the BK API commands" +sudo chown -R :1000 build/distributions/ + +echo "+++ :hammer_and_pick: Publishing $BRANCH $DRA_WORKFLOW DRA artifacts..." +docker run --rm \ + --name release-manager \ + -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ + -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \ + -e VAULT_SECRET_ID="${VAULT_SECRET}" \ + --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \ + docker.elastic.co/infra/release-manager:latest \ + cli collect \ + --project "beats" \ + --branch "${BRANCH}" \ + --commit "${BUILDKITE_COMMIT}" \ + --workflow "${DRA_WORKFLOW}" \ + --version "${BEAT_VERSION}" \ + --artifact-set "main" \ + ${DRY_RUN} \ No newline at end of file diff --git a/.buildkite/scripts/packaging/package-dra.sh b/.buildkite/scripts/packaging/package-dra.sh new file mode 100755 index 00000000000..a70bd455278 --- /dev/null +++ b/.buildkite/scripts/packaging/package-dra.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -ueo pipefail +BEAT_DIR=${1:-""} + +if [ -z "$BEAT_DIR" ]; then + echo "Error: Beat directory must be specified." + exit 1 +fi + +echo "~~~Packaging : $BEAT_DIR" + +WORKSPACE=$(pwd) +BEAT_NAME_SLUG=$(echo "$BEAT_DIR" | sed 's/x-pack\///g') + +cd $BEAT_DIR +mage package +mage ironbank + +mkdir -p $WORKSPACE/build/distributions/$BEAT_NAME_SLUG +cp build/distributions/* $WORKSPACE/build/distributions/$BEAT_NAME_SLUG/ +cd $WORKSPACE \ No newline at end of file diff --git a/.buildkite/scripts/packaging/prepare-release-manager.sh b/.buildkite/scripts/packaging/prepare-release-manager.sh new file mode 100755 index 00000000000..2727fd986cc --- /dev/null +++ b/.buildkite/scripts/packaging/prepare-release-manager.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# +# This script is executed by the DRA stage. +# It prepares the required files to be consumed by the release-manager +# It can be published as snapshot or staging, for such you use +# the paramater $0 "snapshot" or $0 "staging" +# +set -ueo pipefail + +readonly TYPE=${1:-snapshot} + +# rename dependencies.csv to the name expected by release-manager. +VERSION=$(make get-version) +FINAL_VERSION=$VERSION-SNAPSHOT +if [ "$TYPE" != "snapshot" ] ; then + FINAL_VERSION=$VERSION +fi +echo "Rename dependencies to $FINAL_VERSION" +mv build/distributions/dependencies.csv \ + build/distributions/dependencies-"$FINAL_VERSION".csv + +# rename docker files to support the unified release format. +# TODO: this could be supported by the package system itself +# or the unified release process the one to do the transformation +# See https://github.com/elastic/beats/pull/30895 +find build/distributions -name '*linux-arm64.docker.tar.gz*' -print0 | + while IFS= read -r -d '' file + do + echo "Rename file $file" + mv "$file" "${file/linux-arm64.docker.tar.gz/docker-image-linux-arm64.tar.gz}" + done + +find build/distributions -name '*linux-amd64.docker.tar.gz*' -print0 | + while IFS= read -r -d '' file + do + echo "Rename file $file" + mv "$file" "${file/linux-amd64.docker.tar.gz/docker-image-linux-amd64.tar.gz}" + done + +echo 'List all the files' +find build/distributions -type f -ls || true From d066eaf36ea40b520278465c477501df2a83c9d7 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Thu, 25 Apr 2024 15:36:22 +0200 Subject: [PATCH 02/12] Removed agentbeat steps --- .buildkite/packaging.pipeline.yml | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index fb396328d8e..c1d94ab5ca5 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -104,21 +104,6 @@ steps: - x-pack/metricbeat - x-pack/packetbeat - ## Agentbeat needs more CPUs because it builds many other beats - - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" - env: - PLATFORMS: "${PLATFORMS_AMD}" - SNAPSHOT: true - command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" - agents: - provider: gcp - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: - - build/distributions/**/* - matrix: - - x-pack/agentbeat - - group: Packaging staging key: packaging-staging @@ -180,21 +165,6 @@ steps: - x-pack/metricbeat - x-pack/packetbeat - ## Agentbeat needs more CPUs because it builds many other beats - - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" - env: - PLATFORMS: "${PLATFORMS_AMD}" - SNAPSHOT: true - command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" - agents: - provider: gcp - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: - - build/distributions/**/* - matrix: - - x-pack/agentbeat - - group: DRA publish key: dra steps: From 8616c4ce69bdf17e0840afc56e05306be8a3d76b Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Thu, 25 Apr 2024 16:15:45 +0300 Subject: [PATCH 03/12] Fixes for Buildkite packaging pipeline (#39207) This PR fixes a number of bugs from #39126 --- .buildkite/packaging.pipeline.yml | 46 ++++++++++----------- .buildkite/scripts/dra.sh | 18 +++++++- .buildkite/scripts/packaging/package-dra.sh | 2 +- 3 files changed, 41 insertions(+), 25 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index c1d94ab5ca5..0ab265e235a 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -8,7 +8,7 @@ env: GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - PLATFORMS_AMD: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" PLATFORMS_ARM: "linux/arm64" steps: @@ -16,13 +16,13 @@ steps: key: dashboards steps: - label: Snapshot dashboards - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" + if: build.branch =~ /^\d+\.\d+$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: dashboards-snapshot # TODO: container with go and make agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" commands: - make build/distributions/dependencies.csv - make beats-dashboards @@ -32,13 +32,13 @@ steps: - build/distributions/**/* - label: Staging dashboards - if: "build.branch =~ /\\d+\\.\\d+/" + if: build.branch =~ /^\d+\.\d+$/ key: dashboards-staging # TODO: container with go and make agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" commands: - make build/distributions/dependencies.csv - make beats-dashboards @@ -46,19 +46,18 @@ steps: - build/distributions/**/* - group: Packaging snapshot - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" + if: build.branch =~ /^\d+\.\d+$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: packaging-snapshot steps: - # x86 - - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + - label: "SNAPSHOT: {{matrix}}" env: - PLATFORMS: "${PLATFORMS_AMD}" + PLATFORMS: "${PLATFORMS}" SNAPSHOT: true command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: - build/distributions/**/* matrix: @@ -78,8 +77,7 @@ steps: - x-pack/packetbeat - x-pack/winlogbeat - ## ARM - - label: ":linux: {{matrix}}/Packaging Linux arm64 Snapshot" + - label: "SNAPSHOT: {{matrix}} docker Linux/arm64" env: PLATFORMS: "${PLATFORMS_ARM}" PACKAGES: "docker" @@ -108,18 +106,17 @@ steps: key: packaging-staging ## Only for release - if: "build.branch =~ /\\d+\\.\\d+/" + if: build.branch =~ /^\d+\.\d+$/ steps: - # x86 - - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" + - label: "STAGING: {{matrix}}" env: - PLATFORMS: "${PLATFORMS_AMD}" + PLATFORMS: "${PLATFORMS}" SNAPSHOT: false command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: - build/distributions/**/* matrix: @@ -139,8 +136,7 @@ steps: - x-pack/packetbeat - x-pack/winlogbeat - ## ARM - - label: ":linux: {{matrix}}/Packaging Linux arm64 Staging" + - label: "STAGING: {{matrix}} docker Linux/arm64" env: PLATFORMS: "${PLATFORMS_ARM}" PACKAGES: "docker" @@ -170,7 +166,7 @@ steps: steps: - label: DRA Snapshot ## Only for release branches and main - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" + if: build.branch =~ /^\d+\.\d+$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: dra-snapshot env: DRA_WORKFLOW: snapshot @@ -182,11 +178,13 @@ steps: .buildkite/scripts/packaging/prepare-release-manager.sh .buildkite/scripts/dra.sh agents: - provider: "gcp" + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - label: DRA Staging ## Only for release branches - if: "build.branch =~ /\\d+\\.\\d+/" + if: build.branch =~ /^\d+\.\d+$/ key: dra-staging env: DRA_WORKFLOW: staging @@ -198,4 +196,6 @@ steps: .buildkite/scripts/packaging/prepare-release-manager.sh .buildkite/scripts/dra.sh agents: - provider: "gcp" + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index ee0f4b286f0..4957ef36199 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -28,6 +28,22 @@ release_manager_login echo "+++ Changing permissions for the BK API commands" sudo chown -R :1000 build/distributions/ +echo "+++ :hammer_and_pick: Listing $BRANCH $DRA_WORKFLOW DRA artifacts..." +docker run --rm \ + --name release-manager \ + -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ + -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \ + -e VAULT_SECRET_ID="${VAULT_SECRET}" \ + --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \ + docker.elastic.co/infra/release-manager:latest \ + cli list \ + --project "beats" \ + --branch "${BRANCH}" \ + --commit "${BUILDKITE_COMMIT}" \ + --workflow "${DRA_WORKFLOW}" \ + --version "${BEAT_VERSION}" \ + --artifact-set "main" + echo "+++ :hammer_and_pick: Publishing $BRANCH $DRA_WORKFLOW DRA artifacts..." docker run --rm \ --name release-manager \ @@ -43,4 +59,4 @@ docker run --rm \ --workflow "${DRA_WORKFLOW}" \ --version "${BEAT_VERSION}" \ --artifact-set "main" \ - ${DRY_RUN} \ No newline at end of file + ${DRY_RUN} diff --git a/.buildkite/scripts/packaging/package-dra.sh b/.buildkite/scripts/packaging/package-dra.sh index a70bd455278..10d5425da36 100755 --- a/.buildkite/scripts/packaging/package-dra.sh +++ b/.buildkite/scripts/packaging/package-dra.sh @@ -7,7 +7,7 @@ if [ -z "$BEAT_DIR" ]; then exit 1 fi -echo "~~~Packaging : $BEAT_DIR" +echo "~~~ Packaging : $BEAT_DIR" WORKSPACE=$(pwd) BEAT_NAME_SLUG=$(echo "$BEAT_DIR" | sed 's/x-pack\///g') From 4d60a5f0c4a47fbfec3664144c12dc73fa992473 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Thu, 25 Apr 2024 16:38:05 +0300 Subject: [PATCH 04/12] Fix BK DRA pipeline (#39212) This commit fixes the error "interpolating remaining fields: Expected identifier to start with a letter, got /" --- .buildkite/packaging.pipeline.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 0ab265e235a..2a7bef6b89e 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -16,7 +16,7 @@ steps: key: dashboards steps: - label: Snapshot dashboards - if: build.branch =~ /^\d+\.\d+$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: dashboards-snapshot # TODO: container with go and make agents: @@ -32,7 +32,7 @@ steps: - build/distributions/**/* - label: Staging dashboards - if: build.branch =~ /^\d+\.\d+$/ + if: build.branch =~ /^\d+\.\d+$$/ key: dashboards-staging # TODO: container with go and make agents: @@ -46,7 +46,7 @@ steps: - build/distributions/**/* - group: Packaging snapshot - if: build.branch =~ /^\d+\.\d+$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: packaging-snapshot steps: - label: "SNAPSHOT: {{matrix}}" @@ -106,7 +106,7 @@ steps: key: packaging-staging ## Only for release - if: build.branch =~ /^\d+\.\d+$/ + if: build.branch =~ /^\d+\.\d+$$/ steps: - label: "STAGING: {{matrix}}" env: @@ -166,7 +166,7 @@ steps: steps: - label: DRA Snapshot ## Only for release branches and main - if: build.branch =~ /^\d+\.\d+$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: dra-snapshot env: DRA_WORKFLOW: snapshot @@ -184,7 +184,7 @@ steps: - label: DRA Staging ## Only for release branches - if: build.branch =~ /^\d+\.\d+$/ + if: build.branch =~ /^\d+\.\d+$$/ key: dra-staging env: DRA_WORKFLOW: staging From 191d8b42ad07dd82ce0c8d92462bc40a3c5db6a2 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Fri, 26 Apr 2024 12:10:43 +0300 Subject: [PATCH 05/12] Fix permissions for packaging (#39218) This commit fixes the release-manager error > # > java.io.FileNotFoundException: /artifacts/build/distributions/... (Permission denied) by adjusting the permissions of parent directories. Additionally there are a few logging enhancements to improve debugging, as well as the introduction of a new optional `DRA_BRANCH` environment variable, to help running and debugging via PRs. Relates https://github.com/elastic/ingest-dev/issues/3095 --------- Signed-off-by: Alexandros Sapranidis Co-authored-by: Dimitrios Liappis --- .buildkite/scripts/dra.sh | 52 ++++++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index 4957ef36199..b20d6c6a696 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -1,16 +1,29 @@ #!/usr/bin/env bash -## TODO: Set to empty string when Jenkins is disabled -if [[ "$DRY_RUN" == "false" ]]; then echo "--- Running in publish mode"; DRY_RUN=""; else echo "--- Running in dry-run mode"; DRY_RUN="--dry-run"; fi -set -euo pipefail -BRANCH="${BUILDKITE_BRANCH}" +# TODO: uncomment out below when Jenkins packaging has been stopped +# if [[ "$DRY_RUN" == "true" ]]; then +# echo "~~~ Running in dry-run mode -- will NOT publish artifacts" +# DRY_RUN="--dry-run" +# else +# echo "~~~ Running in publish mode" +# DRY_RUN="" +# fi -if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ]]; then - BRANCH=main +# TODO: delete the conditional below (and replace it with the above, uncommented out, section) after Jenkins packaging has been stopped +if [[ "$DRY_RUN" == "false" ]]; then + echo "~~~ Running in publish mode" + DRY_RUN="" +else + echo "~~~ Running in dry-run mode -- will NOT publish artifacts" DRY_RUN="--dry-run" - echo "+++ Running in PR and setting branch main and --dry-run" fi +set -euo pipefail + +# DRA_BRANCH can be used for manually testing packaging with PRs +# e.g. define `DRA_BRANCH="main"` and `RUN_SNAPSHOT="true"` under Options/Environment Variables in the Buildkite UI after clicking new Build +BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}" + BEAT_VERSION=$(make get-version) CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role" @@ -23,12 +36,16 @@ function release_manager_login { export VAULT_ADDR_SECRET VAULT_ROLE_ID_SECRET VAULT_SECRET } +set +x release_manager_login -echo "+++ Changing permissions for the BK API commands" -sudo chown -R :1000 build/distributions/ +# required by the release-manager docker image, otherwise we hit: +# > java.io.FileNotFoundException: /artifacts/build/distributions/agentbeat/agentbeat-8.15.0-SNAPSHOT-darwin-x86_64.tar.gz.sha512 (Permission denied) +chmod -R a+r build/* +chmod -R a+w build -echo "+++ :hammer_and_pick: Listing $BRANCH $DRA_WORKFLOW DRA artifacts..." +echo "+++ :clipboard: Listing DRA artifacts for version [$BEAT_VERSION], branch [$BRANCH] and workflow [$DRA_WORKFLOW]" +set +x docker run --rm \ --name release-manager \ -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ @@ -44,7 +61,9 @@ docker run --rm \ --version "${BEAT_VERSION}" \ --artifact-set "main" -echo "+++ :hammer_and_pick: Publishing $BRANCH $DRA_WORKFLOW DRA artifacts..." +echo "+++ :hammer_and_pick: Publishing DRA artifacts for version [$BEAT_VERSION], branch [$BRANCH], workflow [$DRA_WORKFLOW] and DRY_RUN: [$DRY_RUN]" + +set +x docker run --rm \ --name release-manager \ -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ @@ -59,4 +78,13 @@ docker run --rm \ --workflow "${DRA_WORKFLOW}" \ --version "${BEAT_VERSION}" \ --artifact-set "main" \ - ${DRY_RUN} + ${DRY_RUN} | tee rm-output.txt + +# extract the summary URL from a release manager output line like: +# Report summary-18.22.0.html can be found at https://artifacts-staging.elastic.co/beats/18.22.0-ABCDEFGH/summary-18.22.0.html + +SUMMARY_URL=$(grep -E '^Report summary-.* can be found at ' rm-output.txt | grep -oP 'https://\S+' | awk '{print $1}') +rm rm-output.txt + +# and make it easily clickable as a Builkite annotation +printf "**Summary link:** [${SUMMARY_URL}](${SUMMARY_URL})\n" | buildkite-agent annotate --style=success From 45971068bc49173f3b7647efa84b2eef2e1deaa2 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 26 Apr 2024 11:24:15 +0200 Subject: [PATCH 06/12] Added DEV flag to snapshot. prepare-release-manager.sh staging param (#39217) Added DEV flag to snapshot builds. Specified the workflow for prepare-release-manager.sh Relates: https://github.com/elastic/ingest-dev/issues/3095 --- .buildkite/packaging.pipeline.yml | 47 +++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 2a7bef6b89e..ebbb94483fb 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -28,6 +28,7 @@ steps: - make beats-dashboards env: SNAPSHOT: true + DEV: true artifact_paths: - build/distributions/**/* @@ -42,6 +43,9 @@ steps: commands: - make build/distributions/dependencies.csv - make beats-dashboards + env: + SNAPSHOT: false + DEV: false artifact_paths: - build/distributions/**/* @@ -53,6 +57,7 @@ steps: env: PLATFORMS: "${PLATFORMS}" SNAPSHOT: true + DEV: true command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: gcp @@ -82,6 +87,7 @@ steps: PLATFORMS: "${PLATFORMS_ARM}" PACKAGES: "docker" SNAPSHOT: true + DEV: true command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: "aws" @@ -102,7 +108,25 @@ steps: - x-pack/metricbeat - x-pack/packetbeat +<<<<<<< HEAD - group: Packaging staging +======= + ## Agentbeat needs more CPUs because it builds many other beats + - label: "SNAPSHOT: x-pack/agentbeat" + env: + PLATFORMS: "${PLATFORMS}" + SNAPSHOT: true + DEV: true + command: ".buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: + - build/distributions/**/* + + - group: Packaging Staging +>>>>>>> 5687162622 (Added DEV flag to snapshot. prepare-release-manager.sh staging param (#39217)) key: packaging-staging ## Only for release @@ -112,6 +136,7 @@ steps: env: PLATFORMS: "${PLATFORMS}" SNAPSHOT: false + DEV: false command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: gcp @@ -141,6 +166,7 @@ steps: PLATFORMS: "${PLATFORMS_ARM}" PACKAGES: "docker" SNAPSHOT: false + DEV: false command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: "aws" @@ -161,6 +187,23 @@ steps: - x-pack/metricbeat - x-pack/packetbeat +<<<<<<< HEAD +======= + ## Agentbeat needs more CPUs because it builds many other beats + - label: "STAGING: x-pack/agentbeat" + env: + PLATFORMS: "${PLATFORMS}" + SNAPSHOT: false + DEV: false + command: ".buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: + - build/distributions/**/* + +>>>>>>> 5687162622 (Added DEV flag to snapshot. prepare-release-manager.sh staging param (#39217)) - group: DRA publish key: dra steps: @@ -175,7 +218,7 @@ steps: - dashboards-snapshot command: | buildkite-agent artifact download "build/**/*" . - .buildkite/scripts/packaging/prepare-release-manager.sh + .buildkite/scripts/packaging/prepare-release-manager.sh snapshot .buildkite/scripts/dra.sh agents: provider: gcp @@ -193,7 +236,7 @@ steps: - dashboards-staging command: | buildkite-agent artifact download "build/**" . - .buildkite/scripts/packaging/prepare-release-manager.sh + .buildkite/scripts/packaging/prepare-release-manager.sh staging .buildkite/scripts/dra.sh agents: provider: gcp From c406c0e043f4978ba76f452195117fd890906446 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 26 Apr 2024 18:57:04 +0200 Subject: [PATCH 07/12] merged --- .buildkite/packaging.pipeline.yml | 35 ------------------------------- 1 file changed, 35 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index ebbb94483fb..240be4d8db2 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -108,25 +108,7 @@ steps: - x-pack/metricbeat - x-pack/packetbeat -<<<<<<< HEAD - group: Packaging staging -======= - ## Agentbeat needs more CPUs because it builds many other beats - - label: "SNAPSHOT: x-pack/agentbeat" - env: - PLATFORMS: "${PLATFORMS}" - SNAPSHOT: true - DEV: true - command: ".buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat" - agents: - provider: gcp - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: - - build/distributions/**/* - - - group: Packaging Staging ->>>>>>> 5687162622 (Added DEV flag to snapshot. prepare-release-manager.sh staging param (#39217)) key: packaging-staging ## Only for release @@ -187,23 +169,6 @@ steps: - x-pack/metricbeat - x-pack/packetbeat -<<<<<<< HEAD -======= - ## Agentbeat needs more CPUs because it builds many other beats - - label: "STAGING: x-pack/agentbeat" - env: - PLATFORMS: "${PLATFORMS}" - SNAPSHOT: false - DEV: false - command: ".buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat" - agents: - provider: gcp - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: - - build/distributions/**/* - ->>>>>>> 5687162622 (Added DEV flag to snapshot. prepare-release-manager.sh staging param (#39217)) - group: DRA publish key: dra steps: From 0e2e5c23d340eb987aba8fc7f0d1f1b941e2bb61 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 30 Apr 2024 10:27:44 +0200 Subject: [PATCH 08/12] DRA: Disable summary reports for dry runs (#39240) --- .buildkite/scripts/dra.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index b20d6c6a696..e3bc62202fc 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -80,11 +80,13 @@ docker run --rm \ --artifact-set "main" \ ${DRY_RUN} | tee rm-output.txt -# extract the summary URL from a release manager output line like: -# Report summary-18.22.0.html can be found at https://artifacts-staging.elastic.co/beats/18.22.0-ABCDEFGH/summary-18.22.0.html -SUMMARY_URL=$(grep -E '^Report summary-.* can be found at ' rm-output.txt | grep -oP 'https://\S+' | awk '{print $1}') -rm rm-output.txt +if [[ "$DRY_RUN" != "--dry-run" ]]; then + # extract the summary URL from a release manager output line like: + # Report summary-18.22.0.html can be found at https://artifacts-staging.elastic.co/beats/18.22.0-ABCDEFGH/summary-18.22.0.html + SUMMARY_URL=$(grep -E '^Report summary-.* can be found at ' rm-output.txt | grep -oP 'https://\S+' | awk '{print $1}') + rm rm-output.txt -# and make it easily clickable as a Builkite annotation -printf "**Summary link:** [${SUMMARY_URL}](${SUMMARY_URL})\n" | buildkite-agent annotate --style=success + # and make it easily clickable as a Builkite annotation + printf "**Summary link:** [${SUMMARY_URL}](${SUMMARY_URL})\n" | buildkite-agent annotate --style=success +fi From 29a1372d2a98cf33775f9b5b63b1472fd56ee071 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Tue, 30 Apr 2024 11:36:42 +0300 Subject: [PATCH 09/12] Ensure ordered DRA artifacts (#39270) As things are now we allow parallel builds on the packaging pipeline, which could result in out of order artifacts (depending on which one takes longer to finish). This commit implements two "queues" (snapshot/staging) to ensure ordered builds of DRA artifacts. Relates https://github.com/elastic/ingest-dev/issues/3095 --- .buildkite/packaging.pipeline.yml | 40 +++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 240be4d8db2..842890b7e6e 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -12,11 +12,32 @@ env: PLATFORMS_ARM: "linux/arm64" steps: + # we use concurrency gates (https://buildkite.com/blog/concurrency-gates) + # to implement two FIFO queues for DRA-snapshot and DRA-staging + # this prevents parallel builds and possibility of publishing out of order DRA artifacts if the first job takes longer than the second + + - name: Start of concurrency group for DRA Snapshot + if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + command: echo "--> Start of concurrency gate dra-snapshot" + concurrency_group: "dra-gate-snapshot" + concurrency: 1 + key: start-gate-snapshot + + - name: Start of concurrency group for DRA Staging + if: build.branch =~ /^\d+\.\d+$$/ + command: echo "--> Start of concurrency gate dra-staging" + concurrency_group: "dra-gate-staging" + concurrency: 1 + key: start-gate-staging + + - wait + - group: Beats dashboards key: dashboards steps: - label: Snapshot dashboards if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + depends_on: start-gate-snapshot key: dashboards-snapshot # TODO: container with go and make agents: @@ -34,6 +55,7 @@ steps: - label: Staging dashboards if: build.branch =~ /^\d+\.\d+$$/ + depends_on: start-gate-staging key: dashboards-staging # TODO: container with go and make agents: @@ -52,6 +74,7 @@ steps: - group: Packaging snapshot if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: packaging-snapshot + depends_on: start-gate-snapshot steps: - label: "SNAPSHOT: {{matrix}}" env: @@ -111,6 +134,7 @@ steps: - group: Packaging staging key: packaging-staging + depends_on: start-gate-staging ## Only for release if: build.branch =~ /^\d+\.\d+$$/ steps: @@ -179,6 +203,7 @@ steps: env: DRA_WORKFLOW: snapshot depends_on: + - start-gate-snapshot - packaging-snapshot - dashboards-snapshot command: | @@ -197,6 +222,7 @@ steps: env: DRA_WORKFLOW: staging depends_on: + - start-gate-staging - packaging-staging - dashboards-staging command: | @@ -207,3 +233,17 @@ steps: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + + - wait + + - command: echo "End of concurrency gate dra-snapshot <--" + if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + concurrency_group: "dra-gate-snapshot" + concurrency: 1 + key: end-gate-snapshot + + - command: echo "End of concurrency gate dra-staging <--" + if: build.branch =~ /^\d+\.\d+$$/ + concurrency_group: "dra-gate-staging" + concurrency: 1 + key: end-gate-staging From 0faac6717b54621704859bb2718715a60be3f491 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Tue, 30 Apr 2024 14:41:35 +0300 Subject: [PATCH 10/12] Fix missing docker staging DRA artifacts (#39297) The DRA staging release is failing because the Buildkite step isn't capturing the right artifacts. This commit fixes the issue by adjusting the artifact_paths to match the other steps. --- .buildkite/packaging.pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 842890b7e6e..23b34d5684d 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -179,7 +179,7 @@ steps: imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: - - build/distributions/** + - build/distributions/**/* matrix: - auditbeat - filebeat From 9c618bf109a7f004e7ba4c36ced7071a70712d2f Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 30 Apr 2024 14:58:10 +0300 Subject: [PATCH 11/12] Fix the annotating for snapshot and staging (#39299) * Fix the annotating for snapshot and staging This commit adds the `--append` flag to the buildkite-annotate so that when it is called by the snapshot and staging steps, it will not overwrite the other annotation. Signed-off-by: Alexandros Sapranidis --- .buildkite/scripts/dra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index e3bc62202fc..aeee80908a7 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -88,5 +88,5 @@ if [[ "$DRY_RUN" != "--dry-run" ]]; then rm rm-output.txt # and make it easily clickable as a Builkite annotation - printf "**Summary link:** [${SUMMARY_URL}](${SUMMARY_URL})\n" | buildkite-agent annotate --style=success + printf "**${DRA_WORKFLOW} summary link:** [${SUMMARY_URL}](${SUMMARY_URL})\n" | buildkite-agent annotate --style=success --append fi From 77c25717c52b7b1e626a36f830d3917fea9d8c1c Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Tue, 30 Apr 2024 14:53:42 +0300 Subject: [PATCH 12/12] Branch specific concurrency gates (#39298) PR #39293 introduced one concurrency queue per staging/snapshot but this slows down unnecessarily concurrent DRA builds for main and other release branches. This commit makes the concurrency gates (additionally) specific per branch. --- .buildkite/packaging.pipeline.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 23b34d5684d..a07168c235d 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -19,14 +19,14 @@ steps: - name: Start of concurrency group for DRA Snapshot if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" command: echo "--> Start of concurrency gate dra-snapshot" - concurrency_group: "dra-gate-snapshot" + concurrency_group: "dra-gate-snapshot-$BUILDKITE_BRANCH" concurrency: 1 key: start-gate-snapshot - name: Start of concurrency group for DRA Staging if: build.branch =~ /^\d+\.\d+$$/ command: echo "--> Start of concurrency gate dra-staging" - concurrency_group: "dra-gate-staging" + concurrency_group: "dra-gate-staging-$BUILDKITE_BRANCH" concurrency: 1 key: start-gate-staging @@ -238,12 +238,12 @@ steps: - command: echo "End of concurrency gate dra-snapshot <--" if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" - concurrency_group: "dra-gate-snapshot" + concurrency_group: "dra-gate-snapshot-$BUILDKITE_BRANCH" concurrency: 1 key: end-gate-snapshot - command: echo "End of concurrency gate dra-staging <--" if: build.branch =~ /^\d+\.\d+$$/ - concurrency_group: "dra-gate-staging" + concurrency_group: "dra-gate-staging-$BUILDKITE_BRANCH" concurrency: 1 key: end-gate-staging