Fixes for Buildkite packaging pipeline (#39207)

This PR fixes a number of bugs from #39126
elastic · Apr 26, 2024 · 634a2a6 · 634a2a6
1 parent 3efb10f
commit 634a2a6
Show file tree

Hide file tree

Showing 3 changed files with 49 additions and 37 deletions.
diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml
@@ -8,21 +8,21 @@ env:
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
 
-  PLATFORMS_AMD: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+  PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
   PLATFORMS_ARM: "linux/arm64"
 
 steps:
   - group: Beats dashboards
     key: dashboards
     steps:
       - label: Snapshot dashboards
-        if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true"
+        if: build.branch =~ /^\d+\.\d+$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
         key: dashboards-snapshot
         # TODO: container with go and make
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
         commands:
           - make build/distributions/dependencies.csv
           - make beats-dashboards
@@ -32,33 +32,32 @@ steps:
           - build/distributions/**/*
 
       - label: Staging dashboards
-        if: "build.branch =~ /\\d+\\.\\d+/"
+        if: build.branch =~ /^\d+\.\d+$/
         key: dashboards-staging
         # TODO: container with go and make
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
         commands:
           - make build/distributions/dependencies.csv
           - make beats-dashboards
         artifact_paths:
           - build/distributions/**/*
 
   - group: Packaging snapshot
-    if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true"
+    if: build.branch =~ /^\d+\.\d+$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
     key: packaging-snapshot
     steps:
-      # x86
-      - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot"
+      - label: "SNAPSHOT: {{matrix}}"
         env:
-          PLATFORMS: "${PLATFORMS_AMD}"
+          PLATFORMS: "${PLATFORMS}"
           SNAPSHOT: true
         command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
         artifact_paths:
           - build/distributions/**/*
         matrix:
@@ -78,8 +77,7 @@ steps:
           - x-pack/packetbeat
           - x-pack/winlogbeat
 
-      ## ARM
-      - label: ":linux: {{matrix}}/Packaging Linux arm64 Snapshot"
+      - label: "SNAPSHOT: {{matrix}} docker Linux/arm64"
         env:
           PLATFORMS: "${PLATFORMS_ARM}"
           PACKAGES: "docker"
@@ -105,36 +103,33 @@ steps:
           - x-pack/packetbeat
 
       ## Agentbeat needs more CPUs because it builds many other beats
-      - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot"
+      - label: "SNAPSHOT: x-pack/agentbeat"
         env:
-          PLATFORMS: "${PLATFORMS_AMD}"
+          PLATFORMS: "${PLATFORMS}"
           SNAPSHOT: true
-        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
+        command: ".buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat"
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "c2-standard-16"
         artifact_paths:
           - build/distributions/**/*
-        matrix:
-          - x-pack/agentbeat
 
-  - group: Packaging staging
+  - group: Packaging Staging
 
     key: packaging-staging
     ## Only for release
-    if: "build.branch =~ /\\d+\\.\\d+/"
+    if: build.branch =~ /^\d+\.\d+$/
     steps:
-      # x86
-      - label: ":ubuntu: {{matrix}}/Packaging Linux Staging"
+      - label: "STAGING: {{matrix}}"
         env:
-          PLATFORMS: "${PLATFORMS_AMD}"
+          PLATFORMS: "${PLATFORMS}"
           SNAPSHOT: false
         command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
         artifact_paths:
           - build/distributions/**/*
         matrix:
@@ -154,8 +149,7 @@ steps:
           - x-pack/packetbeat
           - x-pack/winlogbeat
 
-      ## ARM
-      - label: ":linux: {{matrix}}/Packaging Linux arm64 Staging"
+      - label: "STAGING: {{matrix}} docker Linux/arm64"
         env:
           PLATFORMS: "${PLATFORMS_ARM}"
           PACKAGES: "docker"
@@ -181,26 +175,24 @@ steps:
           - x-pack/packetbeat
 
         ## Agentbeat needs more CPUs because it builds many other beats
-      - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot"
+      - label: "STAGING: x-pack/agentbeat"
         env:
-          PLATFORMS: "${PLATFORMS_AMD}"
-          SNAPSHOT: true
-        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
+          PLATFORMS: "${PLATFORMS}"
+          SNAPSHOT: false
+        command: ".buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat"
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "c2-standard-16"
         artifact_paths:
           - build/distributions/**/*
-        matrix:
-          - x-pack/agentbeat
 
   - group: DRA publish
     key: dra
     steps:
       - label: DRA Snapshot
         ## Only for release branches and main
-        if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true"
+        if: build.branch =~ /^\d+\.\d+$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
         key: dra-snapshot
         env:
           DRA_WORKFLOW: snapshot
@@ -212,11 +204,13 @@ steps:
           .buildkite/scripts/packaging/prepare-release-manager.sh
           .buildkite/scripts/dra.sh
         agents:
-          provider: "gcp"
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
 
       - label: DRA Staging
         ## Only for release branches
-        if: "build.branch =~ /\\d+\\.\\d+/"
+        if: build.branch =~ /^\d+\.\d+$/
         key: dra-staging
         env:
           DRA_WORKFLOW: staging
@@ -228,4 +222,6 @@ steps:
           .buildkite/scripts/packaging/prepare-release-manager.sh
           .buildkite/scripts/dra.sh
         agents:
-          provider: "gcp"
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh
@@ -28,6 +28,22 @@ release_manager_login
 echo "+++ Changing permissions for the BK API commands"
 sudo chown -R :1000 build/distributions/
 
+echo "+++ :hammer_and_pick: Listing $BRANCH $DRA_WORKFLOW DRA artifacts..."
+docker run --rm \
+        --name release-manager \
+        -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
+        -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
+        -e VAULT_SECRET_ID="${VAULT_SECRET}" \
+        --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \
+        docker.elastic.co/infra/release-manager:latest \
+        cli list \
+        --project "beats" \
+        --branch "${BRANCH}" \
+        --commit "${BUILDKITE_COMMIT}" \
+        --workflow "${DRA_WORKFLOW}" \
+        --version "${BEAT_VERSION}" \
+        --artifact-set "main"
+
 echo "+++ :hammer_and_pick: Publishing $BRANCH $DRA_WORKFLOW DRA artifacts..."
 docker run --rm \
         --name release-manager \
@@ -43,4 +59,4 @@ docker run --rm \
         --workflow "${DRA_WORKFLOW}" \
         --version "${BEAT_VERSION}" \
         --artifact-set "main" \
-        ${DRY_RUN}
+        ${DRY_RUN}
diff --git a/.buildkite/scripts/packaging/package-dra.sh b/.buildkite/scripts/packaging/package-dra.sh
@@ -7,7 +7,7 @@ if [ -z "$BEAT_DIR" ]; then
     exit 1
 fi
 
-echo "~~~Packaging : $BEAT_DIR"
+echo "~~~ Packaging : $BEAT_DIR"
 
 WORKSPACE=$(pwd)
 BEAT_NAME_SLUG=$(echo "$BEAT_DIR" | sed 's/x-pack\///g')