Fixes for packaging pipeline

.buildkite/packaging.pipeline.yml

@@ -8,15 +8,15 @@ env:
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
 
-  PLATFORMS_AMD: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+  PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
   PLATFORMS_ARM: "linux/arm64"
 
 steps:
   - group: Beats dashboards
     key: dashboards
     steps:
       - label: Snapshot dashboards
-        if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true"
+        if: build.branch =~ /\d+\.\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
         key: dashboards-snapshot
         # TODO: container with go and make
         agents:
@@ -32,7 +32,7 @@ steps:
           - build/distributions/**/*
 
       - label: Staging dashboards
-        if: "build.branch =~ /\\d+\\.\\d+/"
+        if: build.branch =~ /\d+\.\d+/
         key: dashboards-staging
         # TODO: container with go and make
         agents:
@@ -46,13 +46,12 @@ steps:
           - build/distributions/**/*
 
   - group: Packaging snapshot
-    if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true"
+    if: build.branch =~ /\d+\.\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
     key: packaging-snapshot
     steps:
-      # x86
-      - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot"
+      - label: "SNAPSHOT: {{matrix}}/Packaging cross architecture artifacts"
         env:
-          PLATFORMS: "${PLATFORMS_AMD}"
+          PLATFORMS: "${PLATFORMS}"
           SNAPSHOT: true
         command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
         agents:
@@ -78,8 +77,7 @@ steps:
           - x-pack/packetbeat
           - x-pack/winlogbeat
 
-      ## ARM
-      - label: ":linux: {{matrix}}/Packaging Linux arm64 Snapshot"
+      - label: "SNAPSHOT: {{matrix}}/Packaging docker artifacts for Linux/arm64"
         env:
           PLATFORMS: "${PLATFORMS_ARM}"
           PACKAGES: "docker"
@@ -105,30 +103,27 @@ steps:
           - x-pack/packetbeat
 
       ## Agentbeat needs more CPUs because it builds many other beats
-      - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot"
+      - label: "SNAPSHOT: x-pack/agentbeat/Packaging cross architecture artifacts"
         env:
-          PLATFORMS: "${PLATFORMS_AMD}"
+          PLATFORMS: "${PLATFORMS}"
           SNAPSHOT: true
-        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
+        command: ".buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat"
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "c2-standard-16"
         artifact_paths:
           - build/distributions/**/*
-        matrix:
-          - x-pack/agentbeat
 
   - group: Packaging staging
 
     key: packaging-staging
     ## Only for release
-    if: "build.branch =~ /\\d+\\.\\d+/"
+    if: build.branch =~ /\d+\.\d+/
     steps:
-      # x86
-      - label: ":ubuntu: {{matrix}}/Packaging Linux Staging"
+      - label: "STAGING: {{matrix}}/Packaging cross architecture artifacts"
         env:
-          PLATFORMS: "${PLATFORMS_AMD}"
+          PLATFORMS: "${PLATFORMS}"
           SNAPSHOT: false
         command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
         agents:
@@ -154,8 +149,7 @@ steps:
           - x-pack/packetbeat
           - x-pack/winlogbeat
 
-      ## ARM
-      - label: ":linux: {{matrix}}/Packaging Linux arm64 Staging"
+      - label: "STAGING: {{matrix}}/Packaging docker artifacts for Linux/arm64"
         env:
           PLATFORMS: "${PLATFORMS_ARM}"
           PACKAGES: "docker"
@@ -181,26 +175,24 @@ steps:
           - x-pack/packetbeat
 
         ## Agentbeat needs more CPUs because it builds many other beats
-      - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot"
+      - label: "STAGING: x-pack/agentbeat/Packaging cross architecture artifacts"
         env:
-          PLATFORMS: "${PLATFORMS_AMD}"
-          SNAPSHOT: true
-        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
+          PLATFORMS: "${PLATFORMS}"
+          SNAPSHOT: false
+        command: ".buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat"
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "c2-standard-16"
         artifact_paths:
           - build/distributions/**/*
-        matrix:
-          - x-pack/agentbeat
 
   - group: DRA publish
     key: dra
     steps:
       - label: DRA Snapshot
         ## Only for release branches and main
-        if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true"
+        if: build.branch =~ /\d+\.\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
         key: dra-snapshot
         env:
           DRA_WORKFLOW: snapshot
@@ -216,7 +208,7 @@ steps:
 
       - label: DRA Staging
         ## Only for release branches
-        if: "build.branch =~ /\\d+\\.\\d+/"
+        if: build.branch =~ /\d+\.\d+/
         key: dra-staging
         env:
           DRA_WORKFLOW: staging

.buildkite/scripts/dra.sh

@@ -28,6 +28,22 @@ release_manager_login
 echo "+++ Changing permissions for the BK API commands"
 sudo chown -R :1000 build/distributions/
 
+echo "+++ :hammer_and_pick: Listing $BRANCH $DRA_WORKFLOW DRA artifacts..."
+docker run --rm \
+        --name release-manager \
+        -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
+        -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
+        -e VAULT_SECRET_ID="${VAULT_SECRET}" \
+        --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \
+        docker.elastic.co/infra/release-manager:latest \
+        cli list \
+        --project "beats" \
+        --branch "${BRANCH}" \
+        --commit "${BUILDKITE_COMMIT}" \
+        --workflow "${DRA_WORKFLOW}" \
+        --version "${BEAT_VERSION}" \
+        --artifact-set "main"
+
 echo "+++ :hammer_and_pick: Publishing $BRANCH $DRA_WORKFLOW DRA artifacts..."
 docker run --rm \
         --name release-manager \
@@ -43,4 +59,4 @@ docker run --rm \
         --workflow "${DRA_WORKFLOW}" \
         --version "${BEAT_VERSION}" \
         --artifact-set "main" \
-        ${DRY_RUN}
+        ${DRY_RUN}

.buildkite/scripts/packaging/package-dra.sh

@@ -7,7 +7,7 @@ if [ -z "$BEAT_DIR" ]; then
     exit 1
 fi
 
-echo "~~~Packaging : $BEAT_DIR"
+echo "~~~ Packaging : $BEAT_DIR"
 
 WORKSPACE=$(pwd)
 BEAT_NAME_SLUG=$(echo "$BEAT_DIR" | sed 's/x-pack\///g')