Beats DRA pipeline

.buildkite/packaging.pipeline.md

@@ -0,0 +1,24 @@
+### Beats Packaging pipeline
+[Buildkite packaging pipeline](https://buildkite.com/elastic/beats-packaging-pipeline) is used to build and publish the packages for the Beats. The pipeline is triggered by a commit to the `main` or release branches.
+The pipeline definition is located in the `.buildkite/packaging.pipeline.yml`
+
+### Triggers
+Staging packaging DRA is triggered for the `main` and release branches.
+Snapshot can be triggered for any branch by the `/package` comment in the PR. The release-manager dry-run will be used for PR builds.
+
+### Pipeline steps
+
+#### Beats dashboards
+
+Generates `build/distributions/dependencies.csv` and adds it to the `beats-dashboards` artifact. The `dependencies.csv` is required by the release-manager configuration
+
+#### Packaging snapshot/staging
+
+- Builds the Beats packages for all supported platforms and architectures (`mage package, mage ironbank`)
+- Copies artifacts `build/distributions/<beat>/` directory and adds it as an artifact. Where `<beat>` is the name of the beat
+- x-pack artifacts a also copied to `build/distributions/<beat>/` directory, where `<beat>` is the name of the beat. For example, `auditbeat`, not `x-pack/auditbeat`
+
+#### DRA publish
+Downloads the artifacts from the `packaging snapshot/staging` step and publishes them to the Elastic DRA registry.
+
+

.buildkite/packaging.pipeline.yml

@@ -0,0 +1,229 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+# TODO: Pre-cache beats-dev/golang-crossbuild container image
+
+env:
+  ASDF_MAGE_VERSION: 1.15.0
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+  GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
+  IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
+
+steps:
+  - group: Beats dashboards
+    key: dashboards
+    steps:
+      - label: Snapshot dashboards
+        key: dashboards-snapshot
+        # TODO: container with go and make
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        commands:
+          - make build/distributions/dependencies.csv
+          - make beats-dashboards
+        env:
+          SNAPSHOT: true
+        artifact_paths:
+          - build/distributions/**/*
+
+      - label: Staging dashboards
+        if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'"
+        key: dashboards-staging
+        # TODO: container with go and make
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        commands:
+          - make build/distributions/dependencies.csv
+          - make beats-dashboards
+        artifact_paths:
+          - build/distributions/**/*
+
+  - group: Packaging snapshot
+    key: packaging-snapshot
+    steps:
+      # x86
+      - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot"
+        env:
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+          SNAPSHOT: true
+        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        artifact_paths:
+          - build/distributions/**/*
+        matrix:
+          - auditbeat
+          - filebeat
+          - heartbeat
+          - metricbeat
+          - packetbeat
+          - winlogbeat
+          - x-pack/auditbeat
+          - x-pack/dockerlogbeat
+          - x-pack/filebeat
+          - x-pack/functionbeat
+          - x-pack/heartbeat
+          - x-pack/metricbeat
+          - x-pack/packetbeat
+          - x-pack/winlogbeat
+
+      ## ARM
+      - label: ":linux: {{matrix}}/Packaging Linux arm64 Snapshot"
+        env:
+          PLATFORMS: "linux/arm64"
+          PACKAGES: "docker"
+          SNAPSHOT: true
+        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths:
+          - build/distributions/**/*
+        matrix:
+          - auditbeat
+          - filebeat
+          - heartbeat
+          - metricbeat
+          - packetbeat
+          - x-pack/auditbeat
+          - x-pack/dockerlogbeat
+          - x-pack/filebeat
+          - x-pack/heartbeat
+          - x-pack/metricbeat
+          - x-pack/packetbeat
+
+      ## Osquerybeat and agentbeat require msitools to be installed
+      ## Agentbeat needs more CPUs because it builds many other beats
+      ## TODO: pre-install msitools to the VM and delete this exclusion
+      - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot"
+        env:
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+          SNAPSHOT: true
+        command: |
+          .buildkite/scripts/install-msitools.sh
+          .buildkite/scripts/packaging/package-dra.sh {{matrix}}
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "c2-standard-16"
+        artifact_paths:
+          - build/distributions/**/*
+        matrix:
+          - x-pack/agentbeat
+          - x-pack/osquerybeat
+
+  - group: Packaging staging
+
+    key: packaging-staging
+    ## Only for release and main
+    if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'"
+    steps:
+      # x86
+      - label: ":ubuntu: {{matrix}}/Packaging Linux Staging"
+        env:
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+          SNAPSHOT: false
+        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        artifact_paths:
+          - build/distributions/**/*
+        matrix:
+          - auditbeat
+          - filebeat
+          - heartbeat
+          - metricbeat
+          - packetbeat
+          - winlogbeat
+          - x-pack/auditbeat
+          - x-pack/dockerlogbeat
+          - x-pack/filebeat
+          - x-pack/functionbeat
+          - x-pack/heartbeat
+          - x-pack/metricbeat
+          - x-pack/packetbeat
+          - x-pack/winlogbeat
+
+      ## ARM
+      - label: ":linux: {{matrix}}/Packaging Linux arm64 Staging"
+        env:
+          PLATFORMS: "linux/arm64"
+          PACKAGES: "docker"
+          SNAPSHOT: false
+        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths:
+          - build/distributions/**
+        matrix:
+          - auditbeat
+          - filebeat
+          - heartbeat
+          - metricbeat
+          - packetbeat
+          - x-pack/auditbeat
+          - x-pack/dockerlogbeat
+          - x-pack/filebeat
+          - x-pack/heartbeat
+          - x-pack/metricbeat
+          - x-pack/packetbeat
+
+        ## Osquerybeat and agentbeat require msitools to be installed
+        ## TODO: pre-install msitools to the VM and delete this exclusion
+      - label: ":ubuntu: {{matrix}}/Packaging Linux Staging"
+        env:
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+          SNAPSHOT: false
+        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        artifact_paths:
+          - build/distributions/**
+        matrix:
+          - x-pack/agentbeat
+          - x-pack/osquerybeat
+
+  - group: DRA publish
+    key: dra
+    steps:
+      - label: DRA Snapshot
+        key: dra-snapshot
+        env:
+          DRA_WORKFLOW: snapshot
+        depends_on:
+          - packaging-snapshot
+          - dashboards-snapshot
+        command: |
+          buildkite-agent artifact download "build/**/*" . 
+          .buildkite/scripts/packaging/prepare-release-manager.sh
+          .buildkite/scripts/dra.sh
+        agents:
+          provider: "gcp"
+
+      - label: DRA Staging
+        ## Only for release and main
+        if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'"
+        key: dra-staging
+        env:
+          DRA_WORKFLOW: staging
+        depends_on:
+          - packaging-staging
+          - dashboards-staging
+        command: |
+          buildkite-agent artifact download "build/**" .
+          .buildkite/scripts/packaging/prepare-release-manager.sh
+          .buildkite/scripts/dra.sh
+        agents:
+          provider: "gcp"

.buildkite/pull-requests.json

@@ -47,6 +47,20 @@
             "skip_target_branches": [ ],
             "skip_ci_on_only_changed": [ ],
             "always_require_ci_on_changed": ["^winlogbeat/.*", ".buildkite/winlogbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"]
+        },
+        {
+            "enabled": true,
+            "pipelineSlug": "beats-packaging-pipeline",
+            "allow_org_users": true,
+            "allowed_repo_permissions": ["admin", "write"],
+            "allowed_list": [ ],
+            "set_commit_status": false,
+            "build_on_commit": false,
+            "build_on_comment": true,
+            "trigger_comment_regex": "^/package",
+            "always_trigger_comment_regex": "^/package",
+            "skip_ci_labels": [  ],
+            "skip_target_branches": [ ]
         }
     ]
 }

.buildkite/scripts/dra.sh

@@ -0,0 +1,64 @@
+set -uo pipefail
+
+DRY_RUN=""
+BRANCH="${BUILDKITE_BRANCH}"
+
+if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" || "${BUILDKITE_BRANCH}" == "ci_packaging_pipieline" ]]; then
+    BRANCH=main
+    DRY_RUN="--dry-run"
+    echo "+++ Running in PR and setting branch main and --dry-run"
+fi
+
+BEAT_VERSION=$(make get-version)
+
+CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
+
+# TODO use common function
+retry() {
+  local retries=$1
+  shift
+  local count=0
+  until "$@"; do
+    exit=$?
+    wait=$((2 ** count))
+    count=$((count + 1))
+    if [ $count -lt "$retries" ]; then
+      >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+      sleep $wait
+    else
+      >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+      return $exit
+    fi
+  done
+  return 0
+}
+
+function release_manager_login {
+  DRA_CREDS_SECRET=$(retry 5 vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH})
+  VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr')
+  VAULT_ROLE_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id')
+  VAULT_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id')
+  export VAULT_ADDR_SECRET VAULT_ROLE_ID_SECRET VAULT_SECRET
+}
+
+release_manager_login
+
+echo "+++ Changing permissions for the BK API commands"
+sudo chown -R :1000 build/distributions/
+
+echo "+++ :hammer_and_pick: Publishing $BRANCH $DRA_WORKFLOW DRA artifacts..."
+docker run --rm \
+        --name release-manager \
+        -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
+        -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
+        -e VAULT_SECRET_ID="${VAULT_SECRET}" \
+        --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \
+        docker.elastic.co/infra/release-manager:latest \
+        cli collect \
+        --project "beats" \
+        --branch "${BRANCH}" \
+        --commit "${BUILDKITE_COMMIT}" \
+        --workflow "${DRA_WORKFLOW}" \
+        --version "${BEAT_VERSION}" \
+        --artifact-set "main" \
+        ${DRY_RUN}

.buildkite/scripts/packaging/package-dra.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+BEAT_DIR=$1
+
+if [ -z "$BEAT_DIR" ]; then
+    echo "Error: Beat directory must be specified."
+    exit 1
+fi
+
+echo "Packaging : $BEAT_DIR"
+
+WORKSPACE=$(pwd)
+BEAT_NAME_SLUG=$(echo "$BEAT_DIR" | sed 's/x-pack\///g')
+
+cd $BEAT_DIR
+mage package
+mage ironbank
+
+mkdir -p $WORKSPACE/build/distributions/$BEAT_NAME_SLUG
+cp build/distributions/* $WORKSPACE/build/distributions/$BEAT_NAME_SLUG/
+cd $WORKSPACE