[8.10](backport #3573) [CI] Pull requests migration to buildkite

.buildkite/hooks/post-checkout.ps1

@@ -0,0 +1,60 @@
+# Set error handling
+$ErrorActionPreference = "Stop"
+
+# Define a function to checkout and merge
+function Checkout-Merge {
+    param (
+        [string]$targetBranch,
+        [string]$prCommit,
+        [string]$mergeBranch
+    )
+
+    if (-not $targetBranch) {
+        Write-Host "No pull request target branch"
+        exit 1
+    }
+
+    git fetch -v origin $targetBranch
+    git checkout FETCH_HEAD
+    Write-Host "Current branch: $(git rev-parse --abbrev-ref HEAD)"
+
+    # Create a temporary branch to merge the PR with the target branch
+    git checkout -b $mergeBranch
+    Write-Host "New branch created: $(git rev-parse --abbrev-ref HEAD)"
+
+    # Set author identity so it can be used for git merge
+    git config user.name "github-merged-pr-post-checkout"
+    git config user.email "auto-merge@buildkite"
+
+    git merge --no-edit $prCommit
+
+    if ($LASTEXITCODE -ne 0) {
+        $mergeResult = $LASTEXITCODE
+        Write-Host "Merge failed: $mergeResult"
+        git merge --abort
+        exit $mergeResult
+    }
+}
+
+$pullRequest = $env:BUILDKITE_PULL_REQUEST
+
+if ($pullRequest -eq "false") {
+    Write-Host "Not a pull request, skipping"
+    exit 0
+}
+
+$targetBranch = $env:BUILDKITE_PULL_REQUEST_BASE_BRANCH
+$prCommit = $env:BUILDKITE_COMMIT
+$prId = $env:BUILDKITE_PULL_REQUEST
+$mergeBranch = "pr_merge_$prId"
+
+Checkout-Merge $targetBranch $prCommit $mergeBranch
+
+Write-Host "Commit information"
+git --no-pager log --format=%B -n 1
+
+Write-Host "Fixing CRLF in git checkout --"
+git config core.autocrlf true
+
+# Ensure Buildkite groups are rendered
+Write-Host ""

.buildkite/hooks/pre-command.ps1

@@ -0,0 +1,25 @@
+# Install gcc TODO: Move to the VM image
+choco install mingw
+Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1
+refreshenv
+
+# Install gvm and go
+# TODO: Move GVM download to the base VM image
+$env:GvmVersion = "0.5.2"
+[Net.ServicePointManager]::SecurityProtocol = "tls12"
+$env:GoVersion = Get-Content -Path .go-version
+Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v$env:GvmVersion/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe
+gvm --format=powershell $env:GoVersion | Invoke-Expression
+go version
+
+$GOPATH = $(go env GOPATH)
+$env:Path = "$GOPATH\bin;" + $env:Path
+[Environment]::SetEnvironmentVariable("GOPATH", "$GOPATH", [EnvironmentVariableTarget]::Machine)
+[Environment]::SetEnvironmentVariable("Path", "$GOPATH\bin;$env:Path", [EnvironmentVariableTarget]::Machine)
+
+# Install tools
+go install github.com/magefile/mage
+go install github.com/elastic/go-licenser
+go install golang.org/x/tools/cmd/goimports
+go install github.com/jstemmer/go-junit-report
+go install gotest.tools/gotestsum

.buildkite/pipeline.yml

@@ -4,6 +4,148 @@ env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
   DOCKER_REGISTRY: "docker.elastic.co"
 steps:
+  - label: "Unit tests - Ubuntu 22.04"
+    key: "unit-tests-2204"
+    command: ".buildkite/scripts/steps/unit-tests.sh"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: "gcp"
+      image: "family/core-ubuntu-2204"
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Unit tests - Ubuntu 22.04 ARM64"
+    key: "unit-tests-2204-arm64"
+    command: ".buildkite/scripts/steps/unit-tests.sh"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: "aws"
+      imagePrefix: "core-ubuntu-2204-aarch64"
+      diskSizeGb: 200
+      instanceType: "m6g.4xlarge"
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Unit tests - Windows 2022"
+    key: "unit-tests-win2022"
+    command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: "gcp"
+      image: "family/core-windows-2022"
+      machine_type: "n2-standard-8"
+      disk_size: 200
+      disk_type: "pd-ssd"
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Unit tests - Windows 2016"
+    key: "unit-tests-win2016"
+    command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: "gcp"
+      image: "family/core-windows-2016"
+      machine_type: "n2-standard-8"
+      disk_size: 200
+      disk_type: "pd-ssd"
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Unit tests - MacOS 13 ARM"
+    key: "unit-tests-macos-13-arm"
+    command: ".buildkite/scripts/steps/unit-tests.sh"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: orka
+      imagePrefix: generic-13-ventura-arm
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Unit tests - MacOS 13"
+    key: "unit-tests-macos-13"
+    command: ".buildkite/scripts/steps/unit-tests.sh"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: orka
+      imagePrefix: generic-13-ventura-x64
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Merge coverage reports"
+    key: "merge-coverage"
+    env:
+      BUILDKITE_REPO: ""
+    command: "
+      .buildkite/scripts/steps/merge.sh
+      unit-tests-2204
+      unit-tests-2204-arm64
+      unit-tests-win2016
+      unit-tests-win2022
+      unit-tests-macos-13
+      unit-tests-macos-13-arm
+      "
+    artifact_paths:
+      - "build/TEST-**"
+    agents:
+      image: "golang:1.20.10"
+    depends_on:
+      - unit-tests-2204
+      - unit-tests-2204-arm64
+      - unit-tests-win2022
+      - unit-tests-win2016
+      - unit-tests-macos-13
+      - unit-tests-macos-13-arm
+    allow_dependency_failure: true
+
+  - label: ":sonarqube: Continuous Code Inspection"
+    env:
+      VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token"
+    agents:
+      image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
+    command:
+      - "buildkite-agent artifact download --step merge-coverage build/TEST-go-unit.cov ."
+      - "/scan-source-code.sh"
+    depends_on:
+      - "merge-coverage"
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Serverless integration test"
+    key: "serverless-integration-tests"
+    command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestMonitoringLogsShipped" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+
   - label: "Integration tests"
     key: "integration-tests"
     command: ".buildkite/scripts/steps/integration_tests.sh"
@@ -21,4 +163,4 @@ steps:
       provider: "gcp"
     plugins:
       - junit-annotate#v2.4.1:
-          artifacts: build/TEST-go-*.xml
+          artifacts: build/TEST-go-integration*.xml

.buildkite/scripts/common.sh

@@ -22,7 +22,7 @@ getOSOptions() {
       export AGENT_OS_NAME=linux
       ;;
     darwin*)
-      export AGENT_OS_NAME=osx
+      export AGENT_OS_NAME=darwin
       ;;
     msys*)
       export AGENT_OS_NAME=windows

.buildkite/scripts/steps/merge.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Downloads and merges coverage files from multiple steps into a single file (build/TEST-go-unit.cov).
+# Usage: merge.sh <step1> <step2> ... Where <step> is the id of the step that contains the coverage artifact.#  
+
+set -exuo pipefail
+
+COV_ARTIFACT="coverage.out"
+MERGED_COV_FILE="build/TEST-go-unit.cov"
+# Space separated list of paths to coverage files
+COV_PATHS=""
+
+go install github.com/wadey/gocovmerge@latest
+mkdir -p build
+
+for STEP_ID in "$@"; do
+  mkdir -p $STEP_ID 
+  buildkite-agent artifact download --step $STEP_ID $COV_ARTIFACT $STEP_ID
+  COV_PATHS="${COV_PATHS} $STEP_ID/$COV_ARTIFACT"
+done
+
+gocovmerge $COV_PATHS > $MERGED_COV_FILE
+echo "Merged coverage file: $MERGED_COV_FILE. See artifacts"

.buildkite/scripts/steps/unit-tests.ps1

@@ -0,0 +1,28 @@
+$ErrorActionPreference = "Stop"
+
+Write-Host "-- Fixing CRLF in git checkout --"
+git config core.autocrlf input
+git rm --quiet --cached -r .
+git reset --quiet --hard
+
+$env:GOTMPDIR = "$env:BUILDKITE_BUILD_CHECKOUT_PATH"
+
+Write-Host "--- Build"
+mage build
+
+if ($LASTEXITCODE -ne 0) {
+  exit 1 
+}
+
+Write-Host "--- Unit tests"
+$env:TEST_COVERAGE = $true
+$env:RACE_DETECTOR = $true
+mage unitTest
+# Copy coverage file to build directory so it can be downloaded as an artifact
+cp .\build\TEST-go-unit.cov coverage.out
+
+if ($LASTEXITCODE -ne 0) {
+  exit 1 
+}
+
+

.buildkite/scripts/steps/unit-tests.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+source .buildkite/scripts/common.sh
+
+echo "--- Unit tests"
+RACE_DETECTOR=true TEST_COVERAGE=true mage unitTest
+# Copy coverage file to build directory so it can be downloaded as an artifact
+cp build/TEST-go-unit.cov coverage.out