Merge branch '8.9' into update-version-next-8.9.1

.buildkite/hooks/post-checkout

@@ -47,6 +47,7 @@ MERGE_BRANCH="pr_merge_${PR_ID}"
 checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}"
 
 echo "Commit information"
+
 git --no-pager log --format=%B -n 1
 
 # Ensure buildkite groups are rendered

.buildkite/hooks/pre-command

@@ -7,6 +7,7 @@ source .buildkite/scripts/common.sh
 DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
 EC_KEY_SECRET_PATH="kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
 PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/private_ci_artifacts_gcs_credentials"
+JOB_GCS_BUCKET_SECRET_PATH="kv/ci-shared/platform-ingest/fleet_gcs_bucket"
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" ]]; then
   if [[ "$BUILDKITE_STEP_KEY" == "publish" || "$BUILDKITE_STEP_KEY" == "cloud-e2e-test" ]]; then
@@ -24,4 +25,12 @@ fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" && "$BUILDKITE_STEP_KEY" == "release-test" ]]; then
   export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
+  export JOB_GCS_BUCKET=$(retry 5 vault kv get -field plaintext ${JOB_GCS_BUCKET_SECRET_PATH})
 fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
+  if [[ "$BUILDKITE_STEP_KEY" == "package-x86-64" || "$BUILDKITE_STEP_KEY" == "package-arm" ]]; then
+    export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
+    export JOB_GCS_BUCKET=$(retry 5 vault kv get -field plaintext ${JOB_GCS_BUCKET_SECRET_PATH})
+  fi
+fi

.buildkite/hooks/pre-exit

@@ -14,3 +14,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" && "$BUILDKITE_STEP_KEY" == "
     unset GOOGLE_APPLICATIONS_CREDENTIALS
     cleanup
 fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
+  if [[ "$BUILDKITE_STEP_KEY" == "package-x86-64" || "$BUILDKITE_STEP_KEY" == "package-arm" ]]; then
+    unset GOOGLE_APPLICATIONS_CREDENTIALS
+    cleanup
+  fi
+fi

.buildkite/pipeline.package.mbp.yml

@@ -1,12 +1,34 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 name: "fleet server package mbp"
-steps:
-  - label: ":pipeline: test step"
-    command: echo "test-pipeline"
+env:
+  REPO: 'fleet-server'
+  JOB_GCS_BUCKET: 'fleet-ci-artifacts'
+  SETUP_GVM_VERSION: 'v0.3.0'
 
-  - group: "test group"
+steps:
+  - group: "Package and Publish"
+    key: "package-publish"
     steps:
-      - label: "step1 of the group"
-        command: echo "step1 of the group"
+    - label: "Package Ubuntu-20 x86_64"
+      key: "package-x86-64"
+      command: ".buildkite/scripts/package.sh {{matrix.type}}"
+      agents:
+        provider: "gcp"
+        image: "family/core-ubuntu-2004"
+      matrix:
+        setup:
+          type:
+          - "snapshot"
+          - "staging"
 
-      - label: "step2 of the group"
-        command: echo "step2 of the group"
+    - label: "Package Ubuntu-20 aarch64"
+      key: "package-arm"
+      command: ".buildkite/scripts/package.sh {{matrix.type}}"
+      agents:
+        provider: "aws"
+        image: "core-ubuntu-2004-aarch64"
+      matrix:
+        setup:
+          type:
+          - "snapshot"
+          - "staging"

.buildkite/pipeline.yml

@@ -10,7 +10,6 @@ env:
   DOCKER_IMAGE_GIT_TAG: "temp-${BUILDKITE_BRANCH}" # needs to rename after tests
   GO_AGENT_IMAGE: "golang:1.20.5"
   TERRAFORM_VERSION: '1.4.6'
-  JOB_GCS_BUCKET: 'fleet-ci-artifacts'
 
 steps:
   - group: "Check and build"
@@ -74,7 +73,7 @@ steps:
           - step: "int-test"
             allow_failure: false
 
-      - label: "Cloud e2e Test"
+      - label: ":gcloud: Cloud e2e Test"
         key: "cloud-e2e-test"
         command: ".buildkite/scripts/cloud_e2e_test.sh"
         agents:
@@ -124,14 +123,20 @@ steps:
 
   - label: ":jenkins: Release - Package Registry Distribution"
     key: "release-package-registry"
-    command: echo "Triggering the Jenkins Job 'release-package-registry-distribution'..."
+    trigger: "package-registry-release-package-registry-distribution"
+    build:
+      branch: "main"
+      meta_data:
+        DOCKER_TAG: "${BUILDKITE_TAG}"
+    if: "build.env('BUILDKITE_TAG') != ''"
     depends_on:
       - step: "release-test"
         allow_failure: false
 
-  - label: ":jenkins: Downstream - Package"
+  - trigger: "fleet-server-package-mbp"
+    label: ":esbuild: Downstream - Package"
     key: "downstream-package"
-    command: echo "Triggering the Jenkins Job 'fleet-server/fleet-server-package-mbp/env.JOB_BASE_NAME'..."
+    if:  "build.env('BUILDKITE_BRANCH') != '' && build.env('BUILDKITE_TAG') == '' && build.env('BUILDKITE_PULL_REQUEST') == ''"
     depends_on:
       - step: "release-package-registry"
-        allow_failure: false
+        allow_failure: false

.buildkite/scripts/common.sh

@@ -94,6 +94,34 @@ upload_packages_to_gcp_bucket() {
     done
 }
 
+upload_mbp_packages_to_gcp_bucket() {
+    local pattern=${1}
+    local type=${2}
+    local baseUri="gs://${JOB_GCS_BUCKET}/jobs/buildkite"              #TODO: needs to delete the "/buildkite" part after the migration from Jenkins
+    local bucketUri=""
+
+    if [[ ${type} == "snapshot" ]]; then
+        bucketUri="${baseUri}"/commits/${BUILDKITE_COMMIT}
+    else
+        bucketUri="${baseUri}"/${type}/${BUILDKITE_COMMIT}
+    fi
+    gsutil -m -q cp -a public-read -r ${pattern} "${bucketUri}"
+}
+
+with_mage() {
+    local install_packages=(
+            "github.com/magefile/mage"
+            "github.com/elastic/go-licenser"
+            "golang.org/x/tools/cmd/goimports"
+            "github.com/jstemmer/go-junit-report"
+            "gotest.tools/gotestsum"
+    )
+
+    for pkg in "${install_packages[@]}"; do
+        go install "${pkg}@latest"
+    done
+}
+
 cleanup() {
     echo "Deleting temporal files..."
     cd ${WORKSPACE}

.buildkite/scripts/package.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common.sh
+
+VERSION=$(awk '/const DefaultVersion/{print $NF}' version/version.go | tr -d '"')
+PLATFORM_TYPE=$(uname -m)
+MATRIX_TYPE="$1"
+INFRA_REPO="https://github.com/repos/elastic/infra/contents"
+
+if [[ ${BUILDKITE_BRANCH} == "main" && ${MATRIX_TYPE} == "staging" ]]; then
+    echo "INFO: staging artifacts for the main branch are not required."
+    exit 0
+fi
+
+PLATFORMS=""
+PACKAGES=""
+if [[ ${PLATFORM_TYPE} == "arm" || ${PLATFORM_TYPE} == "aarch64" ]]; then
+    PLATFORMS="linux/arm64"
+    PACKAGES="docker"
+fi
+
+add_bin_path
+with_go
+with_mage
+
+if [[ ${MATRIX_TYPE} == "staging" ]]; then
+    make release
+else
+    make SNAPSHOT=true release
+fi
+
+google_cloud_auth
+upload_mbp_packages_to_gcp_bucket "build/distributions/" "${MATRIX_TYPE}"

.mergify.yml

@@ -155,3 +155,16 @@ pull_request_rules:
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
+  - name: backport patches to 8.9 branch
+    conditions:
+      - merged
+      - label=backport-v8.9.0
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        branches:
+          - "8.9"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"

dev-tools/integration/.env

@@ -1,4 +1,4 @@
-ELASTICSEARCH_VERSION=8.9.0-0032a096-SNAPSHOT
+ELASTICSEARCH_VERSION=8.9.0-164e5e60-SNAPSHOT
 ELASTICSEARCH_USERNAME=elastic
 ELASTICSEARCH_PASSWORD=changeme
 TEST_ELASTICSEARCH_HOSTS=localhost:9200

internal/pkg/api/handleFileDelivery.go

@@ -9,7 +9,6 @@ import (
 	"net/http"
 	"strconv"
 
-	"github.com/elastic/fleet-server/v7/internal/pkg/apikey"
 	"github.com/elastic/fleet-server/v7/internal/pkg/bulk"
 	"github.com/elastic/fleet-server/v7/internal/pkg/cache"
 	"github.com/elastic/fleet-server/v7/internal/pkg/config"
@@ -26,7 +25,6 @@ type FileDeliveryT struct {
 	chunkClient *elasticsearch.Client
 	deliverer   *delivery.Deliverer
 	authAgent   func(*http.Request, *string, bulk.Bulk, cache.Cache) (*model.Agent, error) // injectable for testing purposes
-	authAPIKey  func(*http.Request, bulk.Bulk, cache.Cache) (*apikey.APIKey, error)        // as above
 }
 
 func NewFileDeliveryT(cfg *config.Server, bulker bulk.Bulk, chunkClient *elasticsearch.Client, cache cache.Cache) *FileDeliveryT {
@@ -41,12 +39,11 @@ func NewFileDeliveryT(cfg *config.Server, bulker bulk.Bulk, chunkClient *elastic
 		cache:       cache,
 		deliverer:   delivery.New(chunkClient, bulker, maxFileSize),
 		authAgent:   authAgent,
-		authAPIKey:  authAPIKey,
 	}
 }
 
 func (ft *FileDeliveryT) handleSendFile(zlog zerolog.Logger, w http.ResponseWriter, r *http.Request, fileID string) error {
-	agent, err := authAgent(r, nil, ft.bulker, ft.cache)
+	agent, err := ft.authAgent(r, nil, ft.bulker, ft.cache)
 	if err != nil {
 		return err
 	}