Merge branch 'main' into ml-190591-fix-overall-score

.buildkite/pipeline-resource-definitions/kibana-deploy-project.yml

@@ -28,9 +28,11 @@ spec:
       pipeline_file: .buildkite/pipelines/serverless_deployment/project-build-and-deploy-pr.yml
       skip_intermediate_builds: true
       provider_settings:
+        build_pull_requests: true
         prefix_pull_request_fork_branch_names: false
         skip_pull_request_builds_for_existing_commits: true
         trigger_mode: none
+      cancel_intermediate_builds: true
       teams:
         kibana-operations:
           access_level: MANAGE_BUILD_AND_READ

.buildkite/pipeline-utils/github/github.ts

@@ -93,6 +93,26 @@ export const doAnyChangesMatch = async (
   return anyFilesMatchRequired;
 };
 
+export function addComment(
+  comment: string,
+  owner = process.env.GITHUB_PR_BASE_OWNER,
+  repo = process.env.GITHUB_PR_BASE_REPO,
+  prNumber: undefined | string | number = process.env.GITHUB_PR_NUMBER
+) {
+  if (!owner || !repo || !prNumber) {
+    throw Error(
+      "Couldn't retrieve Github PR info from environment variables in order to add a comment"
+    );
+  }
+
+  return github.issues.createComment({
+    owner,
+    repo,
+    issue_number: typeof prNumber === 'number' ? prNumber : parseInt(prNumber, 10),
+    body: comment,
+  });
+}
+
 export function getGithubClient() {
   return github;
 }

.buildkite/pipelines/serverless_deployment/project-build-and-deploy-pr.yml

@@ -1,53 +1,72 @@
-agents:
-  provider: gcp
-  image: family/kibana-ubuntu-2004
-  imageProject: elastic-images-prod
+env:
+  ELASTIC_PR_COMMENTS_ENABLED: 'true'
+  GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true'
+  GITHUB_BUILD_COMMIT_STATUS_CONTEXT: kibana-deploy-project-from-pr
 
 steps:
-  - command: .buildkite/scripts/lifecycle/pre_build.sh
-    label: Pre-Build
-    timeout_in_minutes: 10
-    agents:
-      machineType: n2-standard-2
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
-
-  - wait: ~
-
-  - command: .buildkite/scripts/steps/build_kibana.sh
-    label: Build Kibana Distribution and Plugins
-    agents:
-      machineType: n2-standard-16
-      preemptible: true
-    key: build
-    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
-    timeout_in_minutes: 90
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-
-  - wait: ~
-
-  - command: .buildkite/scripts/steps/artifacts/docker_image.sh
-    label: 'Build Project Image'
-    key: build_project_image
-    agents:
-      machineType: n2-standard-16
-      preemptible: true
-    timeout_in_minutes: 60
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-
-  - wait: ~
-
-  - command: .buildkite/scripts/steps/serverless/deploy.sh
-    label: 'Deploy Project'
-    agents:
-      machineType: n2-standard-4
-      preemptible: true
-    timeout_in_minutes: 10
+  - group: 'Project Deployment'
+    if: "build.env('GITHUB_PR_LABELS') =~ /ci:project-deploy-(elasticsearch|observability|security)/"
+
+    steps:
+      - command: .buildkite/scripts/lifecycle/pre_build.sh
+        label: Pre-Build
+        timeout_in_minutes: 10
+        agents:
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          machineType: n2-standard-2
+        retry:
+          automatic:
+            - exit_status: '*'
+              limit: 1
+
+      - command: |
+          ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts "PR Project deployment started at: $BUILDKITE_BUILD_URL"
+        label: Comment with job URL
+        agents:
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          machineType: n2-standard-2
+        timeout_in_minutes: 5
+
+      - wait: ~
+
+      - command: .buildkite/scripts/steps/artifacts/docker_image.sh
+        label: 'Build Project Image'
+        key: build_project_image
+        agents:
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          machineType: n2-standard-16
+          preemptible: true
+        timeout_in_minutes: 60
+        retry:
+          automatic:
+            - exit_status: '-1'
+              limit: 3
+
+      - wait: ~
+      - command: .buildkite/scripts/steps/serverless/deploy.sh
+        label: 'Deploy Project'
+        agents:
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          machineType: n2-standard-4
+          preemptible: true
+        timeout_in_minutes: 10
+
+      - wait: ~
+
+      - command: |
+          ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts "Project deployed, see credentials at: $BUILDKITE_BUILD_URL"
+        label: Comment with job URL
+        agents:
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          machineType: n2-standard-2
+        timeout_in_minutes: 5

.buildkite/pull_requests.json

@@ -49,14 +49,15 @@
       "repoOwner": "elastic",
       "repoName": "kibana",
       "pipelineSlug": "kibana-deploy-project-from-pr",
-
+      "skip_ci_labels": [],
       "enabled": true,
       "allow_org_users": true,
       "allowed_repo_permissions": ["admin", "write"],
       "allowed_list": ["elastic-vault-github-plugin-prod[bot]"],
-      "set_commit_status": false,
+      "set_commit_status": true,
+      "commit_status_context": "kibana-deploy-project-from-pr",
       "build_on_commit": false,
-      "build_on_comment": false,
+      "build_on_comment": true,
       "build_drafts": false,
       "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:deploy)\\W+(?:project))$",
       "kibana_versions_check": true,

.buildkite/scripts/lifecycle/comment_on_pr.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { addComment } from '#pipeline-utils';
+
+const ALLOWED_ENV_VARS = [
+  'BUILDKITE_BRANCH',
+  'BUILDKITE_BUILD_ID',
+  'BUILDKITE_BUILD_NUMBER',
+  'BUILDKITE_BUILD_URL',
+  'BUILDKITE_COMMIT',
+  'BUILDKITE_PIPELINE_NAME',
+  'BUILDKITE_PIPELINE_SLUG',
+  'GITHUB_PR_BASE_OWNER',
+  'GITHUB_PR_BASE_REPO',
+  'GITHUB_PR_BRANCH',
+  'GITHUB_PR_HEAD_SHA',
+  'GITHUB_PR_HEAD_USER',
+  'GITHUB_PR_LABELS',
+  'GITHUB_PR_NUMBER',
+  'GITHUB_PR_OWNER',
+  'GITHUB_PR_REPO',
+  'GITHUB_PR_TARGET_BRANCH',
+  'GITHUB_PR_TRIGGERED_SHA',
+  'GITHUB_PR_TRIGGER_USER',
+  'GITHUB_PR_USER',
+];
+const DEFAULT_MESSAGE_TEMPLATE =
+  '🚀 Buildkite job started for PR #${GITHUB_PR_NUMBER}: ${BUILDKITE_BUILD_URL}';
+
+export function commentOnPR() {
+  const messageTemplate =
+    process.argv.slice(2)?.join(' ') ||
+    process.env.JOB_START_COMMENT_TEMPLATE ||
+    DEFAULT_MESSAGE_TEMPLATE;
+  if (messageTemplate === DEFAULT_MESSAGE_TEMPLATE) {
+    console.log('No message template provided, using default message');
+  } else {
+    console.log(`Using message template: ${messageTemplate}`);
+  }
+
+  const message = messageTemplate.replace(/\${([^}]+)}/g, (_, envVar) => {
+    if (ALLOWED_ENV_VARS.includes(envVar)) {
+      return process.env[envVar] || '';
+    } else {
+      return '${' + envVar + '}';
+    }
+  });
+
+  return addComment(message);
+}
+
+if (require.main === module) {
+  commentOnPR().catch((error) => {
+    console.error(error);
+    process.exit(1);
+  });
+}

.buildkite/scripts/steps/checks.sh

@@ -4,6 +4,7 @@ set -euo pipefail
 
 export DISABLE_BOOTSTRAP_VALIDATION=false
 .buildkite/scripts/bootstrap.sh
+.buildkite/scripts/copy_es_snapshot_cache.sh
 
 if [[ "${FTR_ENABLE_FIPS_AGENT:-}" == "true" ]]; then
   .buildkite/scripts/steps/checks/verify_fips_enabled.sh

.buildkite/scripts/steps/cloud/build_and_deploy.sh

@@ -19,7 +19,7 @@ download_artifact "kibana-$VERSION-linux-x86_64.tar.gz" ./target --build "${KIBA
 echo "--- Build Cloud Distribution"
 ELASTICSEARCH_MANIFEST_URL="https://storage.googleapis.com/kibana-ci-es-snapshots-daily/$(jq -r '.version' package.json)/manifest-latest-verified.json"
 ELASTICSEARCH_SHA=$(curl -s $ELASTICSEARCH_MANIFEST_URL | jq -r '.sha')
-ELASTICSEARCH_CLOUD_IMAGE="docker.elastic.co/kibana-ci/elasticsearch-cloud:$VERSION-$ELASTICSEARCH_SHA"
+ELASTICSEARCH_CLOUD_IMAGE="docker.elastic.co/kibana-ci/elasticsearch-cloud-ess:$VERSION-$ELASTICSEARCH_SHA"
 
 KIBANA_CLOUD_IMAGE="docker.elastic.co/kibana-ci/kibana-cloud:$VERSION-$GIT_COMMIT"
 CLOUD_DEPLOYMENT_NAME="kibana-pr-$BUILDKITE_PULL_REQUEST"

.buildkite/scripts/steps/es_snapshots/build.sh

@@ -85,11 +85,11 @@ echo "--- Create kibana-ci docker cloud image archives"
 # When we bump versions, these dependencies may not exist yet, but we don't want to
 # block the rest of the snapshot promotion process
 set +e
-./gradlew :distribution:docker:cloud-docker-export:assemble && {
-  ES_CLOUD_ID=$(docker images "docker.elastic.co/elasticsearch-ci/elasticsearch-cloud" --format "{{.ID}}")
-  ES_CLOUD_VERSION=$(docker images "docker.elastic.co/elasticsearch-ci/elasticsearch-cloud" --format "{{.Tag}}")
+./gradlew :distribution:docker:cloud-ess-docker-export:assemble && {
+  ES_CLOUD_ID=$(docker images "docker.elastic.co/elasticsearch/elasticsearch-cloud-ess" --format "{{.ID}}")
+  ES_CLOUD_VERSION=$(docker images "docker.elastic.co/elasticsearch/elasticsearch-cloud-ess" --format "{{.Tag}}")
   KIBANA_ES_CLOUD_VERSION="$ES_CLOUD_VERSION-$ELASTICSEARCH_GIT_COMMIT"
-  KIBANA_ES_CLOUD_IMAGE="docker.elastic.co/kibana-ci/elasticsearch-cloud:$KIBANA_ES_CLOUD_VERSION"
+  KIBANA_ES_CLOUD_IMAGE="docker.elastic.co/kibana-ci/elasticsearch-cloud-ess:$KIBANA_ES_CLOUD_VERSION"
   echo $ES_CLOUD_ID $ES_CLOUD_VERSION $KIBANA_ES_CLOUD_VERSION $KIBANA_ES_CLOUD_IMAGE
   docker tag "$ES_CLOUD_ID" "$KIBANA_ES_CLOUD_IMAGE"
 

.buildkite/scripts/steps/functional/defend_workflows_burn.sh

@@ -5,6 +5,7 @@ set -euo pipefail
 source .buildkite/scripts/steps/functional/common.sh
 
 .buildkite/scripts/bootstrap.sh
+.buildkite/scripts/copy_es_snapshot_cache.sh
 node scripts/build_kibana_platform_plugins.js
 
 export JOB=kibana-defend-workflows-cypress

.buildkite/scripts/steps/functional/defend_workflows_serverless_burn.sh

@@ -5,6 +5,7 @@ set -euo pipefail
 source .buildkite/scripts/steps/functional/common.sh
 
 .buildkite/scripts/bootstrap.sh
+.buildkite/scripts/copy_es_snapshot_cache.sh
 node scripts/build_kibana_platform_plugins.js
 
 export JOB=kibana-defend-workflows-serverless-cypress

.buildkite/scripts/steps/functional/exploratory_view_plugin.sh

@@ -6,6 +6,7 @@ source .buildkite/scripts/common/util.sh
 
 .buildkite/scripts/bootstrap.sh
 .buildkite/scripts/download_build_artifacts.sh
+.buildkite/scripts/copy_es_snapshot_cache.sh
 
 export JOB=kibana-observability-plugin
 

.buildkite/scripts/steps/functional/slo_plugin_e2e.sh

@@ -6,6 +6,7 @@ source .buildkite/scripts/common/util.sh
 
 .buildkite/scripts/bootstrap.sh
 .buildkite/scripts/download_build_artifacts.sh
+.buildkite/scripts/copy_es_snapshot_cache.sh
 
 export JOB=kibana-ux-plugin-synthetics
 

.buildkite/scripts/steps/functional/synthetics_plugin.sh

@@ -6,6 +6,7 @@ source .buildkite/scripts/common/util.sh
 
 .buildkite/scripts/bootstrap.sh
 .buildkite/scripts/download_build_artifacts.sh
+.buildkite/scripts/copy_es_snapshot_cache.sh
 
 export JOB=kibana-synthetics-plugin
 

.buildkite/scripts/steps/functional/uptime_plugin.sh

@@ -6,6 +6,7 @@ source .buildkite/scripts/common/util.sh
 
 .buildkite/scripts/bootstrap.sh
 .buildkite/scripts/download_build_artifacts.sh
+.buildkite/scripts/copy_es_snapshot_cache.sh
 
 export JOB=kibana-uptime-plugin
 

.buildkite/scripts/steps/functional/ux_synthetics_e2e.sh

@@ -6,6 +6,7 @@ source .buildkite/scripts/common/util.sh
 
 .buildkite/scripts/bootstrap.sh
 .buildkite/scripts/download_build_artifacts.sh
+.buildkite/scripts/copy_es_snapshot_cache.sh
 
 export JOB=kibana-ux-plugin-synthetics
 

api_docs/actions.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/actions
 title: "actions"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the actions plugin
-date: 2024-10-29
+date: 2024-10-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'actions']
 ---
 import actionsObj from './actions.devdocs.json';

api_docs/advanced_settings.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/advancedSettings
 title: "advancedSettings"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the advancedSettings plugin
-date: 2024-10-29
+date: 2024-10-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'advancedSettings']
 ---
 import advancedSettingsObj from './advanced_settings.devdocs.json';

api_docs/ai_assistant_management_selection.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/aiAssistantManagementSelection
 title: "aiAssistantManagementSelection"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the aiAssistantManagementSelection plugin
-date: 2024-10-29
+date: 2024-10-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'aiAssistantManagementSelection']
 ---
 import aiAssistantManagementSelectionObj from './ai_assistant_management_selection.devdocs.json';

api_docs/aiops.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/aiops
 title: "aiops"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the aiops plugin
-date: 2024-10-29
+date: 2024-10-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'aiops']
 ---
 import aiopsObj from './aiops.devdocs.json';

api_docs/alerting.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/alerting
 title: "alerting"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the alerting plugin
-date: 2024-10-29
+date: 2024-10-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'alerting']
 ---
 import alertingObj from './alerting.devdocs.json';

api_docs/apm.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/apm
 title: "apm"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the apm plugin
-date: 2024-10-29
+date: 2024-10-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'apm']
 ---
 import apmObj from './apm.devdocs.json';

api_docs/apm_data_access.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/apmDataAccess
 title: "apmDataAccess"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the apmDataAccess plugin
-date: 2024-10-29
+date: 2024-10-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'apmDataAccess']
 ---
 import apmDataAccessObj from './apm_data_access.devdocs.json';