Merge branch 'main' into dupe-policy

.buildkite/pipeline-resource-definitions/kibana-deploy-project.yml

@@ -28,9 +28,11 @@ spec:
       pipeline_file: .buildkite/pipelines/serverless_deployment/project-build-and-deploy-pr.yml
       skip_intermediate_builds: true
       provider_settings:
+        build_pull_requests: true
         prefix_pull_request_fork_branch_names: false
         skip_pull_request_builds_for_existing_commits: true
         trigger_mode: none
+      cancel_intermediate_builds: true
       teams:
         kibana-operations:
           access_level: MANAGE_BUILD_AND_READ

.buildkite/pipeline-utils/github/github.ts

@@ -93,6 +93,26 @@ export const doAnyChangesMatch = async (
   return anyFilesMatchRequired;
 };
 
+export function addComment(
+  comment: string,
+  owner = process.env.GITHUB_PR_BASE_OWNER,
+  repo = process.env.GITHUB_PR_BASE_REPO,
+  prNumber: undefined | string | number = process.env.GITHUB_PR_NUMBER
+) {
+  if (!owner || !repo || !prNumber) {
+    throw Error(
+      "Couldn't retrieve Github PR info from environment variables in order to add a comment"
+    );
+  }
+
+  return github.issues.createComment({
+    owner,
+    repo,
+    issue_number: typeof prNumber === 'number' ? prNumber : parseInt(prNumber, 10),
+    body: comment,
+  });
+}
+
 export function getGithubClient() {
   return github;
 }

.buildkite/pipelines/serverless_deployment/project-build-and-deploy-pr.yml

@@ -1,53 +1,72 @@
-agents:
-  provider: gcp
-  image: family/kibana-ubuntu-2004
-  imageProject: elastic-images-prod
+env:
+  ELASTIC_PR_COMMENTS_ENABLED: 'true'
+  GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true'
+  GITHUB_BUILD_COMMIT_STATUS_CONTEXT: kibana-deploy-project-from-pr
 
 steps:
-  - command: .buildkite/scripts/lifecycle/pre_build.sh
-    label: Pre-Build
-    timeout_in_minutes: 10
-    agents:
-      machineType: n2-standard-2
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
-
-  - wait: ~
-
-  - command: .buildkite/scripts/steps/build_kibana.sh
-    label: Build Kibana Distribution and Plugins
-    agents:
-      machineType: n2-standard-16
-      preemptible: true
-    key: build
-    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
-    timeout_in_minutes: 90
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-
-  - wait: ~
-
-  - command: .buildkite/scripts/steps/artifacts/docker_image.sh
-    label: 'Build Project Image'
-    key: build_project_image
-    agents:
-      machineType: n2-standard-16
-      preemptible: true
-    timeout_in_minutes: 60
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-
-  - wait: ~
-
-  - command: .buildkite/scripts/steps/serverless/deploy.sh
-    label: 'Deploy Project'
-    agents:
-      machineType: n2-standard-4
-      preemptible: true
-    timeout_in_minutes: 10
+  - group: 'Project Deployment'
+    if: "build.env('GITHUB_PR_LABELS') =~ /ci:project-deploy-(elasticsearch|observability|security)/"
+
+    steps:
+      - command: .buildkite/scripts/lifecycle/pre_build.sh
+        label: Pre-Build
+        timeout_in_minutes: 10
+        agents:
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          machineType: n2-standard-2
+        retry:
+          automatic:
+            - exit_status: '*'
+              limit: 1
+
+      - command: |
+          ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts "PR Project deployment started at: $BUILDKITE_BUILD_URL"
+        label: Comment with job URL
+        agents:
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          machineType: n2-standard-2
+        timeout_in_minutes: 5
+
+      - wait: ~
+
+      - command: .buildkite/scripts/steps/artifacts/docker_image.sh
+        label: 'Build Project Image'
+        key: build_project_image
+        agents:
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          machineType: n2-standard-16
+          preemptible: true
+        timeout_in_minutes: 60
+        retry:
+          automatic:
+            - exit_status: '-1'
+              limit: 3
+
+      - wait: ~
+      - command: .buildkite/scripts/steps/serverless/deploy.sh
+        label: 'Deploy Project'
+        agents:
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          machineType: n2-standard-4
+          preemptible: true
+        timeout_in_minutes: 10
+
+      - wait: ~
+
+      - command: |
+          ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts "Project deployed, see credentials at: $BUILDKITE_BUILD_URL"
+        label: Comment with job URL
+        agents:
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          machineType: n2-standard-2
+        timeout_in_minutes: 5

.buildkite/pull_requests.json

@@ -49,14 +49,15 @@
       "repoOwner": "elastic",
       "repoName": "kibana",
       "pipelineSlug": "kibana-deploy-project-from-pr",
-
+      "skip_ci_labels": [],
       "enabled": true,
       "allow_org_users": true,
       "allowed_repo_permissions": ["admin", "write"],
       "allowed_list": ["elastic-vault-github-plugin-prod[bot]"],
-      "set_commit_status": false,
+      "set_commit_status": true,
+      "commit_status_context": "kibana-deploy-project-from-pr",
       "build_on_commit": false,
-      "build_on_comment": false,
+      "build_on_comment": true,
       "build_drafts": false,
       "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:deploy)\\W+(?:project))$",
       "kibana_versions_check": true,

.buildkite/scripts/lifecycle/comment_on_pr.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { addComment } from '#pipeline-utils';
+
+const ALLOWED_ENV_VARS = [
+  'BUILDKITE_BRANCH',
+  'BUILDKITE_BUILD_ID',
+  'BUILDKITE_BUILD_NUMBER',
+  'BUILDKITE_BUILD_URL',
+  'BUILDKITE_COMMIT',
+  'BUILDKITE_PIPELINE_NAME',
+  'BUILDKITE_PIPELINE_SLUG',
+  'GITHUB_PR_BASE_OWNER',
+  'GITHUB_PR_BASE_REPO',
+  'GITHUB_PR_BRANCH',
+  'GITHUB_PR_HEAD_SHA',
+  'GITHUB_PR_HEAD_USER',
+  'GITHUB_PR_LABELS',
+  'GITHUB_PR_NUMBER',
+  'GITHUB_PR_OWNER',
+  'GITHUB_PR_REPO',
+  'GITHUB_PR_TARGET_BRANCH',
+  'GITHUB_PR_TRIGGERED_SHA',
+  'GITHUB_PR_TRIGGER_USER',
+  'GITHUB_PR_USER',
+];
+const DEFAULT_MESSAGE_TEMPLATE =
+  '🚀 Buildkite job started for PR #${GITHUB_PR_NUMBER}: ${BUILDKITE_BUILD_URL}';
+
+export function commentOnPR() {
+  const messageTemplate =
+    process.argv.slice(2)?.join(' ') ||
+    process.env.JOB_START_COMMENT_TEMPLATE ||
+    DEFAULT_MESSAGE_TEMPLATE;
+  if (messageTemplate === DEFAULT_MESSAGE_TEMPLATE) {
+    console.log('No message template provided, using default message');
+  } else {
+    console.log(`Using message template: ${messageTemplate}`);
+  }
+
+  const message = messageTemplate.replace(/\${([^}]+)}/g, (_, envVar) => {
+    if (ALLOWED_ENV_VARS.includes(envVar)) {
+      return process.env[envVar] || '';
+    } else {
+      return '${' + envVar + '}';
+    }
+  });
+
+  return addComment(message);
+}
+
+if (require.main === module) {
+  commentOnPR().catch((error) => {
+    console.error(error);
+    process.exit(1);
+  });
+}

package.json

@@ -1017,7 +1017,7 @@
     "@langchain/langgraph": "0.0.34",
     "@langchain/openai": "^0.1.3",
     "@langtrase/trace-attributes": "^3.0.8",
-    "@launchdarkly/node-server-sdk": "^9.6.0",
+    "@launchdarkly/node-server-sdk": "^9.6.1",
     "@launchdarkly/openfeature-node-server": "^1.0.0",
     "@loaders.gl/core": "^3.4.7",
     "@loaders.gl/json": "^3.4.7",

packages/core/feature-flags/README.mdx

@@ -20,56 +20,53 @@ One example of invalid use cases are settings used during the `setup` lifecycle
 if an HTTP route is registered or not. Instead, you should always register the route, and return `404 - Not found` in the route 
 handler if the feature flag returns a _disabled_ state.  
 
+In summary, Feature flagging is best suited for
+ - Phased rollout of the features (either to a specific customer, a subset of customers, or a % of overall users)
+ - Feature experimentation
+
+Feature flagging is NOT suitable for 
+ - Applying feature availability for licensing and/or tiers
+ - Restricting or applying customer entitlement to specific GA features
+
 For a code example, refer to the [Feature Flags Example plugin](https://github.com/elastic/kibana/blob/main/examples/feature_flags_example/README.md)
 
-## Registering a feature flag
+## Key concepts
 
-> [!IMPORTANT]
-> At the moment, we follow a manual process to manage our feature flags. Refer to [this repo](https://github.com/elastic/kibana-feature-flags) to learn more about our current internal process.
-> Our goal is to achieve the _gitops_ approach detailed below. But, at the moment, it's not available, and you can skip it if you want.
+### Feature Flag
 
-Kibana follows a _gitops_ approach when managing feature flags. To declare a feature flag, add your flags definitions in
-your plugin's `server/index.ts` file:
+A config key that defines a set of [variations](#variations) that will be resolved at runtime when the app calls [the evaluation APIs](https://docs.elastic.dev/kibana-dev-docs/tutorials/feature-flags-service#evaluating-feature-flags). The variation is decided at runtime based on static binary state (ON -> `variationA` vs. OFF -> `variationB`), or via [evaluation/segmentation rules](#evaluationsegmentation-rules).
 
-```typescript
-// <plugin>/server/index.ts
-import type { FeatureFlagDefinitions } from '@kbn/core-feature-flags-server';
-import type { PluginInitializerContext } from '@kbn/core-plugins-server';
-
-export const featureFlags: FeatureFlagDefinitions = [
-  {
-    key: 'myPlugin.myCoolFeature',
-    name: 'My cool feature',
-    description: 'Enables the cool feature to auto-hide the navigation bar',
-    tags: ['my-plugin', 'my-service', 'ui'],
-    variationType: 'boolean',
-    variations: [
-      {
-        name: 'On',
-        description: 'Auto-hides the bar',
-        value: true,
-      },
-      {
-        name: 'Off',
-        description: 'Static always-on',
-        value: false,
-      },
-    ],
-  },
-  {...},
-];
-
-export async function plugin(initializerContext: PluginInitializerContext) {
-  const { FeatureFlagsExamplePlugin } = await import('./plugin');
-  return new FeatureFlagsExamplePlugin(initializerContext);
-}
-```
+### Variations
+
+All the potential values that a feature flag can return based on the [evaluation rules](#evaluationsegmentation-rules). A feature flag should define at least 2 variations: the ON and the OFF states. The typical use case sets the OFF state to match the `fallback` value provided to [the evaluation APIs](https://docs.elastic.dev/kibana-dev-docs/tutorials/feature-flags-service#evaluating-feature-flags), although it's not a hard requirement and Kibana contributors might require special configurations.
+
+### Evaluation/Segmentation rules
+
+Set of rules used to evaluate the variation to resolve, based on the [evaluation context](#evaluation-context) provided by the app.
+
+The rules can vary from a percentage rollout per evaluation context, or more complex IF...THEN filters and clauses.
+
+Refer to the [Evaluation Context guide's examples](https://github.com/elastic/kibana-feature-flags/blob/main/docs/evaluation-context.md#examples) for some typical scenarios.
+
+### Evaluation Context
+
+Kibana reports a set of properties specific to each ECH deployment/Serverless project to help define the [segmentation rules](#evaluationsegmentation-rules). A list of the currently reported context properties can be found in the [Evaluation Context guide](https://github.com/elastic/kibana-feature-flags/blob/main/docs/evaluation-context.md).
+
+### Feature Flag Code References
+
+In the Kibana repo we run a [GH Action](https://github.com/elastic/kibana/actions/workflows/launchdarkly-code-references.yml) that links existing Feature Flags to their code references. This helps us figure out which flags have been removed from the code and, which ones are still being used.
+
+> :information_source: New flags might take a few moments to be updated with their code references.
+> The job runs on every commit to the `main` branch of the [Kibana repository](https://github.com/elastic/kibana), so the wait can take from a few minutes to a few hours, depending on the Kibana repo's activity.
+
+## Registering a feature flag
 
-After merging your PR, the CI will create/update the flags in our third-party feature flags provider.
+At the moment, we follow a manual process to manage our feature flags. Refer to [this repo](https://github.com/elastic/kibana-feature-flags) to learn more about our current internal process.
+Our goal is to achieve a _gitops_ approach eventually. But, at the moment, it's not available.
 
 ### Deprecation/removal strategy
 
-When your code doesn't use the feature flag anymore, it is recommended to clean up the feature flags when possible.
+When your code doesn't use the feature flag anymore, it is recommended to clean up the feature flags.
 There are a few considerations to take into account when performing this clean-up:
 
 1. Always deprecate first, remove after

packages/kbn-esql-editor/src/esql_editor.tsx

@@ -336,7 +336,7 @@ export const ESQLEditor = memo(function ESQLEditor({
         const sources = await memoizedSources(dataViews, core).result;
         return sources;
       },
-      getFieldsFor: async ({ query: queryToExecute }: { query?: string } | undefined = {}) => {
+      getColumnsFor: async ({ query: queryToExecute }: { query?: string } | undefined = {}) => {
         if (queryToExecute) {
           // ES|QL with limit 0 returns only the columns and is more performant
           const esqlQuery = {

packages/kbn-esql-validation-autocomplete/src/__tests__/helpers.ts

@@ -56,7 +56,7 @@ export const policies = [
 
 export function getCallbackMocks() {
   return {
-    getFieldsFor: jest.fn(async ({ query }) => {
+    getColumnsFor: jest.fn(async ({ query }) => {
       if (/enrich/.test(query)) {
         return enrichFields;
       }

packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.suggest.test.ts

@@ -42,6 +42,6 @@ describe('autocomplete.suggest', () => {
     await suggest('sHoW ?');
     await suggest('row ? |');
 
-    expect(callbacks.getFieldsFor.mock.calls.length).toBe(0);
+    expect(callbacks.getColumnsFor.mock.calls.length).toBe(0);
   });
 });