Merge branch 'main' into add-uuid-to-kb-entries-to-avoid-overwriting

.buildkite/ftr_oblt_stateful_configs.yml

@@ -50,5 +50,5 @@ enabled:
   - x-pack/test/observability_ai_assistant_functional/enterprise/config.ts
   - x-pack/test/profiling_api_integration/cloud/config.ts
   - x-pack/test/functional/apps/apm/config.ts
-  # stateful config files that run deployment-agnostic tests
+  # stateful configs that run deployment-agnostic tests
   - x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts

.buildkite/ftr_platform_stateful_configs.yml

@@ -272,6 +272,7 @@ enabled:
   - x-pack/test/functional/config.firefox.js
   - x-pack/test/functional/config.upgrade_assistant.ts
   - x-pack/test/functional_cloud/config.ts
+  - x-pack/test/functional_solution_sidenav/config.ts
   - x-pack/test/kubernetes_security/basic/config.ts
   - x-pack/test/licensing_plugin/config.public.ts
   - x-pack/test/licensing_plugin/config.ts

.buildkite/package.json

@@ -11,7 +11,7 @@
   },
   "dependencies": {
     "@octokit/rest": "^18.10.0",
-    "axios": "^1.6.3",
+    "axios": "^1.7.4",
     "globby": "^11.1.0",
     "js-yaml": "^4.1.0",
     "minimatch": "^5.0.1",

.buildkite/pipeline-resource-definitions/kibana-codeql.yml

@@ -0,0 +1,34 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-codeql
+  description: Run CodeQL
+  links:
+    - title: Pipeline link
+      url: https://buildkite.com/elastic/kibana-codeql
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-operations
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / codeql
+      description: Run CodeQL
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: "#kibana-operations-alerts"
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "false"
+      repository: elastic/kibana
+      branch_configuration: main
+      default_branch: main
+      pipeline_file: ".buildkite/pipelines/codeql/codeql.yml"
+      provider_settings:
+        trigger_mode: none
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY

.buildkite/pipeline-resource-definitions/locations.yml

@@ -44,3 +44,4 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-rule-management.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/trigger-version-dependent-jobs.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-pointer-compression.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-codeql.yml

.buildkite/pipeline-resource-definitions/scripts/fix-location-collection.ts

@@ -36,13 +36,15 @@ async function main() {
 
   const preamble = locationFileLines.slice(0, 1);
 
+  // eslint-disable-next-line @kbn/eslint/no_unsafe_js_yaml
   const locationObj = jsYaml.load(
     locationFileLines.slice(1).join('\n')
   ) as BackstageLocationResource;
   locationObj.spec.targets = pipelines.map(
     (fileName) => `${resourceDefinitionsBaseUrl}/${fileName}`
   );
 
+  // eslint-disable-next-line @kbn/eslint/no_unsafe_js_yaml
   const locationYaml = jsYaml.dump(locationObj, { lineWidth: 400 });
 
   fs.writeFileSync(locationFile, `${preamble.join('\n')}\n${locationYaml}`);

.buildkite/pipeline-utils/agent_images.ts

@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+// eslint-disable-next-line @kbn/eslint/no_unsafe_js_yaml
 import { dump } from 'js-yaml';
 import { BuildkiteClient, BuildkiteCommandStep } from './buildkite';
 

.buildkite/pipeline-utils/buildkite/client.ts

@@ -8,7 +8,10 @@
 
 import axios, { AxiosInstance } from 'axios';
 import { execSync, ExecSyncOptions } from 'child_process';
+
+// eslint-disable-next-line @kbn/eslint/no_unsafe_js_yaml
 import { dump } from 'js-yaml';
+
 import { parseLinkHeader } from './parse_link_header';
 import { Artifact } from './types/artifact';
 import { Build, BuildStatus } from './types/build';

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -10,6 +10,8 @@ import * as Fs from 'fs';
 
 import * as globby from 'globby';
 import minimatch from 'minimatch';
+
+// eslint-disable-next-line @kbn/eslint/no_unsafe_js_yaml
 import { load as loadYaml } from 'js-yaml';
 
 import { BuildkiteClient, BuildkiteStep } from '../buildkite';

.buildkite/pipelines/codeql/codeql.yml

@@ -0,0 +1,2 @@
+steps:
+  - command: echo "Placeholder"

.buildkite/pipelines/on_merge.yml

@@ -37,7 +37,7 @@ steps:
       image: family/kibana-ubuntu-2004
       imageProject: elastic-images-prod
       provider: gcp
-      machineType: n2-standard-2
+      machineType: n2-highcpu-8
       preemptible: true
     key: quick_checks
     timeout_in_minutes: 60
@@ -589,6 +589,11 @@ steps:
       preemptible: true
     artifact_paths:
       "target/plugin_so_types_snapshot.json"
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
 
   - wait: ~
     continue_on_failure: true

.buildkite/pipelines/pull_request/base.yml

@@ -23,7 +23,7 @@ steps:
   - command: .buildkite/scripts/steps/quick_checks.sh
     label: 'Quick Checks'
     agents:
-      machineType: n2-standard-2
+      machineType: n2-highcpu-8
       preemptible: true
     key: quick_checks
     timeout_in_minutes: 60

.buildkite/scripts/common/util.sh

@@ -12,7 +12,7 @@ is_pr_with_label() {
 
   IFS=',' read -ra labels <<< "${GITHUB_PR_LABELS:-}"
 
-  for label in "${labels[@]}"
+  for label in "${labels[@]:-}"
   do
     if [ "$label" == "$match" ]; then
       return

.buildkite/scripts/steps/artifacts/docker_image.sh

@@ -118,6 +118,12 @@ echo "--- Trigger image tag update"
 if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]] && [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]]; then
   cat << EOF | buildkite-agent pipeline upload
 steps:
+  - label: "Trigger cve-slo-status pipeline for $KIBANA_IMAGE"
+    trigger: cve-slo-status
+    build:
+      env:
+        CONTAINER: "$KIBANA_IMAGE"
+    soft_fail: true
   - label: ":argo: Update kibana image tag for kibana-controller using gpctl"
     branches: main
     trigger: gpctl-promote-with-e2e-tests

.buildkite/scripts/steps/checks/event_log.sh

@@ -8,7 +8,25 @@ echo --- Check Event Log Schema
 
 # event log schema is pinned to a specific version of ECS
 ECS_STABLE_VERSION=1.8
-git clone --depth 1 -b $ECS_STABLE_VERSION https://github.com/elastic/ecs.git ../ecs
+
+# we can potentially skip this check on a local env, if ../ecs is present, and modified by the developer
+if [[ "${CI:-false}" =~ ^(0|false)$ ]] && [[ -d '../ecs' ]]; then
+  LOCAL_ECS_BRANCH=$(git -C ../ecs branch --show-current)
+  if [[ "$LOCAL_ECS_BRANCH" != "$ECS_STABLE_VERSION" ]]; then
+    echo "Skipping event log schema check because ECS schema is not on $ECS_STABLE_VERSION."
+    exit 0
+  fi
+
+  TOUCHED_FILES=$(git -C ../ecs status --porcelain)
+  if [[ -n "$TOUCHED_FILES" ]]; then
+    echo "Skipping event log schema check because ECS schema files have been modified."
+    exit 0
+  fi
+
+  echo "../ecs is already cloned and @ $ECS_STABLE_VERSION"
+else
+  git clone --depth 1 -b $ECS_STABLE_VERSION https://github.com/elastic/ecs.git ../ecs
+fi
 
 node x-pack/plugins/event_log/scripts/create_schemas.js
 

.buildkite/scripts/steps/checks/native_modules.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+echo --- Check Production Native Node Modules
+node scripts/check_prod_native_modules

.buildkite/scripts/steps/checks/quick_checks.txt

@@ -0,0 +1,20 @@
+.buildkite/scripts/steps/checks/precommit_hook.sh
+.buildkite/scripts/steps/checks/ts_projects.sh
+.buildkite/scripts/steps/checks/packages.sh
+.buildkite/scripts/steps/checks/bazel_packages.sh
+.buildkite/scripts/steps/checks/verify_notice.sh
+.buildkite/scripts/steps/checks/plugin_list_docs.sh
+.buildkite/scripts/steps/checks/event_log.sh
+.buildkite/scripts/steps/checks/telemetry.sh
+.buildkite/scripts/steps/checks/jest_configs.sh
+.buildkite/scripts/steps/checks/bundle_limits.sh
+.buildkite/scripts/steps/checks/i18n.sh
+.buildkite/scripts/steps/checks/file_casing.sh
+.buildkite/scripts/steps/checks/licenses.sh
+.buildkite/scripts/steps/checks/test_projects.sh
+.buildkite/scripts/steps/checks/test_hardening.sh
+.buildkite/scripts/steps/checks/ftr_configs.sh
+.buildkite/scripts/steps/checks/yarn_deduplicate.sh
+.buildkite/scripts/steps/checks/prettier_topology.sh
+.buildkite/scripts/steps/checks/renovate.sh
+.buildkite/scripts/steps/checks/native_modules.sh

.buildkite/scripts/steps/quick_checks.sh

@@ -2,25 +2,9 @@
 
 set -euo pipefail
 
-export DISABLE_BOOTSTRAP_VALIDATION=false
-.buildkite/scripts/bootstrap.sh
+if [[ "${CI:-}" =~ ^(1|true)$ ]]; then
+  export DISABLE_BOOTSTRAP_VALIDATION=false
+  .buildkite/scripts/bootstrap.sh
+fi
 
-.buildkite/scripts/steps/checks/precommit_hook.sh
-.buildkite/scripts/steps/checks/ts_projects.sh
-.buildkite/scripts/steps/checks/packages.sh
-.buildkite/scripts/steps/checks/bazel_packages.sh
-.buildkite/scripts/steps/checks/verify_notice.sh
-.buildkite/scripts/steps/checks/plugin_list_docs.sh
-.buildkite/scripts/steps/checks/event_log.sh
-.buildkite/scripts/steps/checks/telemetry.sh
-.buildkite/scripts/steps/checks/jest_configs.sh
-.buildkite/scripts/steps/checks/bundle_limits.sh
-.buildkite/scripts/steps/checks/i18n.sh
-.buildkite/scripts/steps/checks/file_casing.sh
-.buildkite/scripts/steps/checks/licenses.sh
-.buildkite/scripts/steps/checks/test_projects.sh
-.buildkite/scripts/steps/checks/test_hardening.sh
-.buildkite/scripts/steps/checks/ftr_configs.sh
-.buildkite/scripts/steps/checks/yarn_deduplicate.sh
-.buildkite/scripts/steps/checks/prettier_topology.sh
-.buildkite/scripts/steps/checks/renovate.sh
+node scripts/quick_checks --file .buildkite/scripts/steps/checks/quick_checks.txt

.devcontainer/.env.template

@@ -0,0 +1,4 @@
+# /bin/bash or /bin/zsh (oh-my-zsh is installed by default as well)
+SHELL=/bin/bash
+# Switch to 1 to enable FIPS environment, any other value to disable
+FIPS=0

.devcontainer/Dockerfile

@@ -0,0 +1,69 @@
+FROM mcr.microsoft.com/devcontainers/base:ubuntu-22.04
+
+ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8
+ENV HOME=/home/vscode
+ENV NVM_DIR=${HOME}/nvm
+ENV NVM_VERSION=v0.39.1
+ENV KBN_DIR=/workspaces/kibana
+ENV OPENSSL_PATH=${HOME}/openssl
+# Only specific versions are FIPS certified.
+ENV OPENSSL_VERSION='3.0.8'
+
+RUN apt-get update && apt-get install -y curl git zsh locales docker.io perl make gcc xvfb
+
+RUN locale-gen en_US.UTF-8
+
+# Oh My Zsh setup
+RUN if [ ! -d "$HOME/.oh-my-zsh" ]; then \
+  sh -c "$(curl -fsSL https://raw.github.com/ohmyzsh/ohmyzsh/master/tools/install.sh)"; \
+  fi && \
+  ZSH_CUSTOM=${ZSH_CUSTOM:-~/.oh-my-zsh/custom} && \
+  if [ ! -d "$ZSH_CUSTOM/plugins/zsh-autosuggestions" ]; then \
+  git clone https://github.com/zsh-users/zsh-autosuggestions $ZSH_CUSTOM/plugins/zsh-autosuggestions; \
+  fi && \
+  sed -i 's/plugins=(git)/plugins=(git ssh-agent npm docker zsh-autosuggestions)/' /home/vscode/.zshrc
+
+# Docker-in-Docker setup
+RUN usermod -aG docker vscode
+
+# FIPS setup
+# https://github.com/openssl/openssl/blob/openssl-3.0/README-FIPS.md
+# https://www.openssl.org/docs/man3.0/man7/fips_module.html
+WORKDIR ${HOME}
+
+RUN set -e ; \
+  mkdir -p "${OPENSSL_PATH}"; \
+  curl --retry 8 -S -L -O "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz" ; \
+  curl --retry 8 -S -L -O "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz.sha256" ; \
+  echo "$(cat openssl-${OPENSSL_VERSION}.tar.gz.sha256) openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c ; \
+  tar -zxf "openssl-${OPENSSL_VERSION}.tar.gz" ; \
+  rm -rf openssl-${OPENSSL_VERSION}.tar* ; \
+  cd "${OPENSSL_PATH}-${OPENSSL_VERSION}" ; \
+  ./Configure --prefix="${OPENSSL_PATH}" --openssldir="${OPENSSL_PATH}/ssl" --libdir="${OPENSSL_PATH}/lib" shared -Wl,-rpath,${OPENSSL_PATH}/lib enable-fips; \
+  make -j $(nproc) > /dev/null ; \
+  make install > /dev/null ; \
+  rm -rf  "${OPENSSL_PATH}-${OPENSSL_VERSION}" ; \
+  chown -R 1000:1000 "${OPENSSL_PATH}";
+
+WORKDIR ${KBN_DIR}
+
+# Node and NVM setup
+COPY .node-version /tmp/
+RUN mkdir -p $NVM_DIR && \
+  curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh | bash && \
+  . "$NVM_DIR/nvm.sh" && \
+  NODE_VERSION=$(cat /tmp/.node-version) && \
+  nvm install ${NODE_VERSION} && \
+  nvm use ${NODE_VERSION} && \
+  nvm alias default ${NODE_VERSION} && \
+  npm install -g yarn && \
+  echo "source $NVM_DIR/nvm.sh" >> ${HOME}/.bashrc && \
+  echo "source $NVM_DIR/nvm.sh" >> ${HOME}/.zshrc  && \
+  chown -R 1000:1000 "${HOME}/.npm"
+
+# Reload the env everytime a new shell is opened incase the .env file changed.
+RUN echo "source $KBN_DIR/.devcontainer/scripts/env.sh" >> ${HOME}/.bashrc && \
+  echo "source $KBN_DIR/.devcontainer/scripts/env.sh" >> ${HOME}/.zshrc
+
+# This is for documentation. Ports are exposed via devcontainer.json
+EXPOSE 9200 5601 9229 9230 9231

.devcontainer/README.md

@@ -0,0 +1 @@
+See the [dev docs](https://github.com/elastic/kibana/blob/main/dev_docs/getting_started/setting_up_a_development_env.mdx#using-the-kibana-dev-container-optional) for information on using the Kibana Dev Container.

.devcontainer/config/nodejs.cnf

@@ -0,0 +1,28 @@
+##########################################################################
+##                                                                      ##
+## This OpenSSL config is only loaded when running Kibana in FIPS mode. ##
+##                                                                      ##
+## See:                                                                 ##
+## https://github.com/openssl/openssl/blob/openssl-3.0/README-FIPS.md   ##
+## https://www.openssl.org/docs/man3.0/man7/fips_module.html            ##
+##                                                                      ##
+##########################################################################
+
+nodejs_conf = nodejs_init
+.include /home/vscode/openssl/ssl/fipsmodule.cnf
+
+[nodejs_init]
+providers = provider_sect
+alg_section = algorithm_sect
+
+[provider_sect]
+default = default_sect
+# The fips section name should match the section name inside the
+# included fipsmodule.cnf.
+fips = fips_sect
+
+[default_sect]
+activate = 1
+
+[algorithm_sect]
+default_properties = fips=yes