[8.x] [Inventory] Adding initial e2e structure (#196560) (#196808)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Inventory] Adding initial e2e structure
(#196560)](#196560)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Cauê
Marcondes","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-10-17T13:50:11Z","message":"[Inventory]
Adding initial e2e structure (#196560)\n\ncloses
https://github.com/elastic/kibana/issues/193992\r\n\r\nHow to open
cypress dashboard locally:\r\n```\r\nnode
x-pack/plugins/observability_solution/inventory/scripts/test/e2e.js
--open \r\n```\r\n\r\nHow to run cypress tests:\r\n```\r\nnode
x-pack/plugins/observability_solution/inventory/scripts/test/e2e.js\r\n```\r\n\r\nHow
to run cypress tests multiple times:\r\n```\r\nnode
x-pack/plugins/observability_solution/inventory/scripts/test/e2e.js
--server\r\nnode
x-pack/plugins/observability_solution/inventory/scripts/test/e2e.js
--runner --times=X\r\n```\r\n\r\n---------\r\n\r\nCo-authored-by:
kibanamachine
<[email protected]>","sha":"3bfa7c00181599541c924d36b593205fd5d9fed4","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","backport:prev-minor","ci:project-deploy-observability","Team:obs-ux-infra_services","v8.16.0"],"number":196560,"url":"https://github.com/elastic/kibana/pull/196560","mergeCommit":{"message":"[Inventory]
Adding initial e2e structure (#196560)\n\ncloses
https://github.com/elastic/kibana/issues/193992\r\n\r\nHow to open
cypress dashboard locally:\r\n```\r\nnode
x-pack/plugins/observability_solution/inventory/scripts/test/e2e.js
--open \r\n```\r\n\r\nHow to run cypress tests:\r\n```\r\nnode
x-pack/plugins/observability_solution/inventory/scripts/test/e2e.js\r\n```\r\n\r\nHow
to run cypress tests multiple times:\r\n```\r\nnode
x-pack/plugins/observability_solution/inventory/scripts/test/e2e.js
--server\r\nnode
x-pack/plugins/observability_solution/inventory/scripts/test/e2e.js
--runner --times=X\r\n```\r\n\r\n---------\r\n\r\nCo-authored-by:
kibanamachine
<[email protected]>","sha":"3bfa7c00181599541c924d36b593205fd5d9fed4"}},"sourceBranch":"main","suggestedTargetBranches":["8.16"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","labelRegex":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/196560","number":196560,"mergeCommit":{"message":"[Inventory]
Adding initial e2e structure (#196560)\n\ncloses
https://github.com/elastic/kibana/issues/193992\r\n\r\nHow to open
cypress dashboard locally:\r\n```\r\nnode
x-pack/plugins/observability_solution/inventory/scripts/test/e2e.js
--open \r\n```\r\n\r\nHow to run cypress tests:\r\n```\r\nnode
x-pack/plugins/observability_solution/inventory/scripts/test/e2e.js\r\n```\r\n\r\nHow
to run cypress tests multiple times:\r\n```\r\nnode
x-pack/plugins/observability_solution/inventory/scripts/test/e2e.js
--server\r\nnode
x-pack/plugins/observability_solution/inventory/scripts/test/e2e.js
--runner --times=X\r\n```\r\n\r\n---------\r\n\r\nCo-authored-by:
kibanamachine
<[email protected]>","sha":"3bfa7c00181599541c924d36b593205fd5d9fed4"}},{"branch":"8.16","label":"v8.16.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

---------

Co-authored-by: kibanamachine <[email protected]>

.buildkite/ftr_oblt_stateful_configs.yml

@@ -3,9 +3,10 @@ disabled:
   - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config_open.ts
   - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config_runner.ts
   - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config.ts
-  - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config_open.ts
   - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config_run.ts
   - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config.ts
+  - x-pack/plugins/observability_solution/inventory/e2e/ftr_config_run.ts
+  - x-pack/plugins/observability_solution/inventory/e2e/ftr_config.ts
   - x-pack/plugins/observability_solution/profiling/e2e/ftr_config_open.ts
   - x-pack/plugins/observability_solution/profiling/e2e/ftr_config_runner.ts
   - x-pack/plugins/observability_solution/profiling/e2e/ftr_config.ts

.buildkite/pipelines/flaky_tests/groups.json

@@ -23,7 +23,7 @@
     {
       "key": "cypress/security_serverless_explore",
       "name": "[Serverless] Security Solution Explore - Cypress"
-    },   
+    },
     {
       "key": "cypress/security_solution_rule_management",
       "name": "Security Solution Rule Management - Cypress"
@@ -87,6 +87,10 @@
     {
       "key": "cypress/apm_cypress",
       "name": "APM - Cypress"
+    },
+    {
+      "key": "cypress/inventory_cypress",
+      "name": "Inventory - Cypress"
     }
   ]
-}
+}

.buildkite/pipelines/on_merge_unsupported_ftrs.yml

@@ -80,3 +80,20 @@ steps:
           limit: 3
         - exit_status: '*'
           limit: 1
+
+  - command: .buildkite/scripts/steps/functional/inventory_cypress.sh
+    label: 'Inventory Cypress Tests'
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
+    depends_on: build
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1

.buildkite/pipelines/pull_request/inventory_cypress.yml

@@ -0,0 +1,17 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/inventory_cypress.sh
+    label: 'Inventory Cypress Tests'
+    agents:
+      machineType: n2-standard-4
+      preemptible: true
+    depends_on:
+      - build
+      - quick_checks
+    timeout_in_minutes: 120
+    parallelism: 1
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -77,6 +77,16 @@ const getPipeline = (filename: string, removeSteps = true) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/apm_cypress.yml'));
     }
 
+    if (
+      (await doAnyChangesMatch([
+        /^x-pack\/plugins\/observability_solution\/inventory/,
+        /^packages\/kbn-apm-synthtrace/,
+      ])) ||
+      GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/inventory_cypress.yml'));
+    }
+
     if (
       (await doAnyChangesMatch([
         /^x-pack\/plugins\/observability_solution\/observability_onboarding/,

.buildkite/scripts/steps/functional/inventory_cypress.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/download_build_artifacts.sh
+
+export JOB=kibana-inventory-onboarding-cypress
+
+echo "--- Observability Inventory Cypress Tests"
+
+cd "$XPACK_DIR"
+
+node plugins/observability_solution/inventory/scripts/test/e2e.js \
+  --kibana-install-dir "$KIBANA_BUILD_LOCATION" \

package.json

@@ -1443,6 +1443,7 @@
     "@kbn/get-repo-files": "link:packages/kbn-get-repo-files",
     "@kbn/import-locator": "link:packages/kbn-import-locator",
     "@kbn/import-resolver": "link:packages/kbn-import-resolver",
+    "@kbn/inventory-e2e": "link:x-pack/plugins/observability_solution/inventory/e2e",
     "@kbn/jest-serializers": "link:packages/kbn-jest-serializers",
     "@kbn/journeys": "link:packages/kbn-journeys",
     "@kbn/json-ast": "link:packages/kbn-json-ast",

packages/kbn-apm-synthtrace-client/src/lib/entities/container_entity.ts

@@ -15,29 +15,28 @@ class ContainerEntity extends Serializable<EntityFields> {
     super({
       ...fields,
       'entity.type': 'container',
-      'entity.definitionId': 'latest',
+      'entity.definitionId': 'builtin_containers_from_ecs_data',
+      'entity.identityFields': ['container.id'],
     });
   }
 }
 
 export function containerEntity({
   agentName,
   dataStreamType,
-  dataStreamDataset,
   containerId,
   entityId,
 }: {
   agentName: string[];
   dataStreamType: EntityDataStreamType[];
-  dataStreamDataset: string;
   containerId: string;
   entityId: string;
 }) {
   return new ContainerEntity({
     'source_data_stream.type': dataStreamType,
-    'source_data_stream.dataset': dataStreamDataset,
     'agent.name': agentName,
     'container.id': containerId,
+    'entity.displayName': containerId,
     'entity.id': entityId,
   });
 }

packages/kbn-apm-synthtrace-client/src/lib/entities/host_entity.ts

@@ -15,29 +15,28 @@ class HostEntity extends Serializable<EntityFields> {
     super({
       ...fields,
       'entity.type': 'host',
-      'entity.definitionId': 'latest',
+      'entity.definitionId': 'builtin_hosts_from_ecs_data',
+      'entity.identityFields': ['host.name'],
     });
   }
 }
 
 export function hostEntity({
   agentName,
   dataStreamType,
-  dataStreamDataset,
   hostName,
   entityId,
 }: {
   agentName: string[];
   dataStreamType: EntityDataStreamType[];
-  dataStreamDataset: string;
   hostName: string;
   entityId: string;
 }) {
   return new HostEntity({
     'source_data_stream.type': dataStreamType,
-    'source_data_stream.dataset': dataStreamDataset,
     'agent.name': agentName,
     'host.name': hostName,
+    'entity.displayName': hostName,
     'entity.id': entityId,
   });
 }

packages/kbn-apm-synthtrace-client/src/lib/entities/service_entity.ts

@@ -15,7 +15,8 @@ class ServiceEntity extends Serializable<EntityFields> {
     super({
       ...fields,
       'entity.type': 'service',
-      'entity.definitionId': 'latest',
+      'entity.definitionId': 'builtin_services_from_ecs_data',
+      'entity.identityFields': ['service.name'],
     });
   }
 }
@@ -35,6 +36,7 @@ export function serviceEntity({
 }) {
   return new ServiceEntity({
     'service.name': serviceName,
+    'entity.displayName': serviceName,
     'service.environment': environment,
     'source_data_stream.type': dataStreamType,
     'agent.name': agentName,

packages/kbn-apm-synthtrace-client/src/lib/logs/index.ts

@@ -6,8 +6,6 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
-
-import { randomInt } from 'crypto';
 import { Fields } from '../entity';
 import { Serializable } from '../serializable';
 
@@ -180,3 +178,12 @@ export const log = {
   create,
   createMinimal,
 };
+
+function randomInt(min: number, max: number) {
+  if (min > max) {
+    throw new Error('Min value must be less than or equal to max value.');
+  }
+
+  const random = Math.floor(Math.random() * (max - min + 1)) + min;
+  return random;
+}

packages/kbn-apm-synthtrace/index.ts

@@ -16,6 +16,7 @@ export { InfraSynthtraceKibanaClient } from './src/lib/infra/infra_synthtrace_ki
 export { MonitoringSynthtraceEsClient } from './src/lib/monitoring/monitoring_synthtrace_es_client';
 export { LogsSynthtraceEsClient } from './src/lib/logs/logs_synthtrace_es_client';
 export { EntitiesSynthtraceEsClient } from './src/lib/entities/entities_synthtrace_es_client';
+export { EntitiesSynthtraceKibanaClient } from './src/lib/entities/entities_synthtrace_kibana_client';
 export { SyntheticsSynthtraceEsClient } from './src/lib/synthetics/synthetics_synthtrace_es_client';
 export {
   addObserverVersionTransform,

packages/kbn-apm-synthtrace/src/cli/scenario.ts

@@ -18,7 +18,7 @@ import {
 import { Logger } from '../lib/utils/create_logger';
 import { ScenarioReturnType } from '../lib/utils/with_client';
 import { RunOptions } from './utils/parse_run_cli_flags';
-import { EntitiesSynthtraceKibanaClient } from '../lib/apm/client/entities_synthtrace_kibana_client';
+import { EntitiesSynthtraceKibanaClient } from '../lib/entities/entities_synthtrace_kibana_client';
 
 interface EsClients {
   apmEsClient: ApmSynthtraceEsClient;

packages/kbn-apm-synthtrace/src/cli/utils/get_entites_kibana_client.ts

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { EntitiesSynthtraceKibanaClient } from '../../lib/apm/client/entities_synthtrace_kibana_client';
+import { EntitiesSynthtraceKibanaClient } from '../../lib/entities/entities_synthtrace_kibana_client';
 import { Logger } from '../../lib/utils/create_logger';
 
 export function getEntitiesKibanaClient({ target, logger }: { target: string; logger: Logger }) {

packages/kbn-apm-synthtrace/src/lib/entities/entities_synthtrace_es_client.ts

@@ -17,6 +17,10 @@ import { Logger } from '../utils/create_logger';
 
 export type EntitiesSynthtraceEsClientOptions = Omit<SynthtraceEsClientOptions, 'pipeline'>;
 
+interface Pipeline {
+  includeSerialization?: boolean;
+}
+
 export class EntitiesSynthtraceEsClient extends SynthtraceEsClient<EntityFields> {
   constructor(options: { client: Client; logger: Logger } & EntitiesSynthtraceEsClientOptions) {
     super({
@@ -25,13 +29,20 @@ export class EntitiesSynthtraceEsClient extends SynthtraceEsClient<EntityFields>
     });
     this.indices = ['.entities.v1.latest.builtin*'];
   }
+
+  getDefaultPipeline({ includeSerialization }: Pipeline = { includeSerialization: true }) {
+    return entitiesPipeline({ includeSerialization });
+  }
 }
 
-function entitiesPipeline() {
+function entitiesPipeline({ includeSerialization }: Pipeline = { includeSerialization: true }) {
   return (base: Readable) => {
+    const serializationTransform = includeSerialization ? [getSerializeTransform()] : [];
+
     return pipeline(
+      // @ts-expect-error Some weird stuff here with the type definition for pipeline. We have tests!
       base,
-      getSerializeTransform(),
+      ...serializationTransform,
       lastSeenTimestampTransform(),
       getRoutingTransform(),
       getDedotTransform(),

packages/kbn-apm-synthtrace/src/lib/apm/client/entities_synthtrace_kibana_client.ts → packages/kbn-apm-synthtrace/src/lib/entities/entities_synthtrace_kibana_client.ts

@@ -8,9 +8,9 @@
  */
 
 import fetch from 'node-fetch';
-import { Logger } from '../../utils/create_logger';
-import { kibanaHeaders } from '../../shared/client_headers';
-import { getFetchAgent } from '../../../cli/utils/ssl';
+import { Logger } from '../utils/create_logger';
+import { kibanaHeaders } from '../shared/client_headers';
+import { getFetchAgent } from '../../cli/utils/ssl';
 
 interface EntityDefinitionResponse {
   definitions: Array<{ type: string; state: { installed: boolean; running: boolean } }>;

packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts

@@ -23,6 +23,10 @@ export const LogsCustom = 'logs@custom';
 
 export type LogsSynthtraceEsClientOptions = Omit<SynthtraceEsClientOptions, 'pipeline'>;
 
+interface Pipeline {
+  includeSerialization?: boolean;
+}
+
 export class LogsSynthtraceEsClient extends SynthtraceEsClient<LogDocument> {
   constructor(options: { client: Client; logger: Logger } & LogsSynthtraceEsClientOptions) {
     super({
@@ -105,13 +109,22 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient<LogDocument> {
       this.logger.error(`Custom pipeline creation failed: ${LogsCustom} - ${err.message}`);
     }
   }
+
+  getDefaultPipeline({ includeSerialization }: Pipeline = { includeSerialization: true }) {
+    return logsPipeline({ includeSerialization });
+  }
 }
 
-function logsPipeline() {
+function logsPipeline({ includeSerialization }: Pipeline = { includeSerialization: true }) {
   return (base: Readable) => {
+    const serializationTransform = includeSerialization
+      ? [getSerializeTransform<LogDocument>()]
+      : [];
+
     return pipeline(
+      // @ts-expect-error Some weird stuff here with the type definition for pipeline. We have tests!
       base,
-      getSerializeTransform<LogDocument>(),
+      ...serializationTransform,
       getRoutingTransform('logs'),
       (err: unknown) => {
         if (err) {

tsconfig.base.json

@@ -1062,6 +1062,8 @@
       "@kbn/interactive-setup-test-endpoints-plugin/*": ["test/interactive_setup_api_integration/plugins/test_endpoints/*"],
       "@kbn/interpreter": ["packages/kbn-interpreter"],
       "@kbn/interpreter/*": ["packages/kbn-interpreter/*"],
+      "@kbn/inventory-e2e": ["x-pack/plugins/observability_solution/inventory/e2e"],
+      "@kbn/inventory-e2e/*": ["x-pack/plugins/observability_solution/inventory/e2e/*"],
       "@kbn/inventory-plugin": ["x-pack/plugins/observability_solution/inventory"],
       "@kbn/inventory-plugin/*": ["x-pack/plugins/observability_solution/inventory/*"],
       "@kbn/investigate-app-plugin": ["x-pack/plugins/observability_solution/investigate_app"],