[8.16] [8.x] [Entity Store] Fix and re-enable entity store integration tests (#196296) (#196656)

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts

@@ -48,9 +48,7 @@ export const getUnitedEntityDefinition = memoize(
       namespace,
       indexPatterns,
     });
-  },
-  ({ entityType, namespace, fieldHistoryLength }: Options) =>
-    `${entityType}-${namespace}-${fieldHistoryLength}`
+  }
 );
 
 export const getUnitedEntityDefinitionVersion = (entityType: EntityType): string =>

x-pack/test/functional/es_archives/security_solution/entity_store/data.json

@@ -2,7 +2,7 @@
   "type": "doc",
   "value": {
     "id": "a4cf452c1e0375c3d4412cb550ad1783358468a3b3b777da4829d72c7d6fb74f",
-    "index": ".entities.v1.latest.ea_default_user_entity_store",
+    "index": ".entities.v1.latest.security_user_default",
     "source": {
       "event": {
         "ingested": "2024-09-11T11:26:49.706875Z"
@@ -27,7 +27,7 @@
         "id": "LBQAgKHGmpup0Kg9nlKmeQ==",
         "type": "node",
         "firstSeenTimestamp": "2024-09-11T10:46:00.000Z",
-        "definitionId": "ea_default_user_entity_store"
+        "definitionId": "security_user_default"
       }
     }
   }
@@ -37,7 +37,7 @@
   "type": "doc",
   "value": {
     "id": "a2cf452c1e0375c3d4412cb550bd1783358468a3b3b777da4829d72c7d6fb71f",
-    "index": ".entities.v1.latest.ea_default_host_entity_store",
+    "index": ".entities.v1.latest.security_host_default",
     "source": {
       "event": {
         "ingested": "2024-09-11T11:26:49.641707Z"
@@ -78,7 +78,7 @@
         "id": "ZXKm6GEcUJY6NHkMgPPmGQ==",
         "type": "node",
         "firstSeenTimestamp": "2024-09-11T10:46:00.000Z",
-        "definitionId": "ea_default_host_entity_store"
+        "definitionId": "security_host_default"
       }
     }
   }

x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts

@@ -7,26 +7,16 @@
 
 import expect from '@kbn/expect';
 import { FtrProviderContext } from '../../../../ftr_provider_context';
-import { EntityStoreUtils, elasticAssetCheckerFactory } from '../../utils';
+import { EntityStoreUtils } from '../../utils';
 import { dataViewRouteHelpersFactory } from '../../utils/data_view';
 export default ({ getService }: FtrProviderContext) => {
   const api = getService('securitySolutionApi');
   const supertest = getService('supertest');
-  const {
-    expectTransformExists,
-    expectTransformNotFound,
-    expectEnrichPolicyExists,
-    expectEnrichPolicyNotFound,
-    expectComponentTemplateExists,
-    expectComponentTemplateNotFound,
-    expectIngestPipelineExists,
-    expectIngestPipelineNotFound,
-  } = elasticAssetCheckerFactory(getService);
 
   const utils = EntityStoreUtils(getService);
 
-  // TODO: unskip once permissions issue is resolved
-  describe.skip('@ess @serverless @skipInServerlessMKI Entity Store Engine APIs', () => {
+  // Failing: See https://github.com/elastic/kibana/issues/196526
+  describe.skip('@ess @skipInServerlessMKI Entity Store Engine APIs', () => {
     const dataView = dataViewRouteHelpersFactory(supertest);
 
     before(async () => {
@@ -45,20 +35,12 @@ export default ({ getService }: FtrProviderContext) => {
 
       it('should have installed the expected user resources', async () => {
         await utils.initEntityEngineForEntityType('user');
-
-        await expectTransformExists('entities-v1-latest-ea_default_user_entity_store');
-        await expectEnrichPolicyExists('entity_store_field_retention_user_default_v1');
-        await expectComponentTemplateExists(`ea_default_user_entity_store-latest@platform`);
-        await expectIngestPipelineExists(`ea_default_user_entity_store-latest@platform`);
+        await utils.expectEngineAssetsExist('user');
       });
 
       it('should have installed the expected host resources', async () => {
         await utils.initEntityEngineForEntityType('host');
-
-        await expectTransformExists('entities-v1-latest-ea_default_host_entity_store');
-        await expectEnrichPolicyExists('entity_store_field_retention_host_default_v1');
-        await expectComponentTemplateExists(`ea_default_host_entity_store-latest@platform`);
-        await expectIngestPipelineExists(`ea_default_host_entity_store-latest@platform`);
+        await utils.expectEngineAssetsExist('host');
       });
     });
 
@@ -188,10 +170,7 @@ export default ({ getService }: FtrProviderContext) => {
           })
           .expect(200);
 
-        await expectTransformNotFound('entities-v1-latest-ea_default_host_entity_store');
-        await expectEnrichPolicyNotFound('entity_store_field_retention_host_default_v1');
-        await expectComponentTemplateNotFound(`ea_default_host_entity_store-latest@platform`);
-        await expectIngestPipelineNotFound(`ea_default_host_entity_store-latest@platform`);
+        await utils.expectEngineAssetsDoNotExist('host');
       });
 
       it('should delete the user entity engine', async () => {
@@ -204,10 +183,7 @@ export default ({ getService }: FtrProviderContext) => {
           })
           .expect(200);
 
-        await expectTransformNotFound('entities-v1-latest-ea_default_user_entity_store');
-        await expectEnrichPolicyNotFound('entity_store_field_retention_user_default_v1');
-        await expectComponentTemplateNotFound(`ea_default_user_entity_store-latest@platform`);
-        await expectIngestPipelineNotFound(`ea_default_user_entity_store-latest@platform`);
+        await utils.expectEngineAssetsDoNotExist('user');
       });
     });
 

x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts

@@ -9,25 +9,31 @@ import expect from '@kbn/expect';
 import { v4 as uuidv4 } from 'uuid';
 import { FtrProviderContextWithSpaces } from '../../../../ftr_provider_context_with_spaces';
 import { EntityStoreUtils } from '../../utils';
+import { dataViewRouteHelpersFactory } from '../../utils/data_view';
+
 export default ({ getService }: FtrProviderContextWithSpaces) => {
   const api = getService('securitySolutionApi');
   const spaces = getService('spaces');
   const namespace = uuidv4().substring(0, 8);
-
+  const supertest = getService('supertest');
   const utils = EntityStoreUtils(getService, namespace);
 
-  // TODO: unskip once kibana system user has entity index privileges
+  // Failing: See https://github.com/elastic/kibana/issues/196546
   describe.skip('@ess Entity Store Engine APIs in non-default space', () => {
+    const dataView = dataViewRouteHelpersFactory(supertest, namespace);
+
     before(async () => {
       await utils.cleanEngines();
       await spaces.create({
         id: namespace,
         name: namespace,
         disabledFeatures: [],
       });
+      await dataView.create('security-solution');
     });
 
     after(async () => {
+      await dataView.delete('security-solution');
       await spaces.delete(namespace);
     });
 
@@ -38,18 +44,12 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
 
       it('should have installed the expected user resources', async () => {
         await utils.initEntityEngineForEntityType('user');
-
-        const expectedTransforms = [`entities-v1-latest-ea_${namespace}_user_entity_store`];
-
-        await utils.expectTransformsExist(expectedTransforms);
+        await utils.expectEngineAssetsExist('user');
       });
 
       it('should have installed the expected host resources', async () => {
         await utils.initEntityEngineForEntityType('host');
-
-        const expectedTransforms = [`entities-v1-latest-ea_${namespace}_host_entity_store`];
-
-        await utils.expectTransformsExist(expectedTransforms);
+        await utils.expectEngineAssetsExist('host');
       });
     });
 
@@ -79,9 +79,9 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
           expect(getResponse.body).to.eql({
             status: 'started',
             type: 'host',
-            indexPattern:
-              'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*',
             filter: '',
+            fieldHistoryLength: 10,
+            indexPattern: '',
           });
         });
 
@@ -98,9 +98,9 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
           expect(getResponse.body).to.eql({
             status: 'started',
             type: 'user',
-            indexPattern:
-              'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*',
             filter: '',
+            fieldHistoryLength: 10,
+            indexPattern: '',
           });
         });
       });
@@ -116,16 +116,16 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
             {
               status: 'started',
               type: 'host',
-              indexPattern:
-                'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*',
               filter: '',
+              fieldHistoryLength: 10,
+              indexPattern: '',
             },
             {
               status: 'started',
               type: 'user',
-              indexPattern:
-                'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*',
               filter: '',
+              fieldHistoryLength: 10,
+              indexPattern: '',
             },
           ]);
         });
@@ -200,10 +200,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
           )
           .expect(200);
 
-        await utils.expectTransformNotFound(
-          `entities-v1-history-ea_${namespace}_host_entity_store`
-        );
-        await utils.expectTransformNotFound(`entities-v1-latest-ea_${namespace}_host_entity_store`);
+        await utils.expectEngineAssetsDoNotExist('host');
       });
 
       it('should delete the user entity engine', async () => {
@@ -219,10 +216,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
           )
           .expect(200);
 
-        await utils.expectTransformNotFound(
-          `entities-v1-history-ea_${namespace}_user_entity_store`
-        );
-        await utils.expectTransformNotFound(`entities-v1-latest-ea_${namespace}_user_entity_store`);
+        await utils.expectEngineAssetsDoNotExist('user');
       });
     });
   });

x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entities_list.ts

@@ -11,8 +11,7 @@ import { FtrProviderContext } from '../../../../ftr_provider_context';
 export default ({ getService }: FtrProviderContext) => {
   const securitySolutionApi = getService('securitySolutionApi');
 
-  // TODO: unskip once permissions issue is resolved
-  describe.skip('@ess Entity store - Entities list API', () => {
+  describe('@ess @skipInServerlessMKI Entity store - Entities list API', () => {
     describe('when the entity store is disable', () => {
       it("should return response with success status when the index doesn't exist", async () => {
         const { body } = await securitySolutionApi.listEntities({

x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/data_view.ts

@@ -12,7 +12,7 @@ export const dataViewRouteHelpersFactory = (
 ) => ({
   create: (name: string) => {
     return supertest
-      .post(`/api/data_views/data_view`)
+      .post(`/s/${namespace}/api/data_views/data_view`)
       .set('kbn-xsrf', 'foo')
       .send({
         data_view: {
@@ -26,13 +26,13 @@ export const dataViewRouteHelpersFactory = (
   },
   delete: (name: string) => {
     return supertest
-      .delete(`/api/data_views/data_view/${name}-${namespace}`)
+      .delete(`/s/${namespace}/api/data_views/data_view/${name}-${namespace}`)
       .set('kbn-xsrf', 'foo')
       .expect(200);
   },
   updateIndexPattern: (name: string, indexPattern: string) => {
     return supertest
-      .post(`/api/data_views/data_view/${name}-${namespace}`)
+      .post(`/s/${namespace}/api/data_views/data_view/${name}-${namespace}`)
       .set('kbn-xsrf', 'foo')
       .send({
         data_view: {

x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts

@@ -6,16 +6,27 @@
  */
 
 import { EntityType } from '@kbn/security-solution-plugin/common/api/entity_analytics/entity_store/common.gen';
-
+import expect from '@kbn/expect';
 import { FtrProviderContext } from '../../../../api_integration/ftr_provider_context';
+import { elasticAssetCheckerFactory } from './elastic_asset_checker';
 
 export const EntityStoreUtils = (
   getService: FtrProviderContext['getService'],
-  namespace?: string
+  namespace: string = 'default'
 ) => {
   const api = getService('securitySolutionApi');
   const es = getService('es');
   const log = getService('log');
+  const {
+    expectTransformExists,
+    expectTransformNotFound,
+    expectEnrichPolicyExists,
+    expectEnrichPolicyNotFound,
+    expectComponentTemplateExists,
+    expectComponentTemplateNotFound,
+    expectIngestPipelineExists,
+    expectIngestPipelineNotFound,
+  } = elasticAssetCheckerFactory(getService);
 
   log.debug(`EntityStoreUtils namespace: ${namespace}`);
 
@@ -37,17 +48,24 @@ export const EntityStoreUtils = (
     }
   };
 
-  const initEntityEngineForEntityType = (entityType: EntityType) => {
-    log.info(`Initializing engine for entity type ${entityType} in namespace ${namespace}`);
-    return api
-      .initEntityEngine(
-        {
-          params: { entityType },
-          body: {},
-        },
-        namespace
-      )
-      .expect(200);
+  const initEntityEngineForEntityType = async (entityType: EntityType) => {
+    log.info(
+      `Initializing engine for entity type ${entityType} in namespace ${namespace || 'default'}`
+    );
+    const res = await api.initEntityEngine(
+      {
+        params: { entityType },
+        body: {},
+      },
+      namespace
+    );
+
+    if (res.status !== 200) {
+      log.error(`Failed to initialize engine for entity type ${entityType}`);
+      log.error(JSON.stringify(res.body));
+    }
+
+    expect(res.status).to.eql(200);
   };
 
   const expectTransformStatus = async (
@@ -78,22 +96,25 @@ export const EntityStoreUtils = (
     }
   };
 
-  const expectTransformNotFound = async (transformId: string, attempts: number = 5) => {
-    return expectTransformStatus(transformId, false);
-  };
-  const expectTransformExists = async (transformId: string) => {
-    return expectTransformStatus(transformId, true);
+  const expectEngineAssetsExist = async (entityType: EntityType) => {
+    await expectTransformExists(`entities-v1-latest-security_${entityType}_${namespace}`);
+    await expectEnrichPolicyExists(`entity_store_field_retention_${entityType}_${namespace}_v1`);
+    await expectComponentTemplateExists(`security_${entityType}_${namespace}-latest@platform`);
+    await expectIngestPipelineExists(`security_${entityType}_${namespace}-latest@platform`);
   };
 
-  const expectTransformsExist = async (transformIds: string[]) =>
-    Promise.all(transformIds.map((id) => expectTransformExists(id)));
+  const expectEngineAssetsDoNotExist = async (entityType: EntityType) => {
+    await expectTransformNotFound(`entities-v1-latest-security_${entityType}_${namespace}`);
+    await expectEnrichPolicyNotFound(`entity_store_field_retention_${entityType}_${namespace}_v1`);
+    await expectComponentTemplateNotFound(`security_${entityType}_${namespace}-latest@platform`);
+    await expectIngestPipelineNotFound(`security_${entityType}_${namespace}-latest@platform`);
+  };
 
   return {
     cleanEngines,
     initEntityEngineForEntityType,
     expectTransformStatus,
-    expectTransformNotFound,
-    expectTransformExists,
-    expectTransformsExist,
+    expectEngineAssetsExist,
+    expectEngineAssetsDoNotExist,
   };
 };