From b28fd4b801d5661ea129ca0ad23b821e6fbab8cc Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Wed, 16 Oct 2024 10:22:18 +0100 Subject: [PATCH 1/5] [Entity Store] Fix and re-enable entity store integration tests (#196296) ## Summary We previously had to skip these tests due to permissions issues, I have now fixed and re-enabled them. (cherry picked from commit 002b757123a12172a4d5d114843bc67d063ebdf8) # Conflicts: # x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts --- .../get_united_definition.ts | 4 +- .../security_solution/entity_store/data.json | 8 +-- .../trial_license_complete_tier/engine.ts | 36 ++-------- .../engine_nondefault_spaces.ts | 47 ++++++------- .../entities_list.ts | 3 +- .../entity_analytics/utils/data_view.ts | 6 +- .../entity_analytics/utils/entity_store.ts | 67 ++++++++++++------- 7 files changed, 79 insertions(+), 92 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts index 6699e160634fd..32cb52a61d469 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts @@ -48,9 +48,7 @@ export const getUnitedEntityDefinition = memoize( namespace, indexPatterns, }); - }, - ({ entityType, namespace, fieldHistoryLength }: Options) => - `${entityType}-${namespace}-${fieldHistoryLength}` + } ); export const getUnitedEntityDefinitionVersion = (entityType: EntityType): string => diff --git a/x-pack/test/functional/es_archives/security_solution/entity_store/data.json b/x-pack/test/functional/es_archives/security_solution/entity_store/data.json index 28498e7cb0917..a7804bd132d20 100644 --- a/x-pack/test/functional/es_archives/security_solution/entity_store/data.json +++ b/x-pack/test/functional/es_archives/security_solution/entity_store/data.json @@ -2,7 +2,7 @@ "type": "doc", "value": { "id": "a4cf452c1e0375c3d4412cb550ad1783358468a3b3b777da4829d72c7d6fb74f", - "index": ".entities.v1.latest.ea_default_user_entity_store", + "index": ".entities.v1.latest.security_user_default", "source": { "event": { "ingested": "2024-09-11T11:26:49.706875Z" @@ -27,7 +27,7 @@ "id": "LBQAgKHGmpup0Kg9nlKmeQ==", "type": "node", "firstSeenTimestamp": "2024-09-11T10:46:00.000Z", - "definitionId": "ea_default_user_entity_store" + "definitionId": "security_user_default" } } } @@ -37,7 +37,7 @@ "type": "doc", "value": { "id": "a2cf452c1e0375c3d4412cb550bd1783358468a3b3b777da4829d72c7d6fb71f", - "index": ".entities.v1.latest.ea_default_host_entity_store", + "index": ".entities.v1.latest.security_host_default", "source": { "event": { "ingested": "2024-09-11T11:26:49.641707Z" @@ -78,7 +78,7 @@ "id": "ZXKm6GEcUJY6NHkMgPPmGQ==", "type": "node", "firstSeenTimestamp": "2024-09-11T10:46:00.000Z", - "definitionId": "ea_default_host_entity_store" + "definitionId": "security_host_default" } } } diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts index 9d7bc9989f3af..bf080759e9a76 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts @@ -7,26 +7,16 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../../ftr_provider_context'; -import { EntityStoreUtils, elasticAssetCheckerFactory } from '../../utils'; +import { EntityStoreUtils } from '../../utils'; import { dataViewRouteHelpersFactory } from '../../utils/data_view'; export default ({ getService }: FtrProviderContext) => { const api = getService('securitySolutionApi'); const supertest = getService('supertest'); - const { - expectTransformExists, - expectTransformNotFound, - expectEnrichPolicyExists, - expectEnrichPolicyNotFound, - expectComponentTemplateExists, - expectComponentTemplateNotFound, - expectIngestPipelineExists, - expectIngestPipelineNotFound, - } = elasticAssetCheckerFactory(getService); const utils = EntityStoreUtils(getService); // TODO: unskip once permissions issue is resolved - describe.skip('@ess @serverless @skipInServerlessMKI Entity Store Engine APIs', () => { + describe.skip('@ess @skipInServerlessMKI Entity Store Engine APIs', () => { const dataView = dataViewRouteHelpersFactory(supertest); before(async () => { @@ -45,20 +35,12 @@ export default ({ getService }: FtrProviderContext) => { it('should have installed the expected user resources', async () => { await utils.initEntityEngineForEntityType('user'); - - await expectTransformExists('entities-v1-latest-ea_default_user_entity_store'); - await expectEnrichPolicyExists('entity_store_field_retention_user_default_v1'); - await expectComponentTemplateExists(`ea_default_user_entity_store-latest@platform`); - await expectIngestPipelineExists(`ea_default_user_entity_store-latest@platform`); + await utils.expectEngineAssetsExist('user'); }); it('should have installed the expected host resources', async () => { await utils.initEntityEngineForEntityType('host'); - - await expectTransformExists('entities-v1-latest-ea_default_host_entity_store'); - await expectEnrichPolicyExists('entity_store_field_retention_host_default_v1'); - await expectComponentTemplateExists(`ea_default_host_entity_store-latest@platform`); - await expectIngestPipelineExists(`ea_default_host_entity_store-latest@platform`); + await utils.expectEngineAssetsExist('host'); }); }); @@ -188,10 +170,7 @@ export default ({ getService }: FtrProviderContext) => { }) .expect(200); - await expectTransformNotFound('entities-v1-latest-ea_default_host_entity_store'); - await expectEnrichPolicyNotFound('entity_store_field_retention_host_default_v1'); - await expectComponentTemplateNotFound(`ea_default_host_entity_store-latest@platform`); - await expectIngestPipelineNotFound(`ea_default_host_entity_store-latest@platform`); + await utils.expectEngineAssetsDoNotExist('host'); }); it('should delete the user entity engine', async () => { @@ -204,10 +183,7 @@ export default ({ getService }: FtrProviderContext) => { }) .expect(200); - await expectTransformNotFound('entities-v1-latest-ea_default_user_entity_store'); - await expectEnrichPolicyNotFound('entity_store_field_retention_user_default_v1'); - await expectComponentTemplateNotFound(`ea_default_user_entity_store-latest@platform`); - await expectIngestPipelineNotFound(`ea_default_user_entity_store-latest@platform`); + await utils.expectEngineAssetsDoNotExist('user'); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts index ee86231fe23d4..de949730d3d10 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts @@ -9,15 +9,18 @@ import expect from '@kbn/expect'; import { v4 as uuidv4 } from 'uuid'; import { FtrProviderContextWithSpaces } from '../../../../ftr_provider_context_with_spaces'; import { EntityStoreUtils } from '../../utils'; +import { dataViewRouteHelpersFactory } from '../../utils/data_view'; + export default ({ getService }: FtrProviderContextWithSpaces) => { const api = getService('securitySolutionApi'); const spaces = getService('spaces'); const namespace = uuidv4().substring(0, 8); - + const supertest = getService('supertest'); const utils = EntityStoreUtils(getService, namespace); - // TODO: unskip once kibana system user has entity index privileges - describe.skip('@ess Entity Store Engine APIs in non-default space', () => { + describe('@ess Entity Store Engine APIs in non-default space', () => { + const dataView = dataViewRouteHelpersFactory(supertest, namespace); + before(async () => { await utils.cleanEngines(); await spaces.create({ @@ -25,9 +28,11 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { name: namespace, disabledFeatures: [], }); + await dataView.create('security-solution'); }); after(async () => { + await dataView.delete('security-solution'); await spaces.delete(namespace); }); @@ -38,18 +43,12 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { it('should have installed the expected user resources', async () => { await utils.initEntityEngineForEntityType('user'); - - const expectedTransforms = [`entities-v1-latest-ea_${namespace}_user_entity_store`]; - - await utils.expectTransformsExist(expectedTransforms); + await utils.expectEngineAssetsExist('user'); }); it('should have installed the expected host resources', async () => { await utils.initEntityEngineForEntityType('host'); - - const expectedTransforms = [`entities-v1-latest-ea_${namespace}_host_entity_store`]; - - await utils.expectTransformsExist(expectedTransforms); + await utils.expectEngineAssetsExist('host'); }); }); @@ -79,9 +78,9 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { expect(getResponse.body).to.eql({ status: 'started', type: 'host', - indexPattern: - 'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*', filter: '', + fieldHistoryLength: 10, + indexPattern: '', }); }); @@ -98,9 +97,9 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { expect(getResponse.body).to.eql({ status: 'started', type: 'user', - indexPattern: - 'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*', filter: '', + fieldHistoryLength: 10, + indexPattern: '', }); }); }); @@ -116,16 +115,16 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { { status: 'started', type: 'host', - indexPattern: - 'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*', filter: '', + fieldHistoryLength: 10, + indexPattern: '', }, { status: 'started', type: 'user', - indexPattern: - 'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*', filter: '', + fieldHistoryLength: 10, + indexPattern: '', }, ]); }); @@ -200,10 +199,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { ) .expect(200); - await utils.expectTransformNotFound( - `entities-v1-history-ea_${namespace}_host_entity_store` - ); - await utils.expectTransformNotFound(`entities-v1-latest-ea_${namespace}_host_entity_store`); + await utils.expectEngineAssetsDoNotExist('host'); }); it('should delete the user entity engine', async () => { @@ -219,10 +215,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { ) .expect(200); - await utils.expectTransformNotFound( - `entities-v1-history-ea_${namespace}_user_entity_store` - ); - await utils.expectTransformNotFound(`entities-v1-latest-ea_${namespace}_user_entity_store`); + await utils.expectEngineAssetsDoNotExist('user'); }); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entities_list.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entities_list.ts index 69f9c14d06086..9d7af16c79441 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entities_list.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entities_list.ts @@ -11,8 +11,7 @@ import { FtrProviderContext } from '../../../../ftr_provider_context'; export default ({ getService }: FtrProviderContext) => { const securitySolutionApi = getService('securitySolutionApi'); - // TODO: unskip once permissions issue is resolved - describe.skip('@ess Entity store - Entities list API', () => { + describe('@ess @skipInServerlessMKI Entity store - Entities list API', () => { describe('when the entity store is disable', () => { it("should return response with success status when the index doesn't exist", async () => { const { body } = await securitySolutionApi.listEntities({ diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/data_view.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/data_view.ts index 4eba56d3a757b..e94f7b7119ddf 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/data_view.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/data_view.ts @@ -12,7 +12,7 @@ export const dataViewRouteHelpersFactory = ( ) => ({ create: (name: string) => { return supertest - .post(`/api/data_views/data_view`) + .post(`/s/${namespace}/api/data_views/data_view`) .set('kbn-xsrf', 'foo') .send({ data_view: { @@ -26,13 +26,13 @@ export const dataViewRouteHelpersFactory = ( }, delete: (name: string) => { return supertest - .delete(`/api/data_views/data_view/${name}-${namespace}`) + .delete(`/s/${namespace}/api/data_views/data_view/${name}-${namespace}`) .set('kbn-xsrf', 'foo') .expect(200); }, updateIndexPattern: (name: string, indexPattern: string) => { return supertest - .post(`/api/data_views/data_view/${name}-${namespace}`) + .post(`/s/${namespace}/api/data_views/data_view/${name}-${namespace}`) .set('kbn-xsrf', 'foo') .send({ data_view: { diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts index 3ac171de1d4fd..24c1434b5e4a5 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts @@ -6,16 +6,27 @@ */ import { EntityType } from '@kbn/security-solution-plugin/common/api/entity_analytics/entity_store/common.gen'; - +import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../../api_integration/ftr_provider_context'; +import { elasticAssetCheckerFactory } from './elastic_asset_checker'; export const EntityStoreUtils = ( getService: FtrProviderContext['getService'], - namespace?: string + namespace: string = 'default' ) => { const api = getService('securitySolutionApi'); const es = getService('es'); const log = getService('log'); + const { + expectTransformExists, + expectTransformNotFound, + expectEnrichPolicyExists, + expectEnrichPolicyNotFound, + expectComponentTemplateExists, + expectComponentTemplateNotFound, + expectIngestPipelineExists, + expectIngestPipelineNotFound, + } = elasticAssetCheckerFactory(getService); log.debug(`EntityStoreUtils namespace: ${namespace}`); @@ -37,17 +48,24 @@ export const EntityStoreUtils = ( } }; - const initEntityEngineForEntityType = (entityType: EntityType) => { - log.info(`Initializing engine for entity type ${entityType} in namespace ${namespace}`); - return api - .initEntityEngine( - { - params: { entityType }, - body: {}, - }, - namespace - ) - .expect(200); + const initEntityEngineForEntityType = async (entityType: EntityType) => { + log.info( + `Initializing engine for entity type ${entityType} in namespace ${namespace || 'default'}` + ); + const res = await api.initEntityEngine( + { + params: { entityType }, + body: {}, + }, + namespace + ); + + if (res.status !== 200) { + log.error(`Failed to initialize engine for entity type ${entityType}`); + log.error(JSON.stringify(res.body)); + } + + expect(res.status).to.eql(200); }; const expectTransformStatus = async ( @@ -78,22 +96,25 @@ export const EntityStoreUtils = ( } }; - const expectTransformNotFound = async (transformId: string, attempts: number = 5) => { - return expectTransformStatus(transformId, false); - }; - const expectTransformExists = async (transformId: string) => { - return expectTransformStatus(transformId, true); + const expectEngineAssetsExist = async (entityType: EntityType) => { + await expectTransformExists(`entities-v1-latest-security_${entityType}_${namespace}`); + await expectEnrichPolicyExists(`entity_store_field_retention_${entityType}_${namespace}_v1`); + await expectComponentTemplateExists(`security_${entityType}_${namespace}-latest@platform`); + await expectIngestPipelineExists(`security_${entityType}_${namespace}-latest@platform`); }; - const expectTransformsExist = async (transformIds: string[]) => - Promise.all(transformIds.map((id) => expectTransformExists(id))); + const expectEngineAssetsDoNotExist = async (entityType: EntityType) => { + await expectTransformNotFound(`entities-v1-latest-security_${entityType}_${namespace}`); + await expectEnrichPolicyNotFound(`entity_store_field_retention_${entityType}_${namespace}_v1`); + await expectComponentTemplateNotFound(`security_${entityType}_${namespace}-latest@platform`); + await expectIngestPipelineNotFound(`security_${entityType}_${namespace}-latest@platform`); + }; return { cleanEngines, initEntityEngineForEntityType, expectTransformStatus, - expectTransformNotFound, - expectTransformExists, - expectTransformsExist, + expectEngineAssetsExist, + expectEngineAssetsDoNotExist, }; }; From cb9ae3e75a79ea5260be2983b4cd0c3a9003fcae Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Thu, 17 Oct 2024 10:54:27 +0100 Subject: [PATCH 2/5] Update engine.ts --- .../entity_store/trial_license_complete_tier/engine.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts index bf080759e9a76..a52d6ac92685e 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts @@ -15,7 +15,7 @@ export default ({ getService }: FtrProviderContext) => { const utils = EntityStoreUtils(getService); - // TODO: unskip once permissions issue is resolved + // Failing: See https://github.com/elastic/kibana/issues/196526 describe.skip('@ess @skipInServerlessMKI Entity Store Engine APIs', () => { const dataView = dataViewRouteHelpersFactory(supertest); From 0f32ce9f48027c7acef39b3cd7e1874642e59ab1 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Thu, 17 Oct 2024 10:56:10 +0100 Subject: [PATCH 3/5] Update engine_nondefault_spaces.ts --- .../trial_license_complete_tier/engine_nondefault_spaces.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts index de949730d3d10..60b928f2214a3 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts @@ -17,8 +17,9 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { const namespace = uuidv4().substring(0, 8); const supertest = getService('supertest'); const utils = EntityStoreUtils(getService, namespace); - - describe('@ess Entity Store Engine APIs in non-default space', () => { + + // Failing: See https://github.com/elastic/kibana/issues/196546 + describe.skip('@ess Entity Store Engine APIs in non-default space', () => { const dataView = dataViewRouteHelpersFactory(supertest, namespace); before(async () => { From 72b41132f32b44544f9c292ba31f70faca476ea9 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Thu, 17 Oct 2024 13:29:47 +0100 Subject: [PATCH 4/5] Update engine_nondefault_spaces.ts --- .../trial_license_complete_tier/engine_nondefault_spaces.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts index 60b928f2214a3..481f7aa4056f6 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts @@ -17,7 +17,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { const namespace = uuidv4().substring(0, 8); const supertest = getService('supertest'); const utils = EntityStoreUtils(getService, namespace); - + // Failing: See https://github.com/elastic/kibana/issues/196546 describe.skip('@ess Entity Store Engine APIs in non-default space', () => { const dataView = dataViewRouteHelpersFactory(supertest, namespace); From 285add2d99c2e0d01c5b313ec03223ed160af237 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 17 Oct 2024 13:14:16 +0000 Subject: [PATCH 5/5] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../entity_store/trial_license_complete_tier/engine.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts index a52d6ac92685e..63a1530744440 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts @@ -15,7 +15,7 @@ export default ({ getService }: FtrProviderContext) => { const utils = EntityStoreUtils(getService); - // Failing: See https://github.com/elastic/kibana/issues/196526 + // Failing: See https://github.com/elastic/kibana/issues/196526 describe.skip('@ess @skipInServerlessMKI Entity Store Engine APIs', () => { const dataView = dataViewRouteHelpersFactory(supertest);