From f9298511768fa57efa802df4c2869e513a0a9108 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Fri, 18 Oct 2024 19:10:24 +0100 Subject: [PATCH] [8.16] [8.x] [Entity Store] Fix and re-enable entity store integration tests (#196296) (#196656) (#196840) # Backport This will backport the following commits from `8.x` to `8.16`: - [[8.x] [Entity Store] Fix and re-enable entity store integration tests (#196296) (#196656)](https://github.com/elastic/kibana/pull/196656) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) \n\n### Questions ?\nPlease refer to the [Backport tool\ndocumentation](https://github.com/sqren/backport)\n\n\n\n---------\n\nCo-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>","sha":"cb119aca152821bb41efe0b47de398960a818c1a","branchLabelMapping":{"^v8.16.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["backport"],"number":196656,"url":"https://github.com/elastic/kibana/pull/196656","mergeCommit":{"message":"[8.x] [Entity Store] Fix and re-enable entity store integration tests (#196296) (#196656)\n\n# Backport\n\nThis will backport the following commits from `main` to `8.x`:\n- [[Entity Store] Fix and re-enable entity store integration tests\n(#196296)](https://github.com/elastic/kibana/pull/196296)\n\n\n\n### Questions ?\nPlease refer to the [Backport tool\ndocumentation](https://github.com/sqren/backport)\n\n\n\n---------\n\nCo-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>","sha":"cb119aca152821bb41efe0b47de398960a818c1a"}},"sourceBranch":"8.x","suggestedTargetBranches":[],"targetPullRequestStates":[]}] BACKPORT--> Co-authored-by: Elastic Machine --- .../get_united_definition.ts | 4 +- .../security_solution/entity_store/data.json | 8 +-- .../trial_license_complete_tier/engine.ts | 38 ++--------- .../engine_nondefault_spaces.ts | 46 ++++++------- .../entities_list.ts | 3 +- .../entity_analytics/utils/data_view.ts | 6 +- .../entity_analytics/utils/entity_store.ts | 67 ++++++++++++------- 7 files changed, 80 insertions(+), 92 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts index 6699e160634fd..32cb52a61d469 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts @@ -48,9 +48,7 @@ export const getUnitedEntityDefinition = memoize( namespace, indexPatterns, }); - }, - ({ entityType, namespace, fieldHistoryLength }: Options) => - `${entityType}-${namespace}-${fieldHistoryLength}` + } ); export const getUnitedEntityDefinitionVersion = (entityType: EntityType): string => diff --git a/x-pack/test/functional/es_archives/security_solution/entity_store/data.json b/x-pack/test/functional/es_archives/security_solution/entity_store/data.json index 28498e7cb0917..a7804bd132d20 100644 --- a/x-pack/test/functional/es_archives/security_solution/entity_store/data.json +++ b/x-pack/test/functional/es_archives/security_solution/entity_store/data.json @@ -2,7 +2,7 @@ "type": "doc", "value": { "id": "a4cf452c1e0375c3d4412cb550ad1783358468a3b3b777da4829d72c7d6fb74f", - "index": ".entities.v1.latest.ea_default_user_entity_store", + "index": ".entities.v1.latest.security_user_default", "source": { "event": { "ingested": "2024-09-11T11:26:49.706875Z" @@ -27,7 +27,7 @@ "id": "LBQAgKHGmpup0Kg9nlKmeQ==", "type": "node", "firstSeenTimestamp": "2024-09-11T10:46:00.000Z", - "definitionId": "ea_default_user_entity_store" + "definitionId": "security_user_default" } } } @@ -37,7 +37,7 @@ "type": "doc", "value": { "id": "a2cf452c1e0375c3d4412cb550bd1783358468a3b3b777da4829d72c7d6fb71f", - "index": ".entities.v1.latest.ea_default_host_entity_store", + "index": ".entities.v1.latest.security_host_default", "source": { "event": { "ingested": "2024-09-11T11:26:49.641707Z" @@ -78,7 +78,7 @@ "id": "ZXKm6GEcUJY6NHkMgPPmGQ==", "type": "node", "firstSeenTimestamp": "2024-09-11T10:46:00.000Z", - "definitionId": "ea_default_host_entity_store" + "definitionId": "security_host_default" } } } diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts index 9d7bc9989f3af..63a1530744440 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts @@ -7,26 +7,16 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../../ftr_provider_context'; -import { EntityStoreUtils, elasticAssetCheckerFactory } from '../../utils'; +import { EntityStoreUtils } from '../../utils'; import { dataViewRouteHelpersFactory } from '../../utils/data_view'; export default ({ getService }: FtrProviderContext) => { const api = getService('securitySolutionApi'); const supertest = getService('supertest'); - const { - expectTransformExists, - expectTransformNotFound, - expectEnrichPolicyExists, - expectEnrichPolicyNotFound, - expectComponentTemplateExists, - expectComponentTemplateNotFound, - expectIngestPipelineExists, - expectIngestPipelineNotFound, - } = elasticAssetCheckerFactory(getService); const utils = EntityStoreUtils(getService); - // TODO: unskip once permissions issue is resolved - describe.skip('@ess @serverless @skipInServerlessMKI Entity Store Engine APIs', () => { + // Failing: See https://github.com/elastic/kibana/issues/196526 + describe.skip('@ess @skipInServerlessMKI Entity Store Engine APIs', () => { const dataView = dataViewRouteHelpersFactory(supertest); before(async () => { @@ -45,20 +35,12 @@ export default ({ getService }: FtrProviderContext) => { it('should have installed the expected user resources', async () => { await utils.initEntityEngineForEntityType('user'); - - await expectTransformExists('entities-v1-latest-ea_default_user_entity_store'); - await expectEnrichPolicyExists('entity_store_field_retention_user_default_v1'); - await expectComponentTemplateExists(`ea_default_user_entity_store-latest@platform`); - await expectIngestPipelineExists(`ea_default_user_entity_store-latest@platform`); + await utils.expectEngineAssetsExist('user'); }); it('should have installed the expected host resources', async () => { await utils.initEntityEngineForEntityType('host'); - - await expectTransformExists('entities-v1-latest-ea_default_host_entity_store'); - await expectEnrichPolicyExists('entity_store_field_retention_host_default_v1'); - await expectComponentTemplateExists(`ea_default_host_entity_store-latest@platform`); - await expectIngestPipelineExists(`ea_default_host_entity_store-latest@platform`); + await utils.expectEngineAssetsExist('host'); }); }); @@ -188,10 +170,7 @@ export default ({ getService }: FtrProviderContext) => { }) .expect(200); - await expectTransformNotFound('entities-v1-latest-ea_default_host_entity_store'); - await expectEnrichPolicyNotFound('entity_store_field_retention_host_default_v1'); - await expectComponentTemplateNotFound(`ea_default_host_entity_store-latest@platform`); - await expectIngestPipelineNotFound(`ea_default_host_entity_store-latest@platform`); + await utils.expectEngineAssetsDoNotExist('host'); }); it('should delete the user entity engine', async () => { @@ -204,10 +183,7 @@ export default ({ getService }: FtrProviderContext) => { }) .expect(200); - await expectTransformNotFound('entities-v1-latest-ea_default_user_entity_store'); - await expectEnrichPolicyNotFound('entity_store_field_retention_user_default_v1'); - await expectComponentTemplateNotFound(`ea_default_user_entity_store-latest@platform`); - await expectIngestPipelineNotFound(`ea_default_user_entity_store-latest@platform`); + await utils.expectEngineAssetsDoNotExist('user'); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts index ee86231fe23d4..481f7aa4056f6 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts @@ -9,15 +9,19 @@ import expect from '@kbn/expect'; import { v4 as uuidv4 } from 'uuid'; import { FtrProviderContextWithSpaces } from '../../../../ftr_provider_context_with_spaces'; import { EntityStoreUtils } from '../../utils'; +import { dataViewRouteHelpersFactory } from '../../utils/data_view'; + export default ({ getService }: FtrProviderContextWithSpaces) => { const api = getService('securitySolutionApi'); const spaces = getService('spaces'); const namespace = uuidv4().substring(0, 8); - + const supertest = getService('supertest'); const utils = EntityStoreUtils(getService, namespace); - // TODO: unskip once kibana system user has entity index privileges + // Failing: See https://github.com/elastic/kibana/issues/196546 describe.skip('@ess Entity Store Engine APIs in non-default space', () => { + const dataView = dataViewRouteHelpersFactory(supertest, namespace); + before(async () => { await utils.cleanEngines(); await spaces.create({ @@ -25,9 +29,11 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { name: namespace, disabledFeatures: [], }); + await dataView.create('security-solution'); }); after(async () => { + await dataView.delete('security-solution'); await spaces.delete(namespace); }); @@ -38,18 +44,12 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { it('should have installed the expected user resources', async () => { await utils.initEntityEngineForEntityType('user'); - - const expectedTransforms = [`entities-v1-latest-ea_${namespace}_user_entity_store`]; - - await utils.expectTransformsExist(expectedTransforms); + await utils.expectEngineAssetsExist('user'); }); it('should have installed the expected host resources', async () => { await utils.initEntityEngineForEntityType('host'); - - const expectedTransforms = [`entities-v1-latest-ea_${namespace}_host_entity_store`]; - - await utils.expectTransformsExist(expectedTransforms); + await utils.expectEngineAssetsExist('host'); }); }); @@ -79,9 +79,9 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { expect(getResponse.body).to.eql({ status: 'started', type: 'host', - indexPattern: - 'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*', filter: '', + fieldHistoryLength: 10, + indexPattern: '', }); }); @@ -98,9 +98,9 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { expect(getResponse.body).to.eql({ status: 'started', type: 'user', - indexPattern: - 'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*', filter: '', + fieldHistoryLength: 10, + indexPattern: '', }); }); }); @@ -116,16 +116,16 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { { status: 'started', type: 'host', - indexPattern: - 'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*', filter: '', + fieldHistoryLength: 10, + indexPattern: '', }, { status: 'started', type: 'user', - indexPattern: - 'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*', filter: '', + fieldHistoryLength: 10, + indexPattern: '', }, ]); }); @@ -200,10 +200,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { ) .expect(200); - await utils.expectTransformNotFound( - `entities-v1-history-ea_${namespace}_host_entity_store` - ); - await utils.expectTransformNotFound(`entities-v1-latest-ea_${namespace}_host_entity_store`); + await utils.expectEngineAssetsDoNotExist('host'); }); it('should delete the user entity engine', async () => { @@ -219,10 +216,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { ) .expect(200); - await utils.expectTransformNotFound( - `entities-v1-history-ea_${namespace}_user_entity_store` - ); - await utils.expectTransformNotFound(`entities-v1-latest-ea_${namespace}_user_entity_store`); + await utils.expectEngineAssetsDoNotExist('user'); }); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entities_list.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entities_list.ts index 69f9c14d06086..9d7af16c79441 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entities_list.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entities_list.ts @@ -11,8 +11,7 @@ import { FtrProviderContext } from '../../../../ftr_provider_context'; export default ({ getService }: FtrProviderContext) => { const securitySolutionApi = getService('securitySolutionApi'); - // TODO: unskip once permissions issue is resolved - describe.skip('@ess Entity store - Entities list API', () => { + describe('@ess @skipInServerlessMKI Entity store - Entities list API', () => { describe('when the entity store is disable', () => { it("should return response with success status when the index doesn't exist", async () => { const { body } = await securitySolutionApi.listEntities({ diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/data_view.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/data_view.ts index 4eba56d3a757b..e94f7b7119ddf 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/data_view.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/data_view.ts @@ -12,7 +12,7 @@ export const dataViewRouteHelpersFactory = ( ) => ({ create: (name: string) => { return supertest - .post(`/api/data_views/data_view`) + .post(`/s/${namespace}/api/data_views/data_view`) .set('kbn-xsrf', 'foo') .send({ data_view: { @@ -26,13 +26,13 @@ export const dataViewRouteHelpersFactory = ( }, delete: (name: string) => { return supertest - .delete(`/api/data_views/data_view/${name}-${namespace}`) + .delete(`/s/${namespace}/api/data_views/data_view/${name}-${namespace}`) .set('kbn-xsrf', 'foo') .expect(200); }, updateIndexPattern: (name: string, indexPattern: string) => { return supertest - .post(`/api/data_views/data_view/${name}-${namespace}`) + .post(`/s/${namespace}/api/data_views/data_view/${name}-${namespace}`) .set('kbn-xsrf', 'foo') .send({ data_view: { diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts index 3ac171de1d4fd..24c1434b5e4a5 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts @@ -6,16 +6,27 @@ */ import { EntityType } from '@kbn/security-solution-plugin/common/api/entity_analytics/entity_store/common.gen'; - +import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../../api_integration/ftr_provider_context'; +import { elasticAssetCheckerFactory } from './elastic_asset_checker'; export const EntityStoreUtils = ( getService: FtrProviderContext['getService'], - namespace?: string + namespace: string = 'default' ) => { const api = getService('securitySolutionApi'); const es = getService('es'); const log = getService('log'); + const { + expectTransformExists, + expectTransformNotFound, + expectEnrichPolicyExists, + expectEnrichPolicyNotFound, + expectComponentTemplateExists, + expectComponentTemplateNotFound, + expectIngestPipelineExists, + expectIngestPipelineNotFound, + } = elasticAssetCheckerFactory(getService); log.debug(`EntityStoreUtils namespace: ${namespace}`); @@ -37,17 +48,24 @@ export const EntityStoreUtils = ( } }; - const initEntityEngineForEntityType = (entityType: EntityType) => { - log.info(`Initializing engine for entity type ${entityType} in namespace ${namespace}`); - return api - .initEntityEngine( - { - params: { entityType }, - body: {}, - }, - namespace - ) - .expect(200); + const initEntityEngineForEntityType = async (entityType: EntityType) => { + log.info( + `Initializing engine for entity type ${entityType} in namespace ${namespace || 'default'}` + ); + const res = await api.initEntityEngine( + { + params: { entityType }, + body: {}, + }, + namespace + ); + + if (res.status !== 200) { + log.error(`Failed to initialize engine for entity type ${entityType}`); + log.error(JSON.stringify(res.body)); + } + + expect(res.status).to.eql(200); }; const expectTransformStatus = async ( @@ -78,22 +96,25 @@ export const EntityStoreUtils = ( } }; - const expectTransformNotFound = async (transformId: string, attempts: number = 5) => { - return expectTransformStatus(transformId, false); - }; - const expectTransformExists = async (transformId: string) => { - return expectTransformStatus(transformId, true); + const expectEngineAssetsExist = async (entityType: EntityType) => { + await expectTransformExists(`entities-v1-latest-security_${entityType}_${namespace}`); + await expectEnrichPolicyExists(`entity_store_field_retention_${entityType}_${namespace}_v1`); + await expectComponentTemplateExists(`security_${entityType}_${namespace}-latest@platform`); + await expectIngestPipelineExists(`security_${entityType}_${namespace}-latest@platform`); }; - const expectTransformsExist = async (transformIds: string[]) => - Promise.all(transformIds.map((id) => expectTransformExists(id))); + const expectEngineAssetsDoNotExist = async (entityType: EntityType) => { + await expectTransformNotFound(`entities-v1-latest-security_${entityType}_${namespace}`); + await expectEnrichPolicyNotFound(`entity_store_field_retention_${entityType}_${namespace}_v1`); + await expectComponentTemplateNotFound(`security_${entityType}_${namespace}-latest@platform`); + await expectIngestPipelineNotFound(`security_${entityType}_${namespace}-latest@platform`); + }; return { cleanEngines, initEntityEngineForEntityType, expectTransformStatus, - expectTransformNotFound, - expectTransformExists, - expectTransformsExist, + expectEngineAssetsExist, + expectEngineAssetsDoNotExist, }; };