From 1148af82c84ae924df04b79379dbc11c60d77eb7 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Wed, 16 Oct 2024 16:13:44 +0100 Subject: [PATCH 1/8] adapt tests to poll status API --- .../trial_license_complete_tier/engine.ts | 19 ++- .../engine_nondefault_spaces.ts | 14 +- .../utils/elastic_asset_checker.ts | 124 ++++++++++++++---- .../entity_analytics/utils/entity_store.ts | 30 +++++ 4 files changed, 142 insertions(+), 45 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts index a7d32767f50ce..c0f0ec467e27b 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts @@ -14,8 +14,7 @@ export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const utils = EntityStoreUtils(getService); - // Failing: See https://github.com/elastic/kibana/issues/196526 - describe.skip('@ess @skipInServerlessMKI Entity Store Engine APIs', () => { + describe('@ess @skipInServerlessMKI Entity Store Engine APIs', () => { const dataView = dataViewRouteHelpersFactory(supertest); before(async () => { @@ -33,12 +32,12 @@ export default ({ getService }: FtrProviderContext) => { }); it('should have installed the expected user resources', async () => { - await utils.initEntityEngineForEntityType('user'); + await utils.initEntityEngineForEntityTypeAndWait('user'); await utils.expectEngineAssetsExist('user'); }); it('should have installed the expected host resources', async () => { - await utils.initEntityEngineForEntityType('host'); + await utils.initEntityEngineForEntityTypeAndWait('host'); await utils.expectEngineAssetsExist('host'); }); }); @@ -46,8 +45,8 @@ export default ({ getService }: FtrProviderContext) => { describe('get and list', () => { before(async () => { await Promise.all([ - utils.initEntityEngineForEntityType('host'), - utils.initEntityEngineForEntityType('user'), + utils.initEntityEngineForEntityTypeAndWait('host'), + utils.initEntityEngineForEntityTypeAndWait('user'), ]); }); @@ -118,7 +117,7 @@ export default ({ getService }: FtrProviderContext) => { describe('start and stop', () => { before(async () => { - await utils.initEntityEngineForEntityType('host'); + await utils.initEntityEngineForEntityTypeAndWait('host'); }); after(async () => { @@ -160,7 +159,7 @@ export default ({ getService }: FtrProviderContext) => { describe('delete', () => { it('should delete the host entity engine', async () => { - await utils.initEntityEngineForEntityType('host'); + await utils.initEntityEngineForEntityTypeAndWait('host'); await api .deleteEntityEngine({ @@ -173,7 +172,7 @@ export default ({ getService }: FtrProviderContext) => { }); it('should delete the user entity engine', async () => { - await utils.initEntityEngineForEntityType('user'); + await utils.initEntityEngineForEntityTypeAndWait('user'); await api .deleteEntityEngine({ @@ -188,7 +187,7 @@ export default ({ getService }: FtrProviderContext) => { describe('apply_dataview_indices', () => { before(async () => { - await utils.initEntityEngineForEntityType('host'); + await utils.initEntityEngineForEntityTypeAndWait('host'); }); after(async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts index de949730d3d10..d9cc5702f6cc5 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts @@ -42,12 +42,12 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { }); it('should have installed the expected user resources', async () => { - await utils.initEntityEngineForEntityType('user'); + await utils.initEntityEngineForEntityTypeAndWait('user'); await utils.expectEngineAssetsExist('user'); }); it('should have installed the expected host resources', async () => { - await utils.initEntityEngineForEntityType('host'); + await utils.initEntityEngineForEntityTypeAndWait('host'); await utils.expectEngineAssetsExist('host'); }); }); @@ -55,8 +55,8 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { describe('get and list', () => { before(async () => { await Promise.all([ - utils.initEntityEngineForEntityType('host'), - utils.initEntityEngineForEntityType('user'), + utils.initEntityEngineForEntityTypeAndWait('host'), + utils.initEntityEngineForEntityTypeAndWait('user'), ]); }); @@ -133,7 +133,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { describe('start and stop', () => { before(async () => { - await utils.initEntityEngineForEntityType('host'); + await utils.initEntityEngineForEntityTypeAndWait('host'); }); after(async () => { @@ -187,7 +187,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { describe('delete', () => { it('should delete the host entity engine', async () => { - await utils.initEntityEngineForEntityType('host'); + await utils.initEntityEngineForEntityTypeAndWait('host'); await api .deleteEntityEngine( @@ -203,7 +203,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { }); it('should delete the user entity engine', async () => { - await utils.initEntityEngineForEntityType('user'); + await utils.initEntityEngineForEntityTypeAndWait('user'); await api .deleteEntityEngine( diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/elastic_asset_checker.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/elastic_asset_checker.ts index 8e8635cefc26b..9d90682e41ae0 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/elastic_asset_checker.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/elastic_asset_checker.ts @@ -46,15 +46,30 @@ export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getSe } }; - const expectEnrichPolicyStatus = async (policyId: string, exists: boolean) => { - try { - await es.enrich.getPolicy({ name: policyId }); - if (!exists) { - throw new Error(`Expected enrich policy ${policyId} to not exist, but it does`); - } - } catch (e) { - if (exists) { - throw new Error(`Expected enrich policy ${policyId} to exist, but it does not: ${e}`); + const expectEnrichPolicyStatus = async ( + policyId: string, + exists: boolean, + attempts: number = 5, + delayMs: number = 2000 + ) => { + let currentAttempt = 1; + while (currentAttempt <= attempts) { + try { + await es.enrich.getPolicy({ name: policyId }); + if (!exists) { + throw new Error(`Expected enrich policy ${policyId} to not exist, but it does`); + } + return; // Enrich policy exists, exit the loop + } catch (e) { + if (currentAttempt === attempts) { + if (exists) { + throw new Error(`Expected enrich policy ${policyId} to exist, but it does not: ${e}`); + } else { + return; // Enrich policy does not exist, exit the loop + } + } + await new Promise((resolve) => setTimeout(resolve, delayMs)); + currentAttempt++; } } }; @@ -65,17 +80,32 @@ export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getSe const expectEnrichPolicyNotFound = async (policyId: string, attempts: number = 5) => expectEnrichPolicyStatus(policyId, false); - const expectComponentTemplatStatus = async (templateName: string, exists: boolean) => { - try { - await es.cluster.getComponentTemplate({ name: templateName }); - if (!exists) { - throw new Error(`Expected component template ${templateName} to not exist, but it does`); - } - } catch (e) { - if (exists) { - throw new Error( - `Expected component template ${templateName} to exist, but it does not: ${e}` - ); + const expectComponentTemplatStatus = async ( + templateName: string, + exists: boolean, + attempts: number = 5, + delayMs: number = 2000 + ) => { + let currentAttempt = 1; + while (currentAttempt <= attempts) { + try { + await es.cluster.getComponentTemplate({ name: templateName }); + if (!exists) { + throw new Error(`Expected component template ${templateName} to not exist, but it does`); + } + return; // Component template exists, exit the loop + } catch (e) { + if (currentAttempt === attempts) { + if (exists) { + throw new Error( + `Expected component template ${templateName} to exist, but it does not: ${e}` + ); + } else { + return; // Component template does not exist, exit the loop + } + } + await new Promise((resolve) => setTimeout(resolve, delayMs)); + currentAttempt++; } } }; @@ -86,24 +116,60 @@ export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getSe const expectComponentTemplateNotFound = async (templateName: string) => expectComponentTemplatStatus(templateName, false); - const expectIngestPipelineStatus = async (pipelineId: string, exists: boolean) => { + const expectIngestPipelineStatus = async ( + pipelineId: string, + exists: boolean, + attempts: number = 5, + delayMs: number = 2000 + ) => { + let currentAttempt = 1; + while (currentAttempt <= attempts) { + try { + await es.ingest.getPipeline({ id: pipelineId }); + if (!exists) { + throw new Error(`Expected ingest pipeline ${pipelineId} to not exist, but it does`); + } + return; // Ingest pipeline exists, exit the loop + } catch (e) { + if (currentAttempt === attempts) { + if (exists) { + throw new Error( + `Expected ingest pipeline ${pipelineId} to exist, but it does not: ${e}` + ); + } else { + return; // Ingest pipeline does not exist, exit the loop + } + } + await new Promise((resolve) => setTimeout(resolve, delayMs)); + currentAttempt++; + } + } + }; + + const expectIngestPipelineExists = async (pipelineId: string) => + expectIngestPipelineStatus(pipelineId, true); + + const expectIngestPipelineNotFound = async (pipelineId: string) => + expectIngestPipelineStatus(pipelineId, false); + + const expectIndexStatus = async (indexName: string, exists: boolean) => { try { - await es.ingest.getPipeline({ id: pipelineId }); + await es.indices.get({ index: indexName }); if (!exists) { - throw new Error(`Expected ingest pipeline ${pipelineId} to not exist, but it does`); + throw new Error(`Expected index ${indexName} to not exist, but it does`); } } catch (e) { if (exists) { - throw new Error(`Expected ingest pipeline ${pipelineId} to exist, but it does not: ${e}`); + throw new Error(`Expected index ${indexName} to exist, but it does not: ${e}`); } } }; - const expectIngestPipelineExists = async (pipelineId: string) => - expectIngestPipelineStatus(pipelineId, true); + const expectEntitiesIndexExists = async (entityType: string, namespace: string) => + expectIndexStatus(`.entities.v1.latest.security_${entityType}_${namespace}`, true); - const expectIngestPipelineNotFound = async (pipelineId: string) => - expectIngestPipelineStatus(pipelineId, false); + const expectEntitiesIndexNotFound = async (entityType: string, namespace: string) => + expectIndexStatus(`.entities.v1.latest.security_${entityType}_${namespace}`, false); return { expectComponentTemplateExists, @@ -112,6 +178,8 @@ export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getSe expectEnrichPolicyNotFound, expectIngestPipelineExists, expectIngestPipelineNotFound, + expectEntitiesIndexExists, + expectEntitiesIndexNotFound, expectTransformExists, expectTransformNotFound, }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts index 24c1434b5e4a5..d758775ff555d 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts @@ -26,6 +26,8 @@ export const EntityStoreUtils = ( expectComponentTemplateNotFound, expectIngestPipelineExists, expectIngestPipelineNotFound, + expectEntitiesIndexExists, + expectEntitiesIndexNotFound, } = elasticAssetCheckerFactory(getService); log.debug(`EntityStoreUtils namespace: ${namespace}`); @@ -68,6 +70,31 @@ export const EntityStoreUtils = ( expect(res.status).to.eql(200); }; + const initEntityEngineForEntityTypeAndWait = async (entityType: EntityType) => { + await initEntityEngineForEntityType(entityType); + + let attempts = 10; + let lastBody: any = null; + const delayMs = 2000; + + while (attempts > 0) { + const { body } = await api.getEntityEngine({ params: { entityType } }, namespace).expect(200); + lastBody = body; + if (body.status === 'started') { + return; + } + + await new Promise((resolve) => setTimeout(resolve, delayMs)); + attempts--; + } + + throw new Error( + `Engine for entity type ${entityType} did not start after multiple attempts, last status: ${JSON.stringify( + lastBody + )}` + ); + }; + const expectTransformStatus = async ( transformId: string, exists: boolean, @@ -101,6 +128,7 @@ export const EntityStoreUtils = ( await expectEnrichPolicyExists(`entity_store_field_retention_${entityType}_${namespace}_v1`); await expectComponentTemplateExists(`security_${entityType}_${namespace}-latest@platform`); await expectIngestPipelineExists(`security_${entityType}_${namespace}-latest@platform`); + await expectEntitiesIndexExists(entityType, namespace); }; const expectEngineAssetsDoNotExist = async (entityType: EntityType) => { @@ -108,11 +136,13 @@ export const EntityStoreUtils = ( await expectEnrichPolicyNotFound(`entity_store_field_retention_${entityType}_${namespace}_v1`); await expectComponentTemplateNotFound(`security_${entityType}_${namespace}-latest@platform`); await expectIngestPipelineNotFound(`security_${entityType}_${namespace}-latest@platform`); + await expectEntitiesIndexNotFound(entityType, namespace); }; return { cleanEngines, initEntityEngineForEntityType, + initEntityEngineForEntityTypeAndWait, expectTransformStatus, expectEngineAssetsExist, expectEngineAssetsDoNotExist, From a4afabcdb23a8b2d01b7dd800c1c1c36b4f3a006 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Wed, 16 Oct 2024 16:15:24 +0100 Subject: [PATCH 2/8] unskip spaces tests --- .../trial_license_complete_tier/engine_nondefault_spaces.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts index cdd34c86e943b..d9cc5702f6cc5 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts @@ -18,8 +18,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { const supertest = getService('supertest'); const utils = EntityStoreUtils(getService, namespace); - // Failing: See https://github.com/elastic/kibana/issues/196546 - describe.skip('@ess Entity Store Engine APIs in non-default space', () => { + describe('@ess Entity Store Engine APIs in non-default space', () => { const dataView = dataViewRouteHelpersFactory(supertest, namespace); before(async () => { From 48ec236d7d28fd8ddc2fde1a7a880a8297b90364 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Wed, 16 Oct 2024 20:15:19 +0100 Subject: [PATCH 3/8] retry on 429 when deleting enrich policy --- .../elasticsearch_assets/enrich_policy.ts | 30 +++++++++++++++++-- .../entity_store/entity_store_data_client.ts | 1 + 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/enrich_policy.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/enrich_policy.ts index 7d6fc6fd8bc24..4b8ce594a6cb7 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/enrich_policy.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/enrich_policy.ts @@ -72,10 +72,36 @@ export const executeFieldRetentionEnrichPolicy = async ({ export const deleteFieldRetentionEnrichPolicy = async ({ unitedDefinition, esClient, + logger, + attempts = 5, + delayMs = 2000, }: { - esClient: ElasticsearchClient; unitedDefinition: DefinitionMetadata; + esClient: ElasticsearchClient; + logger: Logger; + attempts?: number; + delayMs?: number; }) => { const name = getFieldRetentionEnrichPolicyName(unitedDefinition); - return esClient.enrich.deletePolicy({ name }, { ignore: [404] }); + let currentAttempt = 1; + while (currentAttempt <= attempts) { + try { + await esClient.enrich.deletePolicy({ name }, { ignore: [404] }); + return; + } catch (e) { + // a 429 status code indicates that the enrich policy is being executed + if (currentAttempt === attempts || e.statusCode !== 429) { + logger.error( + `Error deleting enrich policy ${name}: ${e.message} after ${currentAttempt} attempts` + ); + throw e; + } + + logger.info( + `Enrich policy ${name} is being executed, waiting for it to finish before deleting` + ); + await new Promise((resolve) => setTimeout(resolve, delayMs)); + currentAttempt++; + } + } }; diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts index d2e21a1d10903..dcce6a2be837c 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts @@ -358,6 +358,7 @@ export class EntityStoreDataClient { await deleteFieldRetentionEnrichPolicy({ unitedDefinition, esClient, + logger, }); if (deleteData) { From 29de6fac6db0e61c01c10f883993663d63da39b0 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Wed, 16 Oct 2024 20:37:05 +0100 Subject: [PATCH 4/8] log which entity store was initiailised --- .../entity_analytics/entity_store/entity_store_data_client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts index dcce6a2be837c..835c12d6f8f85 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts @@ -242,7 +242,7 @@ export class EntityStoreDataClient { logger, taskManager, }); - logger.info(`Entity store initialized`); + logger.info(`Entity store initialized for ${entityType}`); return updated; } catch (err) { From 5aea6d6f6aea85ed2e6b527608c77ef65cd88f43 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Wed, 16 Oct 2024 20:41:25 +0100 Subject: [PATCH 5/8] add initEntityEngineForEntityTypesAndWait util --- .../trial_license_complete_tier/engine.ts | 5 +--- .../engine_nondefault_spaces.ts | 5 +--- .../entity_analytics/utils/entity_store.ts | 24 +++++++++++++++++++ 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts index c0f0ec467e27b..646a950175976 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts @@ -44,10 +44,7 @@ export default ({ getService }: FtrProviderContext) => { describe('get and list', () => { before(async () => { - await Promise.all([ - utils.initEntityEngineForEntityTypeAndWait('host'), - utils.initEntityEngineForEntityTypeAndWait('user'), - ]); + await utils.initEntityEngineForEntityTypesAndWait(['host', 'user']); }); after(async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts index d9cc5702f6cc5..d0029b0098a9f 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts @@ -54,10 +54,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { describe('get and list', () => { before(async () => { - await Promise.all([ - utils.initEntityEngineForEntityTypeAndWait('host'), - utils.initEntityEngineForEntityTypeAndWait('user'), - ]); + await utils.initEntityEngineForEntityTypesAndWait(['host', 'user']); }); after(async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts index d758775ff555d..04daff1c85378 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts @@ -95,6 +95,29 @@ export const EntityStoreUtils = ( ); }; + const initEntityEngineForEntityTypesAndWait = async (entityTypes: EntityType[]) => { + await Promise.all(entityTypes.map((entityType) => initEntityEngineForEntityType(entityType))); + + let attempts = 10; + let lastBody: any = null; + const delayMs = 2000; + + while (attempts > 0) { + const { body } = await api.listEntityEngines(namespace).expect(200); + lastBody = body; + if (body.engines.every((engine: any) => engine.status === 'started')) { + return; + } + + await new Promise((resolve) => setTimeout(resolve, delayMs)); + attempts--; + } + + throw new Error( + `Engines did not start after multiple attempts, last status: ${JSON.stringify(lastBody)}` + ); + }; + const expectTransformStatus = async ( transformId: string, exists: boolean, @@ -143,6 +166,7 @@ export const EntityStoreUtils = ( cleanEngines, initEntityEngineForEntityType, initEntityEngineForEntityTypeAndWait, + initEntityEngineForEntityTypesAndWait, expectTransformStatus, expectEngineAssetsExist, expectEngineAssetsDoNotExist, From 28fa26f1f497a2ea0e059fef4fd133f049084df0 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Fri, 18 Oct 2024 09:58:36 +0100 Subject: [PATCH 6/8] fix phrasing --- .../entity_analytics/entity_store/entity_store_data_client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts index 08f4926e68f7c..50e500fae40f2 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts @@ -451,7 +451,7 @@ export class EntityStoreDataClient { originalStatus === ENGINE_STATUS.UPDATING ) { throw new Error( - `Error updating entity store: There is an changes already in progress for engine ${id}` + `Error updating entity store: There are changes already in progress for engine ${id}` ); } From 2705c2d1b744ebd766fc2baa607bae25ae745cde Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Fri, 18 Oct 2024 09:59:26 +0100 Subject: [PATCH 7/8] use retry util --- .../trial_license_complete_tier/engine.ts | 12 +- .../engine_nondefault_spaces.ts | 10 +- .../utils/elastic_asset_checker.ts | 153 +++++++----------- .../entity_analytics/utils/entity_store.ts | 63 ++------ 4 files changed, 83 insertions(+), 155 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts index 646a950175976..f51fbd15ceead 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts @@ -32,12 +32,12 @@ export default ({ getService }: FtrProviderContext) => { }); it('should have installed the expected user resources', async () => { - await utils.initEntityEngineForEntityTypeAndWait('user'); + await utils.initEntityEngineForEntityTypesAndWait(['user']); await utils.expectEngineAssetsExist('user'); }); it('should have installed the expected host resources', async () => { - await utils.initEntityEngineForEntityTypeAndWait('host'); + await utils.initEntityEngineForEntityTypesAndWait(['host']); await utils.expectEngineAssetsExist('host'); }); }); @@ -114,7 +114,7 @@ export default ({ getService }: FtrProviderContext) => { describe('start and stop', () => { before(async () => { - await utils.initEntityEngineForEntityTypeAndWait('host'); + await utils.initEntityEngineForEntityTypesAndWait(['host']); }); after(async () => { @@ -156,7 +156,7 @@ export default ({ getService }: FtrProviderContext) => { describe('delete', () => { it('should delete the host entity engine', async () => { - await utils.initEntityEngineForEntityTypeAndWait('host'); + await utils.initEntityEngineForEntityTypesAndWait(['host']); await api .deleteEntityEngine({ @@ -169,7 +169,7 @@ export default ({ getService }: FtrProviderContext) => { }); it('should delete the user entity engine', async () => { - await utils.initEntityEngineForEntityTypeAndWait('user'); + await utils.initEntityEngineForEntityTypesAndWait(['user']); await api .deleteEntityEngine({ @@ -184,7 +184,7 @@ export default ({ getService }: FtrProviderContext) => { describe('apply_dataview_indices', () => { before(async () => { - await utils.initEntityEngineForEntityTypeAndWait('host'); + await utils.initEntityEngineForEntityTypesAndWait(['host']); }); after(async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts index d0029b0098a9f..64809533fec7b 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts @@ -42,12 +42,12 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { }); it('should have installed the expected user resources', async () => { - await utils.initEntityEngineForEntityTypeAndWait('user'); + await utils.initEntityEngineForEntityTypesAndWait(['user']); await utils.expectEngineAssetsExist('user'); }); it('should have installed the expected host resources', async () => { - await utils.initEntityEngineForEntityTypeAndWait('host'); + await utils.initEntityEngineForEntityTypesAndWait(['host']); await utils.expectEngineAssetsExist('host'); }); }); @@ -130,7 +130,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { describe('start and stop', () => { before(async () => { - await utils.initEntityEngineForEntityTypeAndWait('host'); + await utils.initEntityEngineForEntityTypesAndWait(['host']); }); after(async () => { @@ -184,7 +184,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { describe('delete', () => { it('should delete the host entity engine', async () => { - await utils.initEntityEngineForEntityTypeAndWait('host'); + await utils.initEntityEngineForEntityTypesAndWait(['host']); await api .deleteEntityEngine( @@ -200,7 +200,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => { }); it('should delete the user entity engine', async () => { - await utils.initEntityEngineForEntityTypeAndWait('user'); + await utils.initEntityEngineForEntityTypesAndWait(['user']); await api .deleteEntityEngine( diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/elastic_asset_checker.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/elastic_asset_checker.ts index 9d90682e41ae0..c0dddb4ddb093 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/elastic_asset_checker.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/elastic_asset_checker.ts @@ -9,6 +9,8 @@ import { FtrProviderContext } from '@kbn/ftr-common-functional-services'; export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getService']) => { const es = getService('es'); + const retry = getService('retry'); + const log = getService('log'); const expectTransformExists = async (transformId: string) => { return expectTransformStatus(transformId, true); @@ -18,60 +20,43 @@ export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getSe return expectTransformStatus(transformId, false); }; - const expectTransformStatus = async ( - transformId: string, - exists: boolean, - attempts: number = 5, - delayMs: number = 2000 - ) => { - let currentAttempt = 1; - while (currentAttempt <= attempts) { - try { - await es.transform.getTransform({ transform_id: transformId }); - if (!exists) { - throw new Error(`Expected transform ${transformId} to not exist, but it does`); + const expectTransformStatus = async (transformId: string, exists: boolean) => { + await retry.waitForWithTimeout( + `transform ${transformId} to ${exists ? 'exist' : 'not exist'}`, + 10_000, + async () => { + try { + await es.transform.getTransform({ transform_id: transformId }); + return exists; + } catch (e) { + log.debug(`Transform ${transformId} not found: ${e}`); + return !exists; } - return; // Transform exists, exit the loop - } catch (e) { - if (currentAttempt === attempts) { - if (exists) { - throw new Error(`Expected transform ${transformId} to exist, but it does not: ${e}`); - } else { - return; // Transform does not exist, exit the loop - } - } - await new Promise((resolve) => setTimeout(resolve, delayMs)); - currentAttempt++; } - } + ); }; - const expectEnrichPolicyStatus = async ( - policyId: string, - exists: boolean, - attempts: number = 5, - delayMs: number = 2000 - ) => { - let currentAttempt = 1; - while (currentAttempt <= attempts) { - try { - await es.enrich.getPolicy({ name: policyId }); - if (!exists) { - throw new Error(`Expected enrich policy ${policyId} to not exist, but it does`); - } - return; // Enrich policy exists, exit the loop - } catch (e) { - if (currentAttempt === attempts) { - if (exists) { - throw new Error(`Expected enrich policy ${policyId} to exist, but it does not: ${e}`); + const expectEnrichPolicyStatus = async (policyId: string, exists: boolean) => { + await retry.waitForWithTimeout( + `enrich policy ${policyId} to ${exists ? 'exist' : 'not exist'}`, + 20_000, + async () => { + try { + const res = await es.enrich.getPolicy({ name: policyId }); + const policy = res.policies?.[0]; + if (policy) { + log.debug(`Enrich policy ${policyId} found: ${JSON.stringify(res)}`); + return exists; } else { - return; // Enrich policy does not exist, exit the loop + log.debug(`Enrich policy ${policyId} not found: ${JSON.stringify(res)}`); + return !exists; } + } catch (e) { + log.debug(`Enrich policy ${policyId} not found: ${e}`); + return !exists; } - await new Promise((resolve) => setTimeout(resolve, delayMs)); - currentAttempt++; } - } + ); }; const expectEnrichPolicyExists = async (policyId: string) => @@ -80,34 +65,20 @@ export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getSe const expectEnrichPolicyNotFound = async (policyId: string, attempts: number = 5) => expectEnrichPolicyStatus(policyId, false); - const expectComponentTemplatStatus = async ( - templateName: string, - exists: boolean, - attempts: number = 5, - delayMs: number = 2000 - ) => { - let currentAttempt = 1; - while (currentAttempt <= attempts) { - try { - await es.cluster.getComponentTemplate({ name: templateName }); - if (!exists) { - throw new Error(`Expected component template ${templateName} to not exist, but it does`); - } - return; // Component template exists, exit the loop - } catch (e) { - if (currentAttempt === attempts) { - if (exists) { - throw new Error( - `Expected component template ${templateName} to exist, but it does not: ${e}` - ); - } else { - return; // Component template does not exist, exit the loop - } + const expectComponentTemplatStatus = async (templateName: string, exists: boolean) => { + await retry.waitForWithTimeout( + `component template ${templateName} to ${exists ? 'exist' : 'not exist'}`, + 10_000, + async () => { + try { + await es.cluster.getComponentTemplate({ name: templateName }); + return exists; // Component template exists + } catch (e) { + log.debug(`Component template ${templateName} not found: ${e}`); + return !exists; // Component template does not exist } - await new Promise((resolve) => setTimeout(resolve, delayMs)); - currentAttempt++; } - } + ); }; const expectComponentTemplateExists = async (templateName: string) => @@ -116,34 +87,20 @@ export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getSe const expectComponentTemplateNotFound = async (templateName: string) => expectComponentTemplatStatus(templateName, false); - const expectIngestPipelineStatus = async ( - pipelineId: string, - exists: boolean, - attempts: number = 5, - delayMs: number = 2000 - ) => { - let currentAttempt = 1; - while (currentAttempt <= attempts) { - try { - await es.ingest.getPipeline({ id: pipelineId }); - if (!exists) { - throw new Error(`Expected ingest pipeline ${pipelineId} to not exist, but it does`); + const expectIngestPipelineStatus = async (pipelineId: string, exists: boolean) => { + await retry.waitForWithTimeout( + `ingest pipeline ${pipelineId} to ${exists ? 'exist' : 'not exist'}`, + 10_000, + async () => { + try { + await es.ingest.getPipeline({ id: pipelineId }); + return exists; // Ingest pipeline exists + } catch (e) { + log.debug(`Ingest pipeline ${pipelineId} not found: ${e}`); + return !exists; // Ingest pipeline does not exist } - return; // Ingest pipeline exists, exit the loop - } catch (e) { - if (currentAttempt === attempts) { - if (exists) { - throw new Error( - `Expected ingest pipeline ${pipelineId} to exist, but it does not: ${e}` - ); - } else { - return; // Ingest pipeline does not exist, exit the loop - } - } - await new Promise((resolve) => setTimeout(resolve, delayMs)); - currentAttempt++; } - } + ); }; const expectIngestPipelineExists = async (pipelineId: string) => diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts index 04daff1c85378..25013519ae4a4 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts @@ -17,6 +17,7 @@ export const EntityStoreUtils = ( const api = getService('securitySolutionApi'); const es = getService('es'); const log = getService('log'); + const retry = getService('retry'); const { expectTransformExists, expectTransformNotFound, @@ -50,7 +51,7 @@ export const EntityStoreUtils = ( } }; - const initEntityEngineForEntityType = async (entityType: EntityType) => { + const _initEntityEngineForEntityType = async (entityType: EntityType) => { log.info( `Initializing engine for entity type ${entityType} in namespace ${namespace || 'default'}` ); @@ -70,51 +71,19 @@ export const EntityStoreUtils = ( expect(res.status).to.eql(200); }; - const initEntityEngineForEntityTypeAndWait = async (entityType: EntityType) => { - await initEntityEngineForEntityType(entityType); - - let attempts = 10; - let lastBody: any = null; - const delayMs = 2000; - - while (attempts > 0) { - const { body } = await api.getEntityEngine({ params: { entityType } }, namespace).expect(200); - lastBody = body; - if (body.status === 'started') { - return; - } - - await new Promise((resolve) => setTimeout(resolve, delayMs)); - attempts--; - } - - throw new Error( - `Engine for entity type ${entityType} did not start after multiple attempts, last status: ${JSON.stringify( - lastBody - )}` - ); - }; - const initEntityEngineForEntityTypesAndWait = async (entityTypes: EntityType[]) => { await Promise.all(entityTypes.map((entityType) => initEntityEngineForEntityType(entityType))); - let attempts = 10; - let lastBody: any = null; - const delayMs = 2000; - - while (attempts > 0) { - const { body } = await api.listEntityEngines(namespace).expect(200); - lastBody = body; - if (body.engines.every((engine: any) => engine.status === 'started')) { - return; + await retry.waitForWithTimeout( + `Engines to start for entity types: ${entityTypes.join(', ')}`, + 60_000, + async () => { + const { body } = await api.listEntityEngines(namespace).expect(200); + if (body.engines.every((engine: any) => engine.status === 'started')) { + return true; + } + return false; } - - await new Promise((resolve) => setTimeout(resolve, delayMs)); - attempts--; - } - - throw new Error( - `Engines did not start after multiple attempts, last status: ${JSON.stringify(lastBody)}` ); }; @@ -148,7 +117,9 @@ export const EntityStoreUtils = ( const expectEngineAssetsExist = async (entityType: EntityType) => { await expectTransformExists(`entities-v1-latest-security_${entityType}_${namespace}`); - await expectEnrichPolicyExists(`entity_store_field_retention_${entityType}_${namespace}_v1`); + await expectEnrichPolicyExists( + `entity_store_field_retention_${entityType}_${namespace}_v1.0.0` + ); await expectComponentTemplateExists(`security_${entityType}_${namespace}-latest@platform`); await expectIngestPipelineExists(`security_${entityType}_${namespace}-latest@platform`); await expectEntitiesIndexExists(entityType, namespace); @@ -156,7 +127,9 @@ export const EntityStoreUtils = ( const expectEngineAssetsDoNotExist = async (entityType: EntityType) => { await expectTransformNotFound(`entities-v1-latest-security_${entityType}_${namespace}`); - await expectEnrichPolicyNotFound(`entity_store_field_retention_${entityType}_${namespace}_v1`); + await expectEnrichPolicyNotFound( + `entity_store_field_retention_${entityType}_${namespace}_v1.0.0` + ); await expectComponentTemplateNotFound(`security_${entityType}_${namespace}-latest@platform`); await expectIngestPipelineNotFound(`security_${entityType}_${namespace}-latest@platform`); await expectEntitiesIndexNotFound(entityType, namespace); @@ -164,8 +137,6 @@ export const EntityStoreUtils = ( return { cleanEngines, - initEntityEngineForEntityType, - initEntityEngineForEntityTypeAndWait, initEntityEngineForEntityTypesAndWait, expectTransformStatus, expectEngineAssetsExist, From 77a3349a16effe2a0ab6f61507f43c8ba574fe2d Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Fri, 18 Oct 2024 10:18:09 +0100 Subject: [PATCH 8/8] fix reference error --- .../test_suites/entity_analytics/utils/entity_store.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts index 25013519ae4a4..029103425af68 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts @@ -72,7 +72,7 @@ export const EntityStoreUtils = ( }; const initEntityEngineForEntityTypesAndWait = async (entityTypes: EntityType[]) => { - await Promise.all(entityTypes.map((entityType) => initEntityEngineForEntityType(entityType))); + await Promise.all(entityTypes.map((entityType) => _initEntityEngineForEntityType(entityType))); await retry.waitForWithTimeout( `Engines to start for entity types: ${entityTypes.join(', ')}`,