Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[8.16] [Entity Store] [FTR Tests] Fix flakiness + poll for engine started on setup (#196564) #197053

Merged
merged 1 commit into from
Oct 23, 2024
Merged

[8.16] [Entity Store] [FTR Tests] Fix flakiness + poll for engine started on setup (#196564) #197053

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 28 additions & 2 deletions ...y_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/enrich_policy.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,10 +72,36 @@ export const executeFieldRetentionEnrichPolicy = async ({
export const deleteFieldRetentionEnrichPolicy = async ({
unitedDefinition,
esClient,
logger,
attempts = 5,
delayMs = 2000,
}: {
esClient: ElasticsearchClient;
unitedDefinition: DefinitionMetadata;
esClient: ElasticsearchClient;
logger: Logger;
attempts?: number;
delayMs?: number;
}) => {
const name = getFieldRetentionEnrichPolicyName(unitedDefinition);
return esClient.enrich.deletePolicy({ name }, { ignore: [404] });
let currentAttempt = 1;
while (currentAttempt <= attempts) {
try {
await esClient.enrich.deletePolicy({ name }, { ignore: [404] });
return;
} catch (e) {
// a 429 status code indicates that the enrich policy is being executed
if (currentAttempt === attempts || e.statusCode !== 429) {
logger.error(
`Error deleting enrich policy ${name}: ${e.message} after ${currentAttempt} attempts`
);
throw e;
}

logger.info(
`Enrich policy ${name} is being executed, waiting for it to finish before deleting`
);
await new Promise((resolve) => setTimeout(resolve, delayMs));
currentAttempt++;
}
}
};
5 changes: 3 additions & 2 deletions ...ns/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,7 @@ export class EntityStoreDataClient {
logger,
taskManager,
});
logger.info(`Entity store initialized`);
logger.info(`Entity store initialized for ${entityType}`);

return updated;
} catch (err) {
Expand Down Expand Up @@ -358,6 +358,7 @@ export class EntityStoreDataClient {
await deleteFieldRetentionEnrichPolicy({
unitedDefinition,
esClient,
logger,
});

if (deleteData) {
Expand Down Expand Up @@ -446,7 +447,7 @@ export class EntityStoreDataClient {
originalStatus === ENGINE_STATUS.UPDATING
) {
throw new Error(
`Error updating entity store: There is an changes already in progress for engine ${id}`
`Error updating entity store: There are changes already in progress for engine ${id}`
);
}

Expand Down
21 changes: 8 additions & 13 deletions ...tegration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,7 @@ export default ({ getService }: FtrProviderContext) => {
const supertest = getService('supertest');

const utils = EntityStoreUtils(getService);

// Failing: See https://github.com/elastic/kibana/issues/196526
describe.skip('@ess @skipInServerlessMKI Entity Store Engine APIs', () => {
describe('@ess @skipInServerlessMKI Entity Store Engine APIs', () => {
const dataView = dataViewRouteHelpersFactory(supertest);

before(async () => {
Expand All @@ -34,22 +32,19 @@ export default ({ getService }: FtrProviderContext) => {
});

it('should have installed the expected user resources', async () => {
await utils.initEntityEngineForEntityType('user');
await utils.initEntityEngineForEntityTypesAndWait(['user']);
await utils.expectEngineAssetsExist('user');
});

it('should have installed the expected host resources', async () => {
await utils.initEntityEngineForEntityType('host');
await utils.initEntityEngineForEntityTypesAndWait(['host']);
await utils.expectEngineAssetsExist('host');
});
});

describe('get and list', () => {
before(async () => {
await Promise.all([
utils.initEntityEngineForEntityType('host'),
utils.initEntityEngineForEntityType('user'),
]);
await utils.initEntityEngineForEntityTypesAndWait(['host', 'user']);
});

after(async () => {
Expand Down Expand Up @@ -119,7 +114,7 @@ export default ({ getService }: FtrProviderContext) => {

describe('start and stop', () => {
before(async () => {
await utils.initEntityEngineForEntityType('host');
await utils.initEntityEngineForEntityTypesAndWait(['host']);
});

after(async () => {
Expand Down Expand Up @@ -161,7 +156,7 @@ export default ({ getService }: FtrProviderContext) => {

describe('delete', () => {
it('should delete the host entity engine', async () => {
await utils.initEntityEngineForEntityType('host');
await utils.initEntityEngineForEntityTypesAndWait(['host']);

await api
.deleteEntityEngine({
Expand All @@ -174,7 +169,7 @@ export default ({ getService }: FtrProviderContext) => {
});

it('should delete the user entity engine', async () => {
await utils.initEntityEngineForEntityType('user');
await utils.initEntityEngineForEntityTypesAndWait(['user']);

await api
.deleteEntityEngine({
Expand All @@ -189,7 +184,7 @@ export default ({ getService }: FtrProviderContext) => {

describe('apply_dataview_indices', () => {
before(async () => {
await utils.initEntityEngineForEntityType('host');
await utils.initEntityEngineForEntityTypesAndWait(['host']);
});

after(async () => {
Expand Down
18 changes: 7 additions & 11 deletions ...tes/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
const supertest = getService('supertest');
const utils = EntityStoreUtils(getService, namespace);

// Failing: See https://github.com/elastic/kibana/issues/196546
describe.skip('@ess Entity Store Engine APIs in non-default space', () => {
describe('@ess Entity Store Engine APIs in non-default space', () => {
const dataView = dataViewRouteHelpersFactory(supertest, namespace);

before(async () => {
Expand All @@ -43,22 +42,19 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
});

it('should have installed the expected user resources', async () => {
await utils.initEntityEngineForEntityType('user');
await utils.initEntityEngineForEntityTypesAndWait(['user']);
await utils.expectEngineAssetsExist('user');
});

it('should have installed the expected host resources', async () => {
await utils.initEntityEngineForEntityType('host');
await utils.initEntityEngineForEntityTypesAndWait(['host']);
await utils.expectEngineAssetsExist('host');
});
});

describe('get and list', () => {
before(async () => {
await Promise.all([
utils.initEntityEngineForEntityType('host'),
utils.initEntityEngineForEntityType('user'),
]);
await utils.initEntityEngineForEntityTypesAndWait(['host', 'user']);
});

after(async () => {
Expand Down Expand Up @@ -134,7 +130,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {

describe('start and stop', () => {
before(async () => {
await utils.initEntityEngineForEntityType('host');
await utils.initEntityEngineForEntityTypesAndWait(['host']);
});

after(async () => {
Expand Down Expand Up @@ -188,7 +184,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {

describe('delete', () => {
it('should delete the host entity engine', async () => {
await utils.initEntityEngineForEntityType('host');
await utils.initEntityEngineForEntityTypesAndWait(['host']);

await api
.deleteEntityEngine(
Expand All @@ -204,7 +200,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
});

it('should delete the user entity engine', async () => {
await utils.initEntityEngineForEntityType('user');
await utils.initEntityEngineForEntityTypesAndWait(['user']);

await api
.deleteEntityEngine(
Expand Down
127 changes: 76 additions & 51 deletions ...rity_solution_api_integration/test_suites/entity_analytics/utils/elastic_asset_checker.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import { FtrProviderContext } from '@kbn/ftr-common-functional-services';

export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getService']) => {
const es = getService('es');
const retry = getService('retry');
const log = getService('log');

const expectTransformExists = async (transformId: string) => {
return expectTransformStatus(transformId, true);
Expand All @@ -18,45 +20,43 @@ export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getSe
return expectTransformStatus(transformId, false);
};

const expectTransformStatus = async (
transformId: string,
exists: boolean,
attempts: number = 5,
delayMs: number = 2000
) => {
let currentAttempt = 1;
while (currentAttempt <= attempts) {
try {
await es.transform.getTransform({ transform_id: transformId });
if (!exists) {
throw new Error(`Expected transform ${transformId} to not exist, but it does`);
const expectTransformStatus = async (transformId: string, exists: boolean) => {
await retry.waitForWithTimeout(
`transform ${transformId} to ${exists ? 'exist' : 'not exist'}`,
10_000,
async () => {
try {
await es.transform.getTransform({ transform_id: transformId });
return exists;
} catch (e) {
log.debug(`Transform ${transformId} not found: ${e}`);
return !exists;
}
return; // Transform exists, exit the loop
} catch (e) {
if (currentAttempt === attempts) {
if (exists) {
throw new Error(`Expected transform ${transformId} to exist, but it does not: ${e}`);
} else {
return; // Transform does not exist, exit the loop
}
}
await new Promise((resolve) => setTimeout(resolve, delayMs));
currentAttempt++;
}
}
);
};

const expectEnrichPolicyStatus = async (policyId: string, exists: boolean) => {
try {
await es.enrich.getPolicy({ name: policyId });
if (!exists) {
throw new Error(`Expected enrich policy ${policyId} to not exist, but it does`);
}
} catch (e) {
if (exists) {
throw new Error(`Expected enrich policy ${policyId} to exist, but it does not: ${e}`);
await retry.waitForWithTimeout(
`enrich policy ${policyId} to ${exists ? 'exist' : 'not exist'}`,
20_000,
async () => {
try {
const res = await es.enrich.getPolicy({ name: policyId });
const policy = res.policies?.[0];
if (policy) {
log.debug(`Enrich policy ${policyId} found: ${JSON.stringify(res)}`);
return exists;
} else {
log.debug(`Enrich policy ${policyId} not found: ${JSON.stringify(res)}`);
return !exists;
}
} catch (e) {
log.debug(`Enrich policy ${policyId} not found: ${e}`);
return !exists;
}
}
}
);
};

const expectEnrichPolicyExists = async (policyId: string) =>
Expand All @@ -66,18 +66,19 @@ export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getSe
expectEnrichPolicyStatus(policyId, false);

const expectComponentTemplatStatus = async (templateName: string, exists: boolean) => {
try {
await es.cluster.getComponentTemplate({ name: templateName });
if (!exists) {
throw new Error(`Expected component template ${templateName} to not exist, but it does`);
}
} catch (e) {
if (exists) {
throw new Error(
`Expected component template ${templateName} to exist, but it does not: ${e}`
);
await retry.waitForWithTimeout(
`component template ${templateName} to ${exists ? 'exist' : 'not exist'}`,
10_000,
async () => {
try {
await es.cluster.getComponentTemplate({ name: templateName });
return exists; // Component template exists
} catch (e) {
log.debug(`Component template ${templateName} not found: ${e}`);
return !exists; // Component template does not exist
}
}
}
);
};

const expectComponentTemplateExists = async (templateName: string) =>
Expand All @@ -87,23 +88,45 @@ export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getSe
expectComponentTemplatStatus(templateName, false);

const expectIngestPipelineStatus = async (pipelineId: string, exists: boolean) => {
await retry.waitForWithTimeout(
`ingest pipeline ${pipelineId} to ${exists ? 'exist' : 'not exist'}`,
10_000,
async () => {
try {
await es.ingest.getPipeline({ id: pipelineId });
return exists; // Ingest pipeline exists
} catch (e) {
log.debug(`Ingest pipeline ${pipelineId} not found: ${e}`);
return !exists; // Ingest pipeline does not exist
}
}
);
};

const expectIngestPipelineExists = async (pipelineId: string) =>
expectIngestPipelineStatus(pipelineId, true);

const expectIngestPipelineNotFound = async (pipelineId: string) =>
expectIngestPipelineStatus(pipelineId, false);

const expectIndexStatus = async (indexName: string, exists: boolean) => {
try {
await es.ingest.getPipeline({ id: pipelineId });
await es.indices.get({ index: indexName });
if (!exists) {
throw new Error(`Expected ingest pipeline ${pipelineId} to not exist, but it does`);
throw new Error(`Expected index ${indexName} to not exist, but it does`);
}
} catch (e) {
if (exists) {
throw new Error(`Expected ingest pipeline ${pipelineId} to exist, but it does not: ${e}`);
throw new Error(`Expected index ${indexName} to exist, but it does not: ${e}`);
}
}
};

const expectIngestPipelineExists = async (pipelineId: string) =>
expectIngestPipelineStatus(pipelineId, true);
const expectEntitiesIndexExists = async (entityType: string, namespace: string) =>
expectIndexStatus(`.entities.v1.latest.security_${entityType}_${namespace}`, true);

const expectIngestPipelineNotFound = async (pipelineId: string) =>
expectIngestPipelineStatus(pipelineId, false);
const expectEntitiesIndexNotFound = async (entityType: string, namespace: string) =>
expectIndexStatus(`.entities.v1.latest.security_${entityType}_${namespace}`, false);

return {
expectComponentTemplateExists,
Expand All @@ -112,6 +135,8 @@ export const elasticAssetCheckerFactory = (getService: FtrProviderContext['getSe
expectEnrichPolicyNotFound,
expectIngestPipelineExists,
expectIngestPipelineNotFound,
expectEntitiesIndexExists,
expectEntitiesIndexNotFound,
expectTransformExists,
expectTransformNotFound,
};
Expand Down
Loading