From 15ae27e790740e25a8d825ce21581f57dc4ba5d1 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Thu, 9 Jul 2020 16:39:08 -0600 Subject: [PATCH 1/5] Adds integration protection to the end to end tests --- .../basic/tests/find_statuses.ts | 3 +- .../detection_engine_api_integration/utils.ts | 57 +++++++++++++------ 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/x-pack/test/detection_engine_api_integration/basic/tests/find_statuses.ts b/x-pack/test/detection_engine_api_integration/basic/tests/find_statuses.ts index c88b094879ac8..cc6fa53939f60 100644 --- a/x-pack/test/detection_engine_api_integration/basic/tests/find_statuses.ts +++ b/x-pack/test/detection_engine_api_integration/basic/tests/find_statuses.ts @@ -22,8 +22,7 @@ export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const es = getService('es'); - // FLAKY: https://github.com/elastic/kibana/issues/69632 - describe.skip('find_statuses', () => { + describe('find_statuses', () => { beforeEach(async () => { await createSignalsIndex(supertest); }); diff --git a/x-pack/test/detection_engine_api_integration/utils.ts b/x-pack/test/detection_engine_api_integration/utils.ts index 6ad9cf4cd5baf..9cc9eba33ee0d 100644 --- a/x-pack/test/detection_engine_api_integration/utils.ts +++ b/x-pack/test/detection_engine_api_integration/utils.ts @@ -235,30 +235,44 @@ export const getSimpleMlRuleOutput = (ruleId = 'rule-1'): Partial = /** * Remove all alerts from the .kibana index + * This will retry 20 times before giving up and hopefully still not interfere with other tests * @param es The ElasticSearch handle */ -export const deleteAllAlerts = async (es: Client): Promise => { - await es.deleteByQuery({ - index: '.kibana', - q: 'type:alert', - wait_for_completion: true, - refresh: true, - body: {}, - }); +export const deleteAllAlerts = async (es: Client, retryCount = 20): Promise => { + if (retryCount > 0) { + try { + await es.deleteByQuery({ + index: '.kibana', + q: 'type:alert', + wait_for_completion: true, + refresh: true, + body: {}, + }); + } catch (err) { + await deleteAllAlerts(es, retryCount - 1); + } + } }; /** * Remove all rules statuses from the .kibana index + * This will retry 20 times before giving up and hopefully still not interfere with other tests * @param es The ElasticSearch handle */ -export const deleteAllRulesStatuses = async (es: Client): Promise => { - await es.deleteByQuery({ - index: '.kibana', - q: 'type:siem-detection-engine-rule-status', - wait_for_completion: true, - refresh: true, - body: {}, - }); +export const deleteAllRulesStatuses = async (es: Client, retryCount = 20): Promise => { + if (retryCount > 0) { + try { + await es.deleteByQuery({ + index: '.kibana', + q: 'type:siem-detection-engine-rule-status', + wait_for_completion: true, + refresh: true, + body: {}, + }); + } catch (err) { + await deleteAllRulesStatuses(es, retryCount - 1); + } + } }; /** @@ -276,9 +290,16 @@ export const createSignalsIndex = async ( * @param supertest The supertest client library */ export const deleteSignalsIndex = async ( - supertest: SuperTest + supertest: SuperTest, + retryCount = 20 ): Promise => { - await supertest.delete(DETECTION_ENGINE_INDEX_URL).set('kbn-xsrf', 'true').send().expect(200); + if (retryCount > 0) { + try { + await supertest.delete(DETECTION_ENGINE_INDEX_URL).set('kbn-xsrf', 'true').send(); + } catch (err) { + await deleteSignalsIndex(supertest, retryCount - 1); + } + } }; /** From 6f62fd0a2f8eaa5ef948ed97c5d31617e531089f Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Thu, 9 Jul 2020 16:49:14 -0600 Subject: [PATCH 2/5] Added console log statements --- .../detection_engine_api_integration/utils.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/x-pack/test/detection_engine_api_integration/utils.ts b/x-pack/test/detection_engine_api_integration/utils.ts index 9cc9eba33ee0d..67387ed0bb066 100644 --- a/x-pack/test/detection_engine_api_integration/utils.ts +++ b/x-pack/test/detection_engine_api_integration/utils.ts @@ -249,8 +249,13 @@ export const deleteAllAlerts = async (es: Client, retryCount = 20): Promise Date: Thu, 9 Jul 2020 16:54:34 -0600 Subject: [PATCH 3/5] Adds the err object to the console lines --- .../detection_engine_api_integration/utils.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/x-pack/test/detection_engine_api_integration/utils.ts b/x-pack/test/detection_engine_api_integration/utils.ts index 67387ed0bb066..e78633dad18c4 100644 --- a/x-pack/test/detection_engine_api_integration/utils.ts +++ b/x-pack/test/detection_engine_api_integration/utils.ts @@ -250,7 +250,7 @@ export const deleteAllAlerts = async (es: Client, retryCount = 20): Promise ): Promise => { - await supertest.post(DETECTION_ENGINE_INDEX_URL).set('kbn-xsrf', 'true').send().expect(200); + try { + await supertest.post(DETECTION_ENGINE_INDEX_URL).set('kbn-xsrf', 'true').send(); + } catch (err) { + // eslint-disable-next-line no-console + console.log( + 'Warning, the create signals index did not happen correctly, this could cause test failures', + err + ); + } }; /** @@ -308,7 +319,7 @@ export const deleteSignalsIndex = async ( await supertest.delete(DETECTION_ENGINE_INDEX_URL).set('kbn-xsrf', 'true').send(); } catch (err) { // eslint-disable-next-line no-console - console.log(`Failure trying to deleteSignalsIndex, retries left are: ${retryCount - 1}`); + console.log(`Failure trying to deleteSignalsIndex, retries left are: ${retryCount - 1}`, err); await deleteSignalsIndex(supertest, retryCount - 1); } } else { From ed8e8884fa90d864d74752483c1d434822dadd45 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Thu, 9 Jul 2020 16:59:27 -0600 Subject: [PATCH 4/5] Adds more retries to the createSignalsIndex --- .../detection_engine_api_integration/utils.ts | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/x-pack/test/detection_engine_api_integration/utils.ts b/x-pack/test/detection_engine_api_integration/utils.ts index e78633dad18c4..601a209f61be2 100644 --- a/x-pack/test/detection_engine_api_integration/utils.ts +++ b/x-pack/test/detection_engine_api_integration/utils.ts @@ -293,16 +293,23 @@ export const deleteAllRulesStatuses = async (es: Client, retryCount = 20): Promi * @param supertest The supertest client library */ export const createSignalsIndex = async ( - supertest: SuperTest + supertest: SuperTest, + retryCount = 20 ): Promise => { - try { - await supertest.post(DETECTION_ENGINE_INDEX_URL).set('kbn-xsrf', 'true').send(); - } catch (err) { + if (retryCount > 0) { + try { + await supertest.post(DETECTION_ENGINE_INDEX_URL).set('kbn-xsrf', 'true').send(); + } catch (err) { + // eslint-disable-next-line no-console + console.log( + `Failure trying to create the signals index, retries left are: ${retryCount - 1}`, + err + ); + await createSignalsIndex(supertest, retryCount - 1); + } + } else { // eslint-disable-next-line no-console - console.log( - 'Warning, the create signals index did not happen correctly, this could cause test failures', - err - ); + console.log('Could not createSignalsIndex, no retries are left'); } }; From 095f3886c56447a10836358a2636108d516ed81c Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Thu, 9 Jul 2020 17:00:29 -0600 Subject: [PATCH 5/5] Adds one line comment --- x-pack/test/detection_engine_api_integration/utils.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test/detection_engine_api_integration/utils.ts b/x-pack/test/detection_engine_api_integration/utils.ts index 601a209f61be2..4b980536d2cd1 100644 --- a/x-pack/test/detection_engine_api_integration/utils.ts +++ b/x-pack/test/detection_engine_api_integration/utils.ts @@ -290,6 +290,7 @@ export const deleteAllRulesStatuses = async (es: Client, retryCount = 20): Promi /** * Creates the signals index for use inside of beforeEach blocks of tests + * This will retry 20 times before giving up and hopefully still not interfere with other tests * @param supertest The supertest client library */ export const createSignalsIndex = async (