From bf3dc72ad13e829d9dc47225f25c606f1bafbb62 Mon Sep 17 00:00:00 2001 From: Alexi Doak <109488926+doakalexi@users.noreply.github.com> Date: Tue, 15 Aug 2023 13:42:37 -0700 Subject: [PATCH] [ResponseOps][Alerting] Add test coverage for Alerting in serverless (#163753) Resolves https://github.com/elastic/response-ops-team/issues/124 ## Summary Adds alerting serverless tests! I copied over from this test file `x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/alerting/alerts.ts` Ran in the QA environment to verify tests are passing --- .../alerting/helpers/alerting_api_helper.ts | 239 ++++ .../helpers/alerting_wait_for_helpers.ts | 303 +++++ .../test_suites/common/alerting/index.ts | 14 + .../test_suites/common/alerting/rules.ts | 1010 +++++++++++++++++ .../test_suites/common/index.ts | 1 + 5 files changed, 1567 insertions(+) create mode 100644 x-pack/test_serverless/api_integration/test_suites/common/alerting/helpers/alerting_api_helper.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/common/alerting/helpers/alerting_wait_for_helpers.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/common/alerting/index.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/common/alerting/rules.ts diff --git a/x-pack/test_serverless/api_integration/test_suites/common/alerting/helpers/alerting_api_helper.ts b/x-pack/test_serverless/api_integration/test_suites/common/alerting/helpers/alerting_api_helper.ts new file mode 100644 index 0000000000000..a8bcd98d89689 --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/common/alerting/helpers/alerting_api_helper.ts @@ -0,0 +1,239 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SuperTest, Test } from 'supertest'; + +interface CreateEsQueryRuleParams { + size: number; + thresholdComparator: string; + threshold: number[]; + timeWindowSize?: number; + timeWindowUnit?: string; + esQuery?: string; + timeField?: string; + searchConfiguration?: unknown; + indexName?: string; + excludeHitsFromPreviousRun?: boolean; + aggType?: string; + aggField?: string; + groupBy?: string; + termField?: string; + termSize?: number; + index?: string[]; +} + +export async function createIndexConnector({ + supertest, + name, + indexName, +}: { + supertest: SuperTest; + name: string; + indexName: string; +}) { + const { body } = await supertest + .post(`/api/actions/connector`) + .set('kbn-xsrf', 'foo') + .set('x-elastic-internal-origin', 'foo') + .send({ + name, + config: { + index: indexName, + refresh: true, + }, + connector_type_id: '.index', + }); + return body.id as string; +} + +export async function createSlackConnector({ + supertest, + name, +}: { + supertest: SuperTest; + name: string; +}) { + const { body } = await supertest + .post(`/api/actions/connector`) + .set('kbn-xsrf', 'foo') + .set('x-elastic-internal-origin', 'foo') + .send({ + name, + config: {}, + secrets: { + webhookUrl: 'http://test', + }, + connector_type_id: '.slack', + }); + return body.id as string; +} + +export async function createEsQueryRule({ + supertest, + name, + ruleTypeId, + params, + actions = [], + tags = [], + schedule, + consumer, + notifyWhen, + enabled = true, +}: { + supertest: SuperTest; + ruleTypeId: string; + name: string; + params: CreateEsQueryRuleParams; + consumer: string; + actions?: any[]; + tags?: any[]; + schedule?: { interval: string }; + notifyWhen?: string; + enabled?: boolean; +}) { + const { body } = await supertest + .post(`/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .set('x-elastic-internal-origin', 'foo') + .send({ + enabled, + params, + consumer, + schedule: schedule || { + interval: '1h', + }, + tags, + name, + rule_type_id: ruleTypeId, + actions, + ...(notifyWhen ? { notify_when: notifyWhen, throttle: '1m' } : {}), + }); + return body; +} + +export async function disableRule({ + supertest, + ruleId, +}: { + supertest: SuperTest; + ruleId: string; +}) { + const { body } = await supertest + .post(`/api/alerting/rule/${ruleId}/_disable`) + .set('kbn-xsrf', 'foo') + .set('x-elastic-internal-origin', 'foo'); + return body; +} + +export async function updateEsQueryRule({ + supertest, + ruleId, + updates, +}: { + supertest: SuperTest; + ruleId: string; + updates: any; +}) { + const { body: r } = await supertest + .get(`/api/alerting/rule/${ruleId}`) + .set('kbn-xsrf', 'foo') + .set('x-elastic-internal-origin', 'foo'); + const body = await supertest + .put(`/api/alerting/rule/${ruleId}`) + .set('kbn-xsrf', 'foo') + .set('x-elastic-internal-origin', 'foo') + .send({ + ...{ + name: r.name, + schedule: r.schedule, + throttle: r.throttle, + tags: r.tags, + params: r.params, + notify_when: r.notifyWhen, + actions: r.actions.map((action: any) => ({ + group: action.group, + params: action.params, + id: action.id, + frequency: action.frequency, + })), + }, + ...updates, + }); + return body; +} + +export async function runRule({ + supertest, + ruleId, +}: { + supertest: SuperTest; + ruleId: string; +}) { + const { body } = await supertest + .post(`/internal/alerting/rule/${ruleId}/_run_soon`) + .set('kbn-xsrf', 'foo') + .set('x-elastic-internal-origin', 'foo'); + return body; +} + +export async function muteRule({ + supertest, + ruleId, +}: { + supertest: SuperTest; + ruleId: string; +}) { + const { body } = await supertest + .post(`/api/alerting/rule/${ruleId}/_mute_all`) + .set('kbn-xsrf', 'foo') + .set('x-elastic-internal-origin', 'foo'); + return body; +} + +export async function enableRule({ + supertest, + ruleId, +}: { + supertest: SuperTest; + ruleId: string; +}) { + const { body } = await supertest + .post(`/api/alerting/rule/${ruleId}/_enable`) + .set('kbn-xsrf', 'foo') + .set('x-elastic-internal-origin', 'foo'); + return body; +} + +export async function muteAlert({ + supertest, + ruleId, + alertId, +}: { + supertest: SuperTest; + ruleId: string; + alertId: string; +}) { + const { body } = await supertest + .post(`/api/alerting/rule/${ruleId}/alert/${alertId}/_mute`) + .set('kbn-xsrf', 'foo') + .set('x-elastic-internal-origin', 'foo'); + return body; +} + +export async function unmuteRule({ + supertest, + ruleId, +}: { + supertest: SuperTest; + ruleId: string; +}) { + const { body } = await supertest + .post(`/api/alerting/rule/${ruleId}/_unmute_all`) + .set('kbn-xsrf', 'foo') + .set('x-elastic-internal-origin', 'foo'); + return body; +} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/alerting/helpers/alerting_wait_for_helpers.ts b/x-pack/test_serverless/api_integration/test_suites/common/alerting/helpers/alerting_wait_for_helpers.ts new file mode 100644 index 0000000000000..1b5723cc07de5 --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/common/alerting/helpers/alerting_wait_for_helpers.ts @@ -0,0 +1,303 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import pRetry from 'p-retry'; +import type { Client } from '@elastic/elasticsearch'; +import type { + AggregationsAggregate, + SearchResponse, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +export async function waitForDocumentInIndex({ + esClient, + indexName, + num = 1, +}: { + esClient: Client; + indexName: string; + num?: number; +}): Promise { + return pRetry( + async () => { + const response = await esClient.search({ index: indexName }); + if (response.hits.hits.length < num) { + throw new Error('No hits found'); + } + return response; + }, + { retries: 10 } + ); +} + +export async function getDocumentsInIndex({ + esClient, + indexName, +}: { + esClient: Client; + indexName: string; +}): Promise { + return await esClient.search({ index: indexName }); +} + +export async function createIndex({ + esClient, + indexName, +}: { + esClient: Client; + indexName: string; +}) { + return await esClient.indices.create( + { + index: indexName, + body: {}, + }, + { meta: true } + ); +} + +export async function waitForAlertInIndex({ + esClient, + indexName, + ruleId, +}: { + esClient: Client; + indexName: string; + ruleId: string; +}): Promise>> { + return pRetry( + async () => { + const response = await esClient.search({ + index: indexName, + body: { + query: { + term: { + 'kibana.alert.rule.uuid': ruleId, + }, + }, + }, + }); + if (response.hits.hits.length === 0) { + throw new Error('No hits found'); + } + return response; + }, + { retries: 10 } + ); +} + +export async function waitForAllTasksIdle({ + esClient, + filter, +}: { + esClient: Client; + filter: Date; +}): Promise { + return pRetry( + async () => { + const response = await esClient.search({ + index: '.kibana_task_manager', + body: { + query: { + bool: { + must: [ + { + terms: { + 'task.scope': ['actions', 'alerting'], + }, + }, + { + range: { + 'task.scheduledAt': { + gte: filter.getTime().toString(), + }, + }, + }, + ], + must_not: [ + { + term: { + 'task.status': 'idle', + }, + }, + ], + }, + }, + }, + }); + if (response.hits.hits.length !== 0) { + throw new Error(`Expected 0 hits but received ${response.hits.hits.length}`); + } + return response; + }, + { retries: 10 } + ); +} + +export async function waitForAllTasks({ + esClient, + filter, + taskType, + attempts, +}: { + esClient: Client; + filter: Date; + taskType: string; + attempts: number; +}): Promise { + return pRetry( + async () => { + const response = await esClient.search({ + index: '.kibana_task_manager', + body: { + query: { + bool: { + must: [ + { + term: { + 'task.status': 'idle', + }, + }, + { + term: { + 'task.attempts': attempts, + }, + }, + { + terms: { + 'task.scope': ['actions', 'alerting'], + }, + }, + { + term: { + 'task.taskType': taskType, + }, + }, + { + range: { + 'task.scheduledAt': { + gte: filter.getTime().toString(), + }, + }, + }, + ], + }, + }, + }, + }); + if (response.hits.hits.length === 0) { + throw new Error('No hits found'); + } + return response; + }, + { retries: 10 } + ); +} + +export async function waitForDisabled({ + esClient, + ruleId, + filter, +}: { + esClient: Client; + ruleId: string; + filter: Date; +}): Promise { + return pRetry( + async () => { + const response = await esClient.search({ + index: '.kibana_task_manager', + body: { + query: { + bool: { + must: [ + { + term: { + 'task.id': `task:${ruleId}`, + }, + }, + { + terms: { + 'task.scope': ['actions', 'alerting'], + }, + }, + { + range: { + 'task.scheduledAt': { + gte: filter.getTime().toString(), + }, + }, + }, + { + term: { + 'task.enabled': true, + }, + }, + ], + }, + }, + }, + }); + if (response.hits.hits.length !== 0) { + throw new Error(`Expected 0 hits but received ${response.hits.hits.length}`); + } + return response; + }, + { retries: 10 } + ); +} + +export async function waitForEventLog({ + esClient, + provider, + filter, + num = 1, +}: { + esClient: Client; + provider: string; + filter: Date; + num?: number; +}): Promise { + return pRetry( + async () => { + const response = await esClient.search({ + index: '.kibana-event-log*', + body: { + query: { + bool: { + filter: [ + { + term: { + 'event.provider': { + value: provider, + }, + }, + }, + { + term: { + 'event.action': 'execute', + }, + }, + { + range: { + '@timestamp': { + gte: filter.getTime().toString(), + }, + }, + }, + ], + }, + }, + }, + }); + if (response.hits.hits.length < num) { + throw new Error('No hits found'); + } + return response; + }, + { retries: 10 } + ); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/alerting/index.ts b/x-pack/test_serverless/api_integration/test_suites/common/alerting/index.ts new file mode 100644 index 0000000000000..4a78d448a7d20 --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/common/alerting/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { + describe('Alerting APIs', function () { + loadTestFile(require.resolve('./rules')); + }); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/alerting/rules.ts b/x-pack/test_serverless/api_integration/test_suites/common/alerting/rules.ts new file mode 100644 index 0000000000000..86fdf7afb842a --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/common/alerting/rules.ts @@ -0,0 +1,1010 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { + createIndexConnector, + createEsQueryRule, + disableRule, + updateEsQueryRule, + runRule, + muteRule, + enableRule, + muteAlert, + unmuteRule, + createSlackConnector, +} from './helpers/alerting_api_helper'; +import { + createIndex, + getDocumentsInIndex, + waitForAllTasks, + waitForAllTasksIdle, + waitForDisabled, + waitForDocumentInIndex, + waitForEventLog, +} from './helpers/alerting_wait_for_helpers'; + +export default function ({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const esClient = getService('es'); + const esDeleteAllIndices = getService('esDeleteAllIndices'); + + describe('Alerting rules', () => { + const RULE_TYPE_ID = '.es-query'; + const ALERT_ACTION_INDEX = 'alert-action-es-query'; + let actionId: string; + let ruleId: string; + + afterEach(async () => { + await supertest + .delete(`/api/actions/connector/${actionId}`) + .set('kbn-xsrf', 'foo') + .set('x-elastic-internal-origin', 'foo'); + await supertest + .delete(`/api/alerting/rule/${ruleId}`) + .set('kbn-xsrf', 'foo') + .set('x-elastic-internal-origin', 'foo'); + await esClient.deleteByQuery({ + index: '.kibana-event-log-*', + query: { term: { 'kibana.alert.rule.consumer': 'alerts' } }, + }); + await esDeleteAllIndices([ALERT_ACTION_INDEX]); + }); + + it('should schedule task, run rule and schedule actions when appropriate', async () => { + const testStart = new Date(); + + actionId = await createIndexConnector({ + supertest, + name: 'Index Connector: Alerting API test', + indexName: ALERT_ACTION_INDEX, + }); + expect(actionId).not.to.be(undefined); + + const createdRule = await createEsQueryRule({ + supertest, + consumer: 'alerts', + name: 'always fire', + ruleTypeId: RULE_TYPE_ID, + params: { + size: 100, + thresholdComparator: '>', + threshold: [-1], + index: ['alert-test-data'], + timeField: 'date', + esQuery: `{\n \"query\":{\n \"match_all\" : {}\n }\n}`, + timeWindowSize: 20, + timeWindowUnit: 's', + }, + actions: [ + { + group: 'query matched', + id: actionId, + params: { + documents: [ + { + ruleId: '{{rule.id}}', + ruleName: '{{rule.name}}', + ruleParams: '{{rule.params}}', + spaceId: '{{rule.spaceId}}', + tags: '{{rule.tags}}', + alertId: '{{alert.id}}', + alertActionGroup: '{{alert.actionGroup}}', + instanceContextValue: '{{context.instanceContextValue}}', + instanceStateValue: '{{state.instanceStateValue}}', + }, + ], + }, + frequency: { + notify_when: 'onActiveAlert', + throttle: null, + summary: false, + }, + }, + ], + }); + ruleId = createdRule.id; + expect(ruleId).not.to.be(undefined); + + // Wait for the action to index a document before disabling the alert and waiting for tasks to finish + const resp = await waitForDocumentInIndex({ + esClient, + indexName: ALERT_ACTION_INDEX, + }); + expect(resp.hits.hits.length).to.be(1); + + await waitForAllTasksIdle({ + esClient, + filter: testStart, + }); + + await disableRule({ + supertest, + ruleId, + }); + + await waitForDisabled({ + esClient, + ruleId, + filter: testStart, + }); + + const document = resp.hits.hits[0]; + expect(document._source).to.eql({ + alertActionGroup: 'query matched', + alertId: 'query matched', + instanceContextValue: '', + instanceStateValue: '', + ruleId, + ruleName: 'always fire', + ruleParams: + '{"size":100,"thresholdComparator":">","threshold":[-1],"index":["alert-test-data"],"timeField":"date","esQuery":"{\\n \\"query\\":{\\n \\"match_all\\" : {}\\n }\\n}","timeWindowSize":20,"timeWindowUnit":"s","excludeHitsFromPreviousRun":true,"aggType":"count","groupBy":"all","searchType":"esQuery"}', + spaceId: 'default', + tags: '', + }); + + const eventLogResp = await waitForEventLog({ + esClient, + provider: 'alerting', + filter: testStart, + }); + expect(eventLogResp.hits.hits.length).to.be(1); + + const eventLogDocument = eventLogResp.hits.hits[0]._source; + await validateEventLog(eventLogDocument, { + ruleId, + ruleTypeId: RULE_TYPE_ID, + outcome: 'success', + name: 'always fire', + message: `rule executed: ${RULE_TYPE_ID}:${ruleId}: 'always fire'`, + }); + }); + + it('should pass updated rule params to executor', async () => { + const testStart = new Date(); + + actionId = await createIndexConnector({ + supertest, + name: 'Index Connector: Alerting API test', + indexName: ALERT_ACTION_INDEX, + }); + expect(actionId).not.to.be(undefined); + + const createdRule = await createEsQueryRule({ + supertest, + consumer: 'alerts', + name: 'always fire', + ruleTypeId: RULE_TYPE_ID, + params: { + size: 100, + thresholdComparator: '>', + threshold: [-1], + index: ['alert-test-data'], + timeField: 'date', + esQuery: `{\n \"query\":{\n \"match_all\" : {}\n }\n}`, + timeWindowSize: 20, + timeWindowUnit: 's', + }, + actions: [ + { + group: 'query matched', + id: actionId, + params: { + documents: [ + { + ruleId: '{{rule.id}}', + ruleName: '{{rule.name}}', + ruleParams: '{{rule.params}}', + spaceId: '{{rule.spaceId}}', + tags: '{{rule.tags}}', + alertId: '{{alert.id}}', + alertActionGroup: '{{alert.actionGroup}}', + instanceContextValue: '{{context.instanceContextValue}}', + instanceStateValue: '{{state.instanceStateValue}}', + }, + ], + }, + frequency: { + notify_when: 'onActiveAlert', + throttle: null, + summary: false, + }, + }, + ], + }); + ruleId = createdRule.id; + expect(ruleId).not.to.be(undefined); + + // Wait for the action to index a document before disabling the alert and waiting for tasks to finish + const resp = await waitForDocumentInIndex({ + esClient, + indexName: ALERT_ACTION_INDEX, + }); + expect(resp.hits.hits.length).to.be(1); + + const document = resp.hits.hits[0]; + expect(document._source).to.eql({ + alertActionGroup: 'query matched', + alertId: 'query matched', + instanceContextValue: '', + instanceStateValue: '', + ruleId, + ruleName: 'always fire', + ruleParams: + '{"size":100,"thresholdComparator":">","threshold":[-1],"index":["alert-test-data"],"timeField":"date","esQuery":"{\\n \\"query\\":{\\n \\"match_all\\" : {}\\n }\\n}","timeWindowSize":20,"timeWindowUnit":"s","excludeHitsFromPreviousRun":true,"aggType":"count","groupBy":"all","searchType":"esQuery"}', + spaceId: 'default', + tags: '', + }); + + await waitForAllTasksIdle({ + esClient, + filter: testStart, + }); + + await updateEsQueryRule({ + supertest, + ruleId, + updates: { + name: 'def', + tags: ['fee', 'fi', 'fo'], + }, + }); + + await runRule({ + supertest, + ruleId, + }); + + // make sure alert info passed to executor is correct + const resp2 = await waitForDocumentInIndex({ + esClient, + indexName: ALERT_ACTION_INDEX, + num: 2, + }); + expect(resp2.hits.hits.length).to.be(2); + + const document2 = resp2.hits.hits[1]; + expect(document2._source).to.eql({ + alertActionGroup: 'query matched', + alertId: 'query matched', + instanceContextValue: '', + instanceStateValue: '', + ruleId, + ruleName: 'def', + ruleParams: + '{"size":100,"thresholdComparator":">","threshold":[-1],"index":["alert-test-data"],"timeField":"date","esQuery":"{\\n \\"query\\":{\\n \\"match_all\\" : {}\\n }\\n}","timeWindowSize":20,"timeWindowUnit":"s","excludeHitsFromPreviousRun":true,"aggType":"count","groupBy":"all","searchType":"esQuery"}', + spaceId: 'default', + tags: 'fee,fi,fo', + }); + }); + + it('should retry when appropriate', async () => { + const testStart = new Date(); + + // Should fail + actionId = await createSlackConnector({ + supertest, + name: 'Slack Connector: Alerting API test', + }); + expect(actionId).not.to.be(undefined); + + const createdRule = await createEsQueryRule({ + supertest, + consumer: 'alerts', + name: 'always fire', + ruleTypeId: RULE_TYPE_ID, + params: { + size: 100, + thresholdComparator: '>', + threshold: [-1], + index: ['alert-test-data'], + timeField: 'date', + esQuery: `{\n \"query\":{\n \"match_all\" : {}\n }\n}`, + timeWindowSize: 20, + timeWindowUnit: 's', + }, + actions: [ + { + group: 'query matched', + id: actionId, + params: { + message: `message: {{rule.id}}`, + }, + frequency: { + notify_when: 'onActiveAlert', + throttle: null, + summary: false, + }, + }, + ], + }); + ruleId = createdRule.id; + expect(ruleId).not.to.be(undefined); + + // Should retry when the the action fails + const resp = await waitForAllTasks({ + esClient, + filter: testStart, + taskType: 'actions:.slack', + attempts: 1, + }); + expect(resp.hits.hits.length).to.be(1); + }); + + it('should throttle alerts when appropriate', async () => { + const testStart = new Date(); + + actionId = await createIndexConnector({ + supertest, + name: 'Index Connector: Alerting API test', + indexName: ALERT_ACTION_INDEX, + }); + expect(actionId).not.to.be(undefined); + + const createdRule = await createEsQueryRule({ + supertest, + consumer: 'alerts', + name: 'always fire', + ruleTypeId: RULE_TYPE_ID, + schedule: { interval: '5s' }, + notifyWhen: 'onThrottleInterval', + params: { + size: 100, + thresholdComparator: '>', + threshold: [-1], + index: ['alert-test-data'], + timeField: 'date', + esQuery: `{\n \"query\":{\n \"match_all\" : {}\n }\n}`, + timeWindowSize: 20, + timeWindowUnit: 's', + }, + actions: [ + { + group: 'query matched', + id: actionId, + params: { + documents: [ + { + ruleId: '{{rule.id}}', + ruleName: '{{rule.name}}', + ruleParams: '{{rule.params}}', + spaceId: '{{rule.spaceId}}', + tags: '{{rule.tags}}', + alertId: '{{alert.id}}', + alertActionGroup: '{{alert.actionGroup}}', + instanceContextValue: '{{context.instanceContextValue}}', + instanceStateValue: '{{state.instanceStateValue}}', + }, + ], + }, + }, + ], + }); + ruleId = createdRule.id; + expect(ruleId).not.to.be(undefined); + + // Wait until alerts ran at least 3 times before disabling the alert and waiting for tasks to finish + const eventLogResp = await waitForEventLog({ + esClient, + provider: 'alerting', + filter: testStart, + num: 3, + }); + expect(eventLogResp.hits.hits.length >= 3).to.be(true); + + await disableRule({ + supertest, + ruleId, + }); + + await waitForDisabled({ + esClient, + ruleId, + filter: testStart, + }); + + // Ensure actions only executed once + const resp = await waitForDocumentInIndex({ + esClient, + indexName: ALERT_ACTION_INDEX, + }); + expect(resp.hits.hits.length).to.be(1); + }); + + it('should throttle alerts with throttled action when appropriate', async () => { + const testStart = new Date(); + + actionId = await createIndexConnector({ + supertest, + name: 'Index Connector: Alerting API test', + indexName: ALERT_ACTION_INDEX, + }); + expect(actionId).not.to.be(undefined); + + const createdRule = await createEsQueryRule({ + supertest, + consumer: 'alerts', + name: 'always fire', + ruleTypeId: RULE_TYPE_ID, + schedule: { interval: '5s' }, + params: { + size: 100, + thresholdComparator: '>', + threshold: [-1], + index: ['alert-test-data'], + timeField: 'date', + esQuery: `{\n \"query\":{\n \"match_all\" : {}\n }\n}`, + timeWindowSize: 20, + timeWindowUnit: 's', + }, + actions: [ + { + group: 'query matched', + id: actionId, + params: { + documents: [ + { + ruleId: '{{rule.id}}', + ruleName: '{{rule.name}}', + ruleParams: '{{rule.params}}', + spaceId: '{{rule.spaceId}}', + tags: '{{rule.tags}}', + alertId: '{{alert.id}}', + alertActionGroup: '{{alert.actionGroup}}', + instanceContextValue: '{{context.instanceContextValue}}', + instanceStateValue: '{{state.instanceStateValue}}', + }, + ], + }, + frequency: { + notify_when: 'onThrottleInterval', + throttle: '1m', + summary: false, + }, + }, + ], + }); + ruleId = createdRule.id; + expect(ruleId).not.to.be(undefined); + + // Wait until alerts ran at least 3 times before disabling the alert and waiting for tasks to finish + const eventLogResp = await waitForEventLog({ + esClient, + provider: 'alerting', + filter: testStart, + num: 3, + }); + expect(eventLogResp.hits.hits.length >= 3).to.be(true); + + await disableRule({ + supertest, + ruleId, + }); + + await waitForDisabled({ + esClient, + ruleId, + filter: testStart, + }); + + // Ensure actions only executed once + const resp = await waitForDocumentInIndex({ + esClient, + indexName: ALERT_ACTION_INDEX, + }); + expect(resp.hits.hits.length).to.be(1); + }); + + it('should reset throttle window when not firing and should not throttle when changing groups', async () => { + const testStart = new Date(); + + actionId = await createIndexConnector({ + supertest, + name: 'Index Connector: Alerting API test', + indexName: ALERT_ACTION_INDEX, + }); + expect(actionId).not.to.be(undefined); + + const createdRule = await createEsQueryRule({ + supertest, + consumer: 'alerts', + name: 'always fire', + ruleTypeId: RULE_TYPE_ID, + schedule: { interval: '1m' }, + params: { + size: 100, + thresholdComparator: '>', + threshold: [-1], + index: ['alert-test-data'], + timeField: 'date', + esQuery: `{\n \"query\":{\n \"match_all\" : {}\n }\n}`, + timeWindowSize: 20, + timeWindowUnit: 's', + }, + actions: [ + { + group: 'query matched', + id: actionId, + params: { + documents: [ + { + ruleId: '{{rule.id}}', + ruleName: '{{rule.name}}', + ruleParams: '{{rule.params}}', + spaceId: '{{rule.spaceId}}', + tags: '{{rule.tags}}', + alertId: '{{alert.id}}', + alertActionGroup: '{{alert.actionGroup}}', + instanceContextValue: '{{context.instanceContextValue}}', + instanceStateValue: '{{state.instanceStateValue}}', + }, + ], + }, + frequency: { + notify_when: 'onThrottleInterval', + throttle: '1m', + summary: false, + }, + }, + { + group: 'recovered', + id: actionId, + params: { + documents: [ + { + ruleId: '{{rule.id}}', + ruleName: '{{rule.name}}', + ruleParams: '{{rule.params}}', + spaceId: '{{rule.spaceId}}', + tags: '{{rule.tags}}', + alertId: '{{alert.id}}', + alertActionGroup: '{{alert.actionGroup}}', + instanceContextValue: '{{context.instanceContextValue}}', + instanceStateValue: '{{state.instanceStateValue}}', + }, + ], + }, + frequency: { + notify_when: 'onThrottleInterval', + throttle: '1m', + summary: false, + }, + }, + ], + }); + ruleId = createdRule.id; + expect(ruleId).not.to.be(undefined); + + // Wait for the action to index a document + const resp = await waitForDocumentInIndex({ + esClient, + indexName: ALERT_ACTION_INDEX, + }); + expect(resp.hits.hits.length).to.be(1); + + await waitForAllTasksIdle({ + esClient, + filter: testStart, + }); + + // Update the rule to recover + await updateEsQueryRule({ + supertest, + ruleId, + updates: { + name: 'never fire', + params: { + size: 100, + thresholdComparator: '<', + threshold: [-1], + index: ['alert-test-data'], + timeField: 'date', + esQuery: `{\n \"query\":{\n \"match_all\" : {}\n }\n}`, + timeWindowSize: 20, + timeWindowUnit: 's', + }, + }, + }); + + await runRule({ + supertest, + ruleId, + }); + + const eventLogResp = await waitForEventLog({ + esClient, + provider: 'alerting', + filter: testStart, + num: 2, + }); + expect(eventLogResp.hits.hits.length).to.be(2); + + await disableRule({ + supertest, + ruleId, + }); + + await waitForDisabled({ + esClient, + ruleId, + filter: testStart, + }); + + // Ensure only 2 actions are executed + const resp2 = await waitForDocumentInIndex({ + esClient, + indexName: ALERT_ACTION_INDEX, + num: 2, + }); + expect(resp2.hits.hits.length).to.be(2); + }); + + it(`shouldn't schedule actions when alert is muted`, async () => { + const testStart = new Date(); + await createIndex({ esClient, indexName: ALERT_ACTION_INDEX }); + + actionId = await createIndexConnector({ + supertest, + name: 'Index Connector: Alerting API test', + indexName: ALERT_ACTION_INDEX, + }); + expect(actionId).not.to.be(undefined); + + const createdRule = await createEsQueryRule({ + supertest, + enabled: false, + consumer: 'alerts', + name: 'always fire', + ruleTypeId: RULE_TYPE_ID, + schedule: { interval: '5s' }, + params: { + size: 100, + thresholdComparator: '>', + threshold: [-1], + index: ['alert-test-data'], + timeField: 'date', + esQuery: `{\n \"query\":{\n \"match_all\" : {}\n }\n}`, + timeWindowSize: 20, + timeWindowUnit: 's', + }, + actions: [ + { + group: 'query matched', + id: actionId, + params: { + documents: [ + { + ruleId: '{{rule.id}}', + ruleName: '{{rule.name}}', + ruleParams: '{{rule.params}}', + spaceId: '{{rule.spaceId}}', + tags: '{{rule.tags}}', + alertId: '{{alert.id}}', + alertActionGroup: '{{alert.actionGroup}}', + instanceContextValue: '{{context.instanceContextValue}}', + instanceStateValue: '{{state.instanceStateValue}}', + }, + ], + }, + frequency: { + notify_when: 'onActiveAlert', + throttle: null, + summary: false, + }, + }, + ], + }); + ruleId = createdRule.id; + expect(ruleId).not.to.be(undefined); + + await muteRule({ + supertest, + ruleId, + }); + + await enableRule({ + supertest, + ruleId, + }); + + // Wait until alerts schedule actions twice to ensure actions had a chance to skip + // execution once before disabling the alert and waiting for tasks to finish + const eventLogResp = await waitForEventLog({ + esClient, + provider: 'alerting', + filter: testStart, + num: 2, + }); + expect(eventLogResp.hits.hits.length >= 2).to.be(true); + + await disableRule({ + supertest, + ruleId, + }); + + await waitForDisabled({ + esClient, + ruleId, + filter: testStart, + }); + + // Should not have executed any action + const resp2 = await getDocumentsInIndex({ + esClient, + indexName: ALERT_ACTION_INDEX, + }); + expect(resp2.hits.hits.length).to.be(0); + }); + + it(`shouldn't schedule actions when alert instance is muted`, async () => { + const testStart = new Date(); + await createIndex({ esClient, indexName: ALERT_ACTION_INDEX }); + + actionId = await createIndexConnector({ + supertest, + name: 'Index Connector: Alerting API test', + indexName: ALERT_ACTION_INDEX, + }); + expect(actionId).not.to.be(undefined); + + const createdRule = await createEsQueryRule({ + supertest, + enabled: false, + consumer: 'alerts', + name: 'always fire', + ruleTypeId: RULE_TYPE_ID, + schedule: { interval: '5s' }, + params: { + size: 100, + thresholdComparator: '>', + threshold: [-1], + index: ['alert-test-data'], + timeField: 'date', + esQuery: `{\n \"query\":{\n \"match_all\" : {}\n }\n}`, + timeWindowSize: 20, + timeWindowUnit: 's', + }, + actions: [ + { + group: 'query matched', + id: actionId, + params: { + documents: [ + { + ruleId: '{{rule.id}}', + ruleName: '{{rule.name}}', + ruleParams: '{{rule.params}}', + spaceId: '{{rule.spaceId}}', + tags: '{{rule.tags}}', + alertId: '{{alert.id}}', + alertActionGroup: '{{alert.actionGroup}}', + instanceContextValue: '{{context.instanceContextValue}}', + instanceStateValue: '{{state.instanceStateValue}}', + }, + ], + }, + frequency: { + notify_when: 'onActiveAlert', + throttle: null, + summary: false, + }, + }, + ], + }); + ruleId = createdRule.id; + expect(ruleId).not.to.be(undefined); + + await muteAlert({ + supertest, + ruleId, + alertId: 'query matched', + }); + + await enableRule({ + supertest, + ruleId, + }); + + // Wait until alerts schedule actions twice to ensure actions had a chance to skip + // execution once before disabling the alert and waiting for tasks to finish + const eventLogResp = await waitForEventLog({ + esClient, + provider: 'alerting', + filter: testStart, + num: 2, + }); + expect(eventLogResp.hits.hits.length >= 2).to.be(true); + + await disableRule({ + supertest, + ruleId, + }); + + await waitForDisabled({ + esClient, + ruleId, + filter: testStart, + }); + + // Should not have executed any action + const resp2 = await getDocumentsInIndex({ + esClient, + indexName: ALERT_ACTION_INDEX, + }); + expect(resp2.hits.hits.length).to.be(0); + }); + + it(`should unmute all instances when unmuting an alert`, async () => { + actionId = await createIndexConnector({ + supertest, + name: 'Index Connector: Alerting API test', + indexName: ALERT_ACTION_INDEX, + }); + expect(actionId).not.to.be(undefined); + + const createdRule = await createEsQueryRule({ + supertest, + enabled: false, + consumer: 'alerts', + name: 'always fire', + ruleTypeId: RULE_TYPE_ID, + params: { + size: 100, + thresholdComparator: '>', + threshold: [-1], + index: ['alert-test-data'], + timeField: 'date', + esQuery: `{\n \"query\":{\n \"match_all\" : {}\n }\n}`, + timeWindowSize: 20, + timeWindowUnit: 's', + }, + actions: [ + { + group: 'query matched', + id: actionId, + params: { + documents: [ + { + ruleId: '{{rule.id}}', + ruleName: '{{rule.name}}', + ruleParams: '{{rule.params}}', + spaceId: '{{rule.spaceId}}', + tags: '{{rule.tags}}', + alertId: '{{alert.id}}', + alertActionGroup: '{{alert.actionGroup}}', + instanceContextValue: '{{context.instanceContextValue}}', + instanceStateValue: '{{state.instanceStateValue}}', + }, + ], + }, + frequency: { + notify_when: 'onActiveAlert', + throttle: null, + summary: false, + }, + }, + ], + }); + ruleId = createdRule.id; + expect(ruleId).not.to.be(undefined); + + await muteAlert({ + supertest, + ruleId, + alertId: 'query matched', + }); + + await muteRule({ + supertest, + ruleId, + }); + + await unmuteRule({ + supertest, + ruleId, + }); + + await enableRule({ + supertest, + ruleId, + }); + + // Should have one document indexed by the action + const resp = await waitForDocumentInIndex({ + esClient, + indexName: ALERT_ACTION_INDEX, + }); + expect(resp.hits.hits.length).to.be(1); + }); + }); +} + +interface ValidateEventLogParams { + ruleId: string; + ruleTypeId: string; + outcome: string; + name: string; + message: string; + errorMessage?: string; +} + +function validateEventLog(event: any, params: ValidateEventLogParams) { + const duration = event?.event?.duration; + const eventStart = Date.parse(event?.event?.start || 'undefined'); + const eventEnd = Date.parse(event?.event?.end || 'undefined'); + const dateNow = Date.now(); + + expect(typeof duration).to.be('string'); + expect(eventStart).to.be.ok(); + expect(eventEnd).to.be.ok(); + + expect(eventStart <= eventEnd).to.equal(true); + expect(eventEnd <= dateNow).to.equal(true); + + const outcome = params.outcome; + expect(event?.event?.outcome).to.equal(outcome); + expect(event?.kibana?.alerting?.outcome).to.equal(outcome); + + expect(event?.kibana?.saved_objects).to.eql([ + { + rel: 'primary', + type: 'alert', + id: params.ruleId, + type_id: params.ruleTypeId, + }, + ]); + + expect(event?.kibana?.alert?.rule?.execution?.metrics?.number_of_triggered_actions).to.be(1); + expect(event?.kibana?.alert?.rule?.execution?.metrics?.number_of_searches).to.be(1); + expect(event?.kibana?.alert?.rule?.execution?.metrics?.es_search_duration_ms).to.be(0); + expect( + event?.kibana?.alert?.rule?.execution?.metrics?.total_search_duration_ms + ).to.be.greaterThan(0); + expect(event?.kibana?.alert?.rule?.execution?.metrics?.alert_counts?.active).to.be(1); + expect(event?.kibana?.alert?.rule?.execution?.metrics?.alert_counts?.new).to.be(1); + expect(event?.kibana?.alert?.rule?.execution?.metrics?.alert_counts?.recovered).to.be(0); + + expect( + event?.kibana?.alert?.rule?.execution?.metrics?.claim_to_start_duration_ms + ).to.be.greaterThan(0); + expect(event?.kibana?.alert?.rule?.execution?.metrics?.total_run_duration_ms).to.be.greaterThan( + 0 + ); + expect( + event?.kibana?.alert?.rule?.execution?.metrics?.prepare_rule_duration_ms + ).to.be.greaterThan(0); + expect( + event?.kibana?.alert?.rule?.execution?.metrics?.rule_type_run_duration_ms + ).to.be.greaterThan(0); + // Process alerts is fast enough that it will sometimes report 0ms + const procesAlertsDurationMs = + event?.kibana?.alert?.rule?.execution?.metrics?.process_alerts_duration_ms; + expect( + (typeof procesAlertsDurationMs === 'number' ? procesAlertsDurationMs : -1) >= 0 + ).to.be.ok(); + expect( + event?.kibana?.alert?.rule?.execution?.metrics?.trigger_actions_duration_ms + ).to.be.greaterThan(0); + expect( + event?.kibana?.alert?.rule?.execution?.metrics?.process_rule_duration_ms + ).to.be.greaterThan(0); + + expect(event?.rule).to.eql({ + id: params.ruleId, + license: 'basic', + category: params.ruleTypeId, + ruleset: 'stackAlerts', + name: params.name, + }); + + expect(event?.message).to.eql(params.message); + + if (params.errorMessage) { + expect(event?.error?.message).to.eql(params.errorMessage); + } +} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/index.ts b/x-pack/test_serverless/api_integration/test_suites/common/index.ts index 3ca6b715102d9..1bfb13f2c5f2c 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/index.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/index.ts @@ -14,5 +14,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./security_response_headers')); loadTestFile(require.resolve('./rollups')); loadTestFile(require.resolve('./index_management')); + loadTestFile(require.resolve('./alerting')); }); }