diff --git a/packages/kbn-securitysolution-utils/src/esql/get_index_list_from_esql_query.test.ts b/packages/kbn-securitysolution-utils/src/esql/get_index_list_from_esql_query.test.ts index ece5c72d5d5ff..5965e51e694e3 100644 --- a/packages/kbn-securitysolution-utils/src/esql/get_index_list_from_esql_query.test.ts +++ b/packages/kbn-securitysolution-utils/src/esql/get_index_list_from_esql_query.test.ts @@ -33,4 +33,9 @@ describe('getIndexListFromEsqlQuery', () => { getIndexPatternFromESQLQueryMock.mockReturnValue('test-1 , test-2 '); expect(getIndexListFromEsqlQuery('From test-1, test-2 ')).toEqual(['test-1', 'test-2']); }); + + it('should return empty array when getIndexPatternFromESQLQuery throws error', () => { + getIndexPatternFromESQLQueryMock.mockReturnValue(new Error('Fail to parse')); + expect(getIndexListFromEsqlQuery('From test-1 []')).toEqual([]); + }); }); diff --git a/packages/kbn-securitysolution-utils/src/esql/get_index_list_from_esql_query.ts b/packages/kbn-securitysolution-utils/src/esql/get_index_list_from_esql_query.ts index 64374732dc716..9464f041bdd64 100644 --- a/packages/kbn-securitysolution-utils/src/esql/get_index_list_from_esql_query.ts +++ b/packages/kbn-securitysolution-utils/src/esql/get_index_list_from_esql_query.ts @@ -11,9 +11,13 @@ import { getIndexPatternFromESQLQuery } from '@kbn/esql-utils'; * parses ES|QL query and returns array of indices */ export const getIndexListFromEsqlQuery = (query: string | undefined): string[] => { - const indexString = getIndexPatternFromESQLQuery(query); + try { + const indexString = getIndexPatternFromESQLQuery(query); - return getIndexListFromIndexString(indexString); + return getIndexListFromIndexString(indexString); + } catch (e) { + return []; + } }; /** diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts index 0b89cbdc72889..b7435c7dd86e8 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts @@ -1267,6 +1267,7 @@ describe('rules schema', () => { // behaviour common for multiple rule types const cases = [ { ruleType: 'threat_match', ruleMock: getCreateThreatMatchRulesSchemaMock() }, + { ruleType: 'esql', ruleMock: getCreateEsqlRulesSchemaMock() }, { ruleType: 'query', ruleMock: getCreateRulesSchemaMock() }, { ruleType: 'saved_query', ruleMock: getCreateSavedQueryRulesSchemaMock() }, { ruleType: 'eql', ruleMock: getCreateEqlRuleSchemaMock() }, diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts index d2523a9a5c557..278b4679cd93e 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts @@ -560,14 +560,19 @@ export const EsqlRuleRequiredFields = z.object({ query: RuleQuery, }); +export type EsqlRuleOptionalFields = z.infer; +export const EsqlRuleOptionalFields = z.object({ + alert_suppression: AlertSuppression.optional(), +}); + export type EsqlRulePatchFields = z.infer; -export const EsqlRulePatchFields = EsqlRuleRequiredFields.partial(); +export const EsqlRulePatchFields = EsqlRuleOptionalFields.merge(EsqlRuleRequiredFields.partial()); export type EsqlRuleResponseFields = z.infer; -export const EsqlRuleResponseFields = EsqlRuleRequiredFields; +export const EsqlRuleResponseFields = EsqlRuleOptionalFields.merge(EsqlRuleRequiredFields); export type EsqlRuleCreateFields = z.infer; -export const EsqlRuleCreateFields = EsqlRuleRequiredFields; +export const EsqlRuleCreateFields = EsqlRuleOptionalFields.merge(EsqlRuleRequiredFields); export type EsqlRule = z.infer; export const EsqlRule = SharedResponseProps.merge(EsqlRuleResponseFields); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml index ae1a5657d2ab4..de424af505c1f 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml @@ -826,17 +826,26 @@ components: - language - query + EsqlRuleOptionalFields: + type: object + properties: + alert_suppression: + $ref: './common_attributes.schema.yaml#/components/schemas/AlertSuppression' + EsqlRulePatchFields: allOf: + - $ref: '#/components/schemas/EsqlRuleOptionalFields' - $ref: '#/components/schemas/EsqlRuleRequiredFields' x-modify: partial EsqlRuleResponseFields: allOf: + - $ref: '#/components/schemas/EsqlRuleOptionalFields' - $ref: '#/components/schemas/EsqlRuleRequiredFields' EsqlRuleCreateFields: allOf: + - $ref: '#/components/schemas/EsqlRuleOptionalFields' - $ref: '#/components/schemas/EsqlRuleRequiredFields' EsqlRule: diff --git a/x-pack/plugins/security_solution/common/detection_engine/constants.ts b/x-pack/plugins/security_solution/common/detection_engine/constants.ts index b3bc1f434ff51..54c81cf93568f 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/constants.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/constants.ts @@ -41,6 +41,7 @@ export const MINIMUM_LICENSE_FOR_SUPPRESSION = 'platinum' as const; export const SUPPRESSIBLE_ALERT_RULES: Type[] = [ 'threshold', + 'esql', 'saved_query', 'query', 'new_terms', diff --git a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts index e0c76253c416a..2e5ac39936fa3 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts @@ -229,6 +229,7 @@ describe('Alert Suppression Rules', () => { describe('isSuppressibleAlertRule', () => { test('should return true for a suppressible rule type', () => { // Rule types that support alert suppression: + expect(isSuppressibleAlertRule('esql')).toBe(true); expect(isSuppressibleAlertRule('threshold')).toBe(true); expect(isSuppressibleAlertRule('saved_query')).toBe(true); expect(isSuppressibleAlertRule('query')).toBe(true); @@ -238,7 +239,6 @@ describe('Alert Suppression Rules', () => { // Rule types that don't support alert suppression: expect(isSuppressibleAlertRule('machine_learning')).toBe(false); - expect(isSuppressibleAlertRule('esql')).toBe(false); }); test('should return false for an unknown rule type', () => { @@ -266,6 +266,7 @@ describe('Alert Suppression Rules', () => { describe('isSuppressionRuleConfiguredWithDuration', () => { test('should return true for a suppressible rule type', () => { // Rule types that support alert suppression: + expect(isSuppressionRuleConfiguredWithDuration('esql')).toBe(true); expect(isSuppressionRuleConfiguredWithDuration('threshold')).toBe(true); expect(isSuppressionRuleConfiguredWithDuration('saved_query')).toBe(true); expect(isSuppressionRuleConfiguredWithDuration('query')).toBe(true); @@ -275,7 +276,6 @@ describe('Alert Suppression Rules', () => { // Rule types that don't support alert suppression: expect(isSuppressionRuleConfiguredWithDuration('machine_learning')).toBe(false); - expect(isSuppressionRuleConfiguredWithDuration('esql')).toBe(false); }); test('should return false for an unknown rule type', () => { @@ -288,6 +288,7 @@ describe('Alert Suppression Rules', () => { describe('isSuppressionRuleConfiguredWithGroupBy', () => { test('should return true for a suppressible rule type with groupBy', () => { // Rule types that support alert suppression groupBy: + expect(isSuppressionRuleConfiguredWithGroupBy('esql')).toBe(true); expect(isSuppressionRuleConfiguredWithGroupBy('saved_query')).toBe(true); expect(isSuppressionRuleConfiguredWithGroupBy('query')).toBe(true); expect(isSuppressionRuleConfiguredWithGroupBy('threat_match')).toBe(true); @@ -296,7 +297,6 @@ describe('Alert Suppression Rules', () => { // Rule types that don't support alert suppression: expect(isSuppressionRuleConfiguredWithGroupBy('machine_learning')).toBe(false); - expect(isSuppressionRuleConfiguredWithGroupBy('esql')).toBe(false); }); test('should return false for a threshold rule type', () => { @@ -314,6 +314,7 @@ describe('Alert Suppression Rules', () => { describe('isSuppressionRuleConfiguredWithMissingFields', () => { test('should return true for a suppressible rule type with missing fields', () => { // Rule types that support alert suppression groupBy: + expect(isSuppressionRuleConfiguredWithMissingFields('esql')).toBe(true); expect(isSuppressionRuleConfiguredWithMissingFields('saved_query')).toBe(true); expect(isSuppressionRuleConfiguredWithMissingFields('query')).toBe(true); expect(isSuppressionRuleConfiguredWithMissingFields('threat_match')).toBe(true); @@ -322,7 +323,6 @@ describe('Alert Suppression Rules', () => { // Rule types that don't support alert suppression: expect(isSuppressionRuleConfiguredWithMissingFields('machine_learning')).toBe(false); - expect(isSuppressionRuleConfiguredWithMissingFields('esql')).toBe(false); }); test('should return false for a threshold rule type', () => { diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index da993a18debe3..565177fa8b560 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -180,6 +180,11 @@ export const allowedExperimentalValues = Object.freeze({ */ disableTimelineSaveTour: false, + /** + * Enables alerts suppression for ES|QL rules + */ + alertSuppressionForEsqlRuleEnabled: false, + /** * Enables the risk engine privileges route * and associated callout in the UI diff --git a/x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/installation_and_upgrade.md b/x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/installation_and_upgrade.md index 24aaf34764034..beda8a9517830 100644 --- a/x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/installation_and_upgrade.md +++ b/x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/installation_and_upgrade.md @@ -198,6 +198,9 @@ Examples: │ New Terms │ Custom query │ Overview │ Definition │ │ New Terms │ Filters │ Overview │ Definition │ │ ESQL │ ESQL query │ Overview │ Definition │ +│ ESQL │ Suppress alerts by │ Overview │ Definition │ +│ ESQL │ Suppress alerts for │ Overview │ Definition │ +│ ESQL │ If a suppression field is missing │ Overview │ Definition │ ``` ## Scenarios diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts index 889d74a1c6503..07f14830d6a71 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts @@ -5,7 +5,13 @@ * 2.0. */ -import { computeHasMetadataOperator } from './esql_validator'; +import { parseEsqlQuery, computeHasMetadataOperator } from './esql_validator'; + +import { computeIsESQLQueryAggregating } from '@kbn/securitysolution-utils'; + +jest.mock('@kbn/securitysolution-utils', () => ({ computeIsESQLQueryAggregating: jest.fn() })); + +const computeIsESQLQueryAggregatingMock = computeIsESQLQueryAggregating as jest.Mock; describe('computeHasMetadataOperator', () => { it('should be false if query does not have operator', () => { @@ -44,3 +50,37 @@ describe('computeHasMetadataOperator', () => { ).toBe(true); }); }); + +describe('parseEsqlQuery', () => { + it('returns isMissingMetadataOperator true when query is not aggregating and does not have metadata operator', () => { + computeIsESQLQueryAggregatingMock.mockReturnValueOnce(false); + + expect(parseEsqlQuery('from test*')).toEqual({ + isEsqlQueryAggregating: false, + isMissingMetadataOperator: true, + }); + }); + + it('returns isMissingMetadataOperator false when query is not aggregating and has metadata operator', () => { + computeIsESQLQueryAggregatingMock.mockReturnValueOnce(false); + + expect(parseEsqlQuery('from test* metadata _id')).toEqual({ + isEsqlQueryAggregating: false, + isMissingMetadataOperator: false, + }); + }); + + it('returns isMissingMetadataOperator false when query is aggregating', () => { + computeIsESQLQueryAggregatingMock.mockReturnValue(true); + + expect(parseEsqlQuery('from test*')).toEqual({ + isEsqlQueryAggregating: true, + isMissingMetadataOperator: false, + }); + + expect(parseEsqlQuery('from test* metadata _id')).toEqual({ + isEsqlQueryAggregating: true, + isMissingMetadataOperator: false, + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts index b7e9f1b033e31..1f0bcb6596b40 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts @@ -6,11 +6,10 @@ */ import { isEmpty } from 'lodash'; - +import type { QueryClient } from '@tanstack/react-query'; import { computeIsESQLQueryAggregating } from '@kbn/securitysolution-utils'; import { KibanaServices } from '../../../common/lib/kibana'; -import { securitySolutionQueryClient } from '../../../common/containers/query_client/query_client_provider'; import type { ValidationError, ValidationFunc } from '../../../shared_imports'; import { isEsqlRule } from '../../../../common/detection_engine/utils'; @@ -48,7 +47,7 @@ export const computeHasMetadataOperator = (esqlQuery: string) => { export const esqlValidator = async ( ...args: Parameters ): Promise | void | undefined> => { - const [{ value, formData }] = args; + const [{ value, formData, customData }] = args; const { query: queryValue } = value as FieldValueQueryBar; const query = queryValue.query as string; const { ruleType } = formData as DefineStepRule; @@ -59,19 +58,19 @@ export const esqlValidator = async ( } try { - const services = KibanaServices.get(); + const queryClient = (customData.value as { queryClient: QueryClient | undefined })?.queryClient; - const isEsqlQueryAggregating = computeIsESQLQueryAggregating(query); + const services = KibanaServices.get(); + const { isEsqlQueryAggregating, isMissingMetadataOperator } = parseEsqlQuery(query); - // non-aggregating query which does not have metadata, is not a valid one - if (!isEsqlQueryAggregating && !computeHasMetadataOperator(query)) { + if (isMissingMetadataOperator) { return { code: ERROR_CODES.ERR_MISSING_ID_FIELD_FROM_RESULT, message: i18n.ESQL_VALIDATION_MISSING_ID_IN_QUERY_ERROR, }; } - const columns = await securitySolutionQueryClient.fetchQuery( + const columns = await queryClient?.fetchQuery( getEsqlQueryConfig({ esqlQuery: query, search: services.data.search.search }) ); @@ -92,3 +91,17 @@ export const esqlValidator = async ( return constructValidationError(error); } }; + +/** + * check if esql query valid for Security rule: + * - if it's non aggregation query it must have metadata operator + */ +export const parseEsqlQuery = (query: string) => { + const isEsqlQueryAggregating = computeIsESQLQueryAggregating(query); + + return { + isEsqlQueryAggregating, + // non-aggregating query which does not have [metadata], is not a valid one + isMissingMetadataOperator: !isEsqlQueryAggregating && !computeHasMetadataOperator(query), + }; +}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx index 7950493a1b989..8695041697120 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx @@ -575,7 +575,7 @@ describe('description_step', () => { }); describe('alert suppression', () => { - const ruleTypesWithoutSuppression: Type[] = ['esql', 'machine_learning']; + const ruleTypesWithoutSuppression: Type[] = ['machine_learning']; const suppressionFields = { groupByDuration: { unit: 'm', diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.tsx index 3fa1852e6aa05..7666a9ba8aee3 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.tsx @@ -42,7 +42,7 @@ import { useKibana } from '../../../../common/lib/kibana'; import { useRuleIndices } from '../../../rule_management/logic/use_rule_indices'; import { EsqlAutocomplete } from '../esql_autocomplete'; import { MultiSelectFieldsAutocomplete } from '../multi_select_fields'; -import { useInvestigationFields } from '../../hooks/use_investigation_fields'; +import { useAllEsqlRuleFields } from '../../hooks'; import { MaxSignals } from '../max_signals'; const CommonUseField = getUseField({ component: Field }); @@ -133,10 +133,11 @@ const StepAboutRuleComponent: FC = ({ [getFields] ); - const { investigationFields, isLoading: isInvestigationFieldsLoading } = useInvestigationFields({ - esqlQuery: isEsqlRuleValue ? esqlQuery : undefined, - indexPatternsFields: indexPattern.fields, - }); + const { fields: investigationFields, isLoading: isInvestigationFieldsLoading } = + useAllEsqlRuleFields({ + esqlQuery: isEsqlRuleValue ? esqlQuery : undefined, + indexPatternsFields: indexPattern.fields, + }); return ( <> diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx index 56073e2a6af59..839454922a14a 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx @@ -28,6 +28,7 @@ import type { FieldSpec } from '@kbn/data-views-plugin/common'; import usePrevious from 'react-use/lib/usePrevious'; import type { BrowserFields } from '@kbn/timelines-plugin/common'; import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; +import { useQueryClient } from '@tanstack/react-query'; import type { SavedQuery } from '@kbn/data-plugin/public'; import type { DataViewBase } from '@kbn/es-query'; @@ -99,6 +100,7 @@ import { AlertSuppressionMissingFieldsStrategyEnum } from '../../../../../common import { DurationInput } from '../duration_input'; import { MINIMUM_LICENSE_FOR_SUPPRESSION } from '../../../../../common/detection_engine/constants'; import { useUpsellingMessage } from '../../../../common/hooks/use_upselling'; +import { useAllEsqlRuleFields } from '../../hooks'; import { useAlertSuppression } from '../../../rule_management/logic/use_alert_suppression'; import { RelatedIntegrations } from '../../../rule_creation/components/related_integrations'; @@ -191,6 +193,8 @@ const StepDefineRuleComponent: FC = ({ thresholdFields, enableThresholdSuppression, }) => { + const queryClient = useQueryClient(); + const { isSuppressionEnabled: isAlertSuppressionEnabled } = useAlertSuppression(ruleType); const mlCapabilities = useMlCapabilities(); const [openTimelineSearch, setOpenTimelineSearch] = useState(false); @@ -453,6 +457,13 @@ const StepDefineRuleComponent: FC = ({ ); const [{ queryBar }] = useFormData({ form, watch: ['queryBar'] }); + + const { fields: esqlSuppressionFields, isLoading: isEsqlSuppressionLoading } = + useAllEsqlRuleFields({ + esqlQuery: isEsqlRule(ruleType) ? (queryBar?.query?.query as string) : undefined, + indexPatternsFields: indexPattern.fields, + }); + const areSuppressionFieldsDisabledBySequence = isEqlRule(ruleType) && isEqlSequenceQuery(queryBar?.query?.query as string) && @@ -745,6 +756,7 @@ const StepDefineRuleComponent: FC = ({ path="queryBar" config={esqlQueryBarConfig} component={QueryBarDefineRule} + validationData={{ queryClient }} componentProps={{ ...queryBarProps, dataTestSubj: 'detectionEngineStepDefineRuleEsqlQueryBar', @@ -752,7 +764,7 @@ const StepDefineRuleComponent: FC = ({ }} /> ), - [queryBarProps, esqlQueryBarConfig] + [queryBarProps, esqlQueryBarConfig, queryClient] ); const QueryBarMemo = useMemo( @@ -1060,9 +1072,13 @@ const StepDefineRuleComponent: FC = ({ path="groupByFields" component={MultiSelectFieldsAutocomplete} componentProps={{ - browserFields: termsAggregationFields, + browserFields: isEsqlRule(ruleType) + ? esqlSuppressionFields + : termsAggregationFields, isDisabled: - !isAlertSuppressionLicenseValid || areSuppressionFieldsDisabledBySequence, + !isAlertSuppressionLicenseValid || + areSuppressionFieldsDisabledBySequence || + isEsqlSuppressionLoading, disabledText: areSuppressionFieldsDisabledBySequence ? i18n.EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP : alertSuppressionUpsellingMessage, diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts index c035fef5af6e4..c92c35688dd3b 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts @@ -7,6 +7,8 @@ import { useCallback } from 'react'; import type { DefineStepRule } from '../../../../detections/pages/detection_engine/rules/types'; +import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features'; +import { isEsqlRule } from '../../../../../common/detection_engine/utils'; /** * transforms DefineStepRule fields according to experimental feature flags @@ -14,9 +16,30 @@ import type { DefineStepRule } from '../../../../detections/pages/detection_engi export const useExperimentalFeatureFieldsTransform = >(): (( fields: T ) => T) => { - const transformer = useCallback((fields: T) => { - return fields; - }, []); + const isAlertSuppressionForEsqlRuleEnabled = useIsExperimentalFeatureEnabled( + 'alertSuppressionForEsqlRuleEnabled' + ); + + const transformer = useCallback( + (fields: T) => { + const isSuppressionDisabled = + isEsqlRule(fields.ruleType) && !isAlertSuppressionForEsqlRuleEnabled; + + // reset any alert suppression values hidden behind feature flag + if (isSuppressionDisabled) { + return { + ...fields, + groupByFields: [], + groupByRadioSelection: undefined, + groupByDuration: undefined, + suppressionMissingFields: undefined, + }; + } + + return fields; + }, + [isAlertSuppressionForEsqlRuleEnabled] + ); return transformer; }; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/index.tsx index d56c317b93fb1..ea248587365aa 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/index.tsx @@ -7,3 +7,4 @@ export { useEsqlIndex } from './use_esql_index'; export { useEsqlQueryForAboutStep } from './use_esql_query_for_about_step'; +export { useAllEsqlRuleFields } from './use_all_esql_rule_fields'; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_investigation_fields.test.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_all_esql_rule_fields.test.ts similarity index 50% rename from x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_investigation_fields.test.ts rename to x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_all_esql_rule_fields.test.ts index 597d44f47f0d9..996b3ca044864 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_investigation_fields.test.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_all_esql_rule_fields.test.ts @@ -7,16 +7,15 @@ import { renderHook } from '@testing-library/react-hooks'; import type { DataViewFieldBase } from '@kbn/es-query'; +import { getESQLQueryColumns } from '@kbn/esql-utils'; -import { useInvestigationFields } from './use_investigation_fields'; +import { useAllEsqlRuleFields } from './use_all_esql_rule_fields'; import { createQueryWrapperMock } from '../../../common/__mocks__/query_wrapper'; +import { parseEsqlQuery } from '../../rule_creation/logic/esql_validator'; -import { computeIsESQLQueryAggregating } from '@kbn/securitysolution-utils'; -import { getESQLQueryColumns } from '@kbn/esql-utils'; - -jest.mock('@kbn/securitysolution-utils', () => ({ - computeIsESQLQueryAggregating: jest.fn(), +jest.mock('../../rule_creation/logic/esql_validator', () => ({ + parseEsqlQuery: jest.fn(), })); jest.mock('@kbn/esql-utils', () => { @@ -26,7 +25,7 @@ jest.mock('@kbn/esql-utils', () => { }; }); -const computeIsESQLQueryAggregatingMock = computeIsESQLQueryAggregating as jest.Mock; +const parseEsqlQueryMock = parseEsqlQuery as jest.Mock; const getESQLQueryColumnsMock = getESQLQueryColumns as jest.Mock; const { wrapper } = createQueryWrapperMock(); @@ -48,16 +47,26 @@ const mockEsqlDatatable = { columns: [{ id: '_custom_field', name: '_custom_field', meta: { type: 'string' } }], }; -describe('useInvestigationFields', () => { +describe('useAllEsqlRuleFields', () => { beforeEach(() => { jest.clearAllMocks(); - getESQLQueryColumnsMock.mockResolvedValue(mockEsqlDatatable.columns); + getESQLQueryColumnsMock.mockImplementation(({ esqlQuery }) => + Promise.resolve( + esqlQuery === 'deduplicate_test' + ? [ + { id: 'agent.name', name: 'agent.name', meta: { type: 'string' } }, // agent.name is already present in mockIndexPatternFields + { id: '_custom_field_0', name: '_custom_field_0', meta: { type: 'string' } }, + ] + : mockEsqlDatatable.columns + ) + ); + parseEsqlQueryMock.mockReturnValue({ isEsqlQueryAggregating: false }); }); it('should return loading true when esql fields still loading', () => { const { result } = renderHook( () => - useInvestigationFields({ + useAllEsqlRuleFields({ esqlQuery: mockEsqlQuery, indexPatternsFields: mockIndexPatternFields, }), @@ -68,71 +77,103 @@ describe('useInvestigationFields', () => { }); it('should return only index pattern fields when ES|QL query is empty', async () => { - const { result, waitForNextUpdate } = renderHook( + const { result } = renderHook( () => - useInvestigationFields({ + useAllEsqlRuleFields({ esqlQuery: '', indexPatternsFields: mockIndexPatternFields, }), { wrapper } ); - await waitForNextUpdate(); - - expect(result.current.investigationFields).toEqual(mockIndexPatternFields); + expect(result.current.fields).toEqual(mockIndexPatternFields); }); it('should return only index pattern fields when ES|QL query is undefined', async () => { const { result } = renderHook( () => - useInvestigationFields({ + useAllEsqlRuleFields({ esqlQuery: undefined, indexPatternsFields: mockIndexPatternFields, }), { wrapper } ); - expect(result.current.investigationFields).toEqual(mockIndexPatternFields); + expect(result.current.fields).toEqual(mockIndexPatternFields); }); it('should return index pattern fields concatenated with ES|QL fields when ES|QL query is non-aggregating', async () => { - computeIsESQLQueryAggregatingMock.mockReturnValue(false); + parseEsqlQueryMock.mockReturnValue({ isEsqlQueryAggregating: false }); - const { result } = renderHook( + const { result, waitFor } = renderHook( () => - useInvestigationFields({ + useAllEsqlRuleFields({ esqlQuery: mockEsqlQuery, indexPatternsFields: mockIndexPatternFields, }), { wrapper } ); - expect(result.current.investigationFields).toEqual([ - { - name: '_custom_field', - type: 'string', - }, - ...mockIndexPatternFields, - ]); + await waitFor(() => { + expect(result.current.fields).toEqual([ + { + name: '_custom_field', + type: 'string', + }, + ...mockIndexPatternFields, + ]); + }); }); it('should return only ES|QL fields when ES|QL query is aggregating', async () => { - computeIsESQLQueryAggregatingMock.mockReturnValue(true); + parseEsqlQueryMock.mockReturnValue({ isEsqlQueryAggregating: true }); - const { result } = renderHook( + const { result, waitFor } = renderHook( () => - useInvestigationFields({ + useAllEsqlRuleFields({ esqlQuery: mockEsqlQuery, indexPatternsFields: mockIndexPatternFields, }), { wrapper } ); + await waitFor(() => { + expect(result.current.fields).toEqual([ + { + name: '_custom_field', + type: 'string', + }, + ]); + }); + }); + + it('should deduplicate index pattern fields and ES|QL fields when fields have same name', async () => { + // getESQLQueryColumnsMock.mockClear(); + parseEsqlQueryMock.mockReturnValue({ isEsqlQueryAggregating: false }); + + const { result, waitFor } = renderHook( + () => + useAllEsqlRuleFields({ + esqlQuery: 'deduplicate_test', + indexPatternsFields: mockIndexPatternFields, + }), + { wrapper } + ); - expect(result.current.investigationFields).toEqual([ - { - name: '_custom_field', - type: 'string', - }, - ]); + await waitFor(() => { + expect(result.current.fields).toEqual([ + { + name: 'agent.name', + type: 'string', + }, + { + name: '_custom_field_0', + type: 'string', + }, + { + name: 'agent.type', + type: 'string', + }, + ]); + }); }); }); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_all_esql_rule_fields.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_all_esql_rule_fields.ts new file mode 100644 index 0000000000000..a67b990c88b80 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_all_esql_rule_fields.ts @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { useMemo, useState } from 'react'; +import type { DatatableColumn } from '@kbn/expressions-plugin/public'; +import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; +import type { DataViewFieldBase } from '@kbn/es-query'; +import useDebounce from 'react-use/lib/useDebounce'; + +import { useQuery } from '@tanstack/react-query'; + +import { useKibana } from '@kbn/kibana-react-plugin/public'; +import { parseEsqlQuery } from '../../rule_creation/logic/esql_validator'; + +import { getEsqlQueryConfig } from '../../rule_creation/logic/get_esql_query_config'; + +const esqlToFields = ( + columns: { error: unknown } | DatatableColumn[] | undefined | null +): DataViewFieldBase[] => { + if (columns && 'error' in columns) { + return []; + } + + const fields = (columns ?? []).map(({ id, meta }) => { + return { + name: id, + type: meta.type, + }; + }); + + return fields; +}; + +type UseEsqlFields = (esqlQuery: string | undefined) => { + isLoading: boolean; + fields: DataViewFieldBase[]; +}; + +/** + * fetches ES|QL fields and convert them to DataViewBase fields + */ +export const useEsqlFields: UseEsqlFields = (esqlQuery) => { + const kibana = useKibana<{ data: DataPublicPluginStart }>(); + + const { data: dataService } = kibana.services; + + const queryConfig = getEsqlQueryConfig({ esqlQuery, search: dataService?.search?.search }); + const { data, isLoading } = useQuery(queryConfig); + + const fields = useMemo(() => { + return esqlToFields(data); + }, [data]); + + return { + fields, + isLoading, + }; +}; + +/** + * if ES|QL fields and index pattern fields have same name, duplicates will be removed and the rest of fields merged + * ES|QL fields are first in order, since these are the fields that returned in ES|QL response + * */ +const deduplicateAndMergeFields = ( + esqlFields: DataViewFieldBase[], + indexPatternsFields: DataViewFieldBase[] +) => { + const esqlFieldsSet = new Set(esqlFields.map((field) => field.name)); + return [...esqlFields, ...indexPatternsFields.filter((field) => !esqlFieldsSet.has(field.name))]; +}; + +type UseAllEsqlRuleFields = (params: { + esqlQuery: string | undefined; + indexPatternsFields: DataViewFieldBase[]; +}) => { + isLoading: boolean; + fields: DataViewFieldBase[]; +}; + +/** + * returns all fields available for ES|QL rule: + * - fields returned from ES|QL query for aggregating queries + * - fields returned from ES|QL query + index fields for non-aggregating queries + */ +export const useAllEsqlRuleFields: UseAllEsqlRuleFields = ({ esqlQuery, indexPatternsFields }) => { + const [debouncedEsqlQuery, setDebouncedEsqlQuery] = useState(undefined); + const { fields: esqlFields, isLoading } = useEsqlFields(debouncedEsqlQuery); + + const { isEsqlQueryAggregating } = useMemo( + () => parseEsqlQuery(debouncedEsqlQuery ?? ''), + [debouncedEsqlQuery] + ); + + useDebounce( + () => { + setDebouncedEsqlQuery(esqlQuery); + }, + 300, + [esqlQuery] + ); + + const fields = useMemo(() => { + if (!debouncedEsqlQuery) { + return indexPatternsFields; + } + return isEsqlQueryAggregating + ? esqlFields + : deduplicateAndMergeFields(esqlFields, indexPatternsFields); + }, [esqlFields, debouncedEsqlQuery, indexPatternsFields, isEsqlQueryAggregating]); + + return { + fields, + isLoading, + }; +}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_investigation_fields.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_investigation_fields.ts deleted file mode 100644 index 1627bddaa82be..0000000000000 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_investigation_fields.ts +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { useMemo } from 'react'; -import type { DatatableColumn } from '@kbn/expressions-plugin/public'; -import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; -import type { DataViewFieldBase } from '@kbn/es-query'; -import { computeIsESQLQueryAggregating } from '@kbn/securitysolution-utils'; - -import { useQuery } from '@tanstack/react-query'; - -import { useKibana } from '@kbn/kibana-react-plugin/public'; - -import { getEsqlQueryConfig } from '../../rule_creation/logic/get_esql_query_config'; - -const esqlToFields = ( - columns: { error: unknown } | DatatableColumn[] | undefined | null -): DataViewFieldBase[] => { - if (columns && 'error' in columns) { - return []; - } - - const fields = (columns ?? []).map(({ id, meta }) => { - return { - name: id, - type: meta.type, - }; - }); - - return fields; -}; - -type UseEsqlFields = (esqlQuery: string | undefined) => { - isLoading: boolean; - fields: DataViewFieldBase[]; -}; - -/** - * fetches ES|QL fields and convert them to DataViewBase fields - */ -const useEsqlFields: UseEsqlFields = (esqlQuery) => { - const kibana = useKibana<{ data: DataPublicPluginStart }>(); - - const { data: dataService } = kibana.services; - - const queryConfig = getEsqlQueryConfig({ esqlQuery, search: dataService?.search?.search }); - const { data, isLoading } = useQuery(queryConfig); - - const fields = useMemo(() => { - return esqlToFields(data); - }, [data]); - - return { - fields, - isLoading, - }; -}; - -type UseInvestigationFields = (params: { - esqlQuery: string | undefined; - indexPatternsFields: DataViewFieldBase[]; -}) => { - isLoading: boolean; - investigationFields: DataViewFieldBase[]; -}; - -export const useInvestigationFields: UseInvestigationFields = ({ - esqlQuery, - indexPatternsFields, -}) => { - const { fields: esqlFields, isLoading } = useEsqlFields(esqlQuery); - - const investigationFields = useMemo(() => { - if (!esqlQuery) { - return indexPatternsFields; - } - - // alerts generated from non-aggregating queries are enriched with source document - // so, index patterns fields should be included in the list of investigation fields - const isEsqlQueryAggregating = computeIsESQLQueryAggregating(esqlQuery); - - return isEsqlQueryAggregating ? esqlFields : [...esqlFields, ...indexPatternsFields]; - }, [esqlFields, esqlQuery, indexPatternsFields]); - - return { - investigationFields, - isLoading, - }; -}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts index 11da431e3e602..f281b3b6b4a2b 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts @@ -512,6 +512,7 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep language: ruleFields.queryBar?.query?.language, query: ruleFields.queryBar?.query?.query as string, required_fields: requiredFields, + ...alertSuppressionFields, } : { ...alertSuppressionFields, diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx index 0d9c0ebb75ca4..d12a5ff97d50a 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx @@ -36,4 +36,19 @@ describe('useAlertSuppression', () => { expect(result.current.isSuppressionEnabled).toBe(false); }); + + it('should return isSuppressionEnabled false if ES|QL Feature Flag is disabled', () => { + const { result } = renderHook(() => useAlertSuppression('esql')); + + expect(result.current.isSuppressionEnabled).toBe(false); + }); + + it('should return isSuppressionEnabled true if ES|QL Feature Flag is enabled', () => { + jest + .spyOn(useIsExperimentalFeatureEnabledMock, 'useIsExperimentalFeatureEnabled') + .mockImplementation((flag) => flag === 'alertSuppressionForEsqlRuleEnabled'); + const { result } = renderHook(() => useAlertSuppression('esql')); + + expect(result.current.isSuppressionEnabled).toBe(true); + }); }); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx index 6e1b2a4d6163f..1c9f139633c8c 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx @@ -7,19 +7,28 @@ import { useCallback } from 'react'; import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; import { isSuppressibleAlertRule } from '../../../../common/detection_engine/utils'; +import { useIsExperimentalFeatureEnabled } from '../../../common/hooks/use_experimental_features'; export interface UseAlertSuppressionReturn { isSuppressionEnabled: boolean; } export const useAlertSuppression = (ruleType: Type | undefined): UseAlertSuppressionReturn => { + const isAlertSuppressionForEsqlRuleEnabled = useIsExperimentalFeatureEnabled( + 'alertSuppressionForEsqlRuleEnabled' + ); + const isSuppressionEnabledForRuleType = useCallback(() => { if (!ruleType) { return false; } + // Remove this condition when the Feature Flag for enabling Suppression in the New terms rule is removed. + if (ruleType === 'esql') { + return isSuppressibleAlertRule(ruleType) && isAlertSuppressionForEsqlRuleEnabled; + } return isSuppressibleAlertRule(ruleType); - }, [ruleType]); + }, [ruleType, isAlertSuppressionForEsqlRuleEnabled]); return { isSuppressionEnabled: isSuppressionEnabledForRuleType(), diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx index 3a8153d9fa2b3..7a3ed25b8084e 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx @@ -42,7 +42,7 @@ import { ALERT_NEW_TERMS, ALERT_RULE_INDICES, } from '../../../../common/field_maps/field_names'; -import { isEqlRule } from '../../../../common/detection_engine/utils'; +import { isEqlRule, isEsqlRule } from '../../../../common/detection_engine/utils'; import type { TimelineResult } from '../../../../common/api/timeline'; import { TimelineId } from '../../../../common/types/timeline'; import { TimelineStatus, TimelineType } from '../../../../common/api/timeline'; @@ -279,6 +279,11 @@ export const isEqlAlert = (ecsData: Ecs): boolean => { return isEqlRule(ruleType) || (Array.isArray(ruleType) && isEqlRule(ruleType[0])); }; +export const isEsqlAlert = (ecsData: Ecs): boolean => { + const ruleType = getField(ecsData, ALERT_RULE_TYPE); + return isEsqlRule(ruleType) || (Array.isArray(ruleType) && isEsqlRule(ruleType[0])); +}; + export const isNewTermsAlert = (ecsData: Ecs): boolean => { const ruleType = getField(ecsData, ALERT_RULE_TYPE); return ( @@ -1026,8 +1031,8 @@ export const sendAlertToTimelineAction = async ({ }, getExceptionFilter ); - // The Query field should remain unpopulated with the suppressed EQL alert. - } else if (isSuppressedAlert(ecsData) && !isEqlAlert(ecsData)) { + // The Query field should remain unpopulated with the suppressed EQL/ES|QL alert. + } else if (isSuppressedAlert(ecsData) && !isEqlAlert(ecsData) && !isEsqlAlert(ecsData)) { return createSuppressedTimeline( ecsData, createTimeline, @@ -1097,8 +1102,8 @@ export const sendAlertToTimelineAction = async ({ return createThresholdTimeline(ecsData, createTimeline, noteContent, {}, getExceptionFilter); } else if (isNewTermsAlert(ecsData)) { return createNewTermsTimeline(ecsData, createTimeline, noteContent, {}, getExceptionFilter); - // The Query field should remain unpopulated with the suppressed EQL alert. - } else if (isSuppressedAlert(ecsData) && !isEqlAlert(ecsData)) { + // The Query field should remain unpopulated with the suppressed EQL/ES|QL alert. + } else if (isSuppressedAlert(ecsData) && !isEqlAlert(ecsData) && !isEsqlAlert(ecsData)) { return createSuppressedTimeline(ecsData, createTimeline, noteContent, {}, getExceptionFilter); } else { let { dataProviders, filters } = buildTimelineDataProviderOrFilter( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts index d0a14151cabac..537d7b6abaf8a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts @@ -13,6 +13,7 @@ import { import { getBaseRuleParams, getEqlRuleParams, + getEsqlRuleParams, getMlRuleParams, getNewTermsRuleParams, getQueryRuleParams, @@ -219,6 +220,27 @@ describe('rule_converters', () => { ); }); + test('should accept ES|QL alerts suppression params', () => { + const patchParams = { + alert_suppression: { + group_by: ['agent.name'], + duration: { value: 4, unit: 'h' as const }, + missing_fields_strategy: 'doNotSuppress' as const, + }, + }; + const rule = getEsqlRuleParams(); + const patchedParams = patchTypeSpecificSnakeToCamel(patchParams, rule); + expect(patchedParams).toEqual( + expect.objectContaining({ + alertSuppression: { + groupBy: ['agent.name'], + missingFieldsStrategy: 'doNotSuppress', + duration: { value: 4, unit: 'h' }, + }, + }) + ); + }); + test('should accept threshold alerts suppression params', () => { const patchParams = { alert_suppression: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts index 355fa626f7848..fd77213b178b5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts @@ -121,6 +121,7 @@ export const typeSpecificSnakeToCamel = ( type: params.type, language: params.language, query: params.query, + alertSuppression: convertAlertSuppressionToCamel(params.alert_suppression), }; } case 'threat_match': { @@ -237,6 +238,8 @@ const patchEsqlParams = ( type: existingRule.type, language: params.language ?? existingRule.language, query: params.query ?? existingRule.query, + alertSuppression: + convertAlertSuppressionToCamel(params.alert_suppression) ?? existingRule.alertSuppression, }; }; @@ -568,6 +571,7 @@ export const typeSpecificCamelToSnake = ( type: params.type, language: params.language, query: params.query, + alert_suppression: convertAlertSuppressionToSnake(params.alertSuppression), }; } case 'threat_match': { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.mock.ts index 9ebf17caf9a85..3a4fa1dadd778 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.mock.ts @@ -12,6 +12,7 @@ import type { BaseRuleParams, CompleteRule, EqlRuleParams, + EsqlRuleParams, MachineLearningRuleParams, NewTermsRuleParams, QueryRuleParams, @@ -103,6 +104,16 @@ export const getEqlRuleParams = (rewrites?: Partial): EqlRulePara }; }; +export const getEsqlRuleParams = (rewrites?: Partial): EsqlRuleParams => { + return { + ...getBaseRuleParams(), + type: 'esql', + language: 'esql', + query: 'from auditbeat* metadata _id', + ...rewrites, + }; +}; + export const getMlRuleParams = ( rewrites?: Partial ): MachineLearningRuleParams => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts index d116f6fd71209..120ddd3981165 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts @@ -169,6 +169,7 @@ export const EsqlSpecificRuleParams = z.object({ type: z.literal('esql'), language: z.literal('esql'), query: RuleQuery, + alertSuppression: AlertSuppressionCamel.optional(), }); export type EsqlRuleParams = BaseRuleParams & EsqlSpecificRuleParams; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/create_esql_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/create_esql_alert_type.ts index 15ff7adb11013..10c82ad8fed7c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/create_esql_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/create_esql_alert_type.ts @@ -16,7 +16,7 @@ import type { CreateRuleOptions, SecurityAlertType } from '../types'; export const createEsqlAlertType = ( createOptions: CreateRuleOptions ): SecurityAlertType => { - const { version } = createOptions; + const { version, experimentalFeatures, licensing } = createOptions; return { id: ESQL_RULE_TYPE_ID, name: 'ES|QL Rule', @@ -44,6 +44,6 @@ export const createEsqlAlertType = ( isExportable: false, category: DEFAULT_APP_CATEGORIES.security.id, producer: SERVER_APP_ID, - executor: (params) => esqlExecutor({ ...params, version }), + executor: (params) => esqlExecutor({ ...params, experimentalFeatures, version, licensing }), }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts index 64ed0560f3609..3887f5db81a5a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts @@ -11,19 +11,24 @@ import type { AlertInstanceState, RuleExecutorServices, } from '@kbn/alerting-plugin/server'; +import type * as estypes from '@elastic/elasticsearch/lib/api/types'; import { computeIsESQLQueryAggregating, getIndexListFromEsqlQuery, } from '@kbn/securitysolution-utils'; +import type { LicensingPluginSetup } from '@kbn/licensing-plugin/server'; import { buildEsqlSearchRequest } from './build_esql_search_request'; import { performEsqlRequest } from './esql_request'; import { wrapEsqlAlerts } from './wrap_esql_alerts'; +import { wrapSuppressedEsqlAlerts } from './wrap_suppressed_esql_alerts'; +import { bulkCreateSuppressedAlertsInMemory } from '../utils/bulk_create_suppressed_alerts_in_memory'; import { createEnrichEventsFunction } from '../utils/enrichments'; import { rowToDocument } from './utils'; import { fetchSourceDocuments } from './fetch_source_documents'; +import { buildReasonMessageForEsqlAlert } from '../utils/reason_formatters'; -import type { RunOpts } from '../types'; +import type { RunOpts, SignalSource } from '../types'; import { addToSearchAfterReturn, @@ -31,16 +36,12 @@ import { makeFloatString, getUnprocessedExceptionsWarnings, getMaxSignalsWarning, + getSuppressionMaxSignalsWarning, } from '../utils/utils'; import type { EsqlRuleParams } from '../../rule_schema'; import { withSecuritySpan } from '../../../../utils/with_security_span'; - -/** - * ES|QL returns results as a single page. max size of 10,000 - * while we try increase size of the request to catch all events - * we don't want to overload ES/Kibana with large responses - */ -const ESQL_PAGE_SIZE_CIRCUIT_BREAKER = 1000; +import { getIsAlertSuppressionActive } from '../utils/get_is_alert_suppression_active'; +import type { ExperimentalFeatures } from '../../../../../common'; export const esqlExecutor = async ({ runOpts: { @@ -55,18 +56,29 @@ export const esqlExecutor = async ({ unprocessedExceptions, alertTimestampOverride, publicBaseUrl, + alertWithSuppression, }, services, state, spaceId, + experimentalFeatures, + licensing, }: { runOpts: RunOpts; services: RuleExecutorServices; state: object; spaceId: string; version: string; + experimentalFeatures: ExperimentalFeatures; + licensing: LicensingPluginSetup; }) => { const ruleParams = completeRule.ruleParams; + /** + * ES|QL returns results as a single page. max size of 10,000 + * while we try increase size of the request to catch all alerts that might been deduplicated + * we don't want to overload ES/Kibana with large responses + */ + const ESQL_PAGE_SIZE_CIRCUIT_BREAKER = tuple.maxSignals * 3; return withSecuritySpan('esqlExecutor', async () => { const result = createSearchAfterReturnType(); @@ -120,35 +132,100 @@ export const esqlExecutor = async ({ isRuleAggregating, }); - const wrappedAlerts = wrapEsqlAlerts({ - sourceDocuments, - isRuleAggregating, - results, - spaceId, - completeRule, - mergeStrategy, - alertTimestampOverride, - ruleExecutionLogger, - publicBaseUrl, - tuple, + const isAlertSuppressionActive = await getIsAlertSuppressionActive({ + alertSuppression: completeRule.ruleParams.alertSuppression, + licensing, + isFeatureDisabled: !experimentalFeatures?.alertSuppressionForEsqlRuleEnabled, }); - const enrichAlerts = createEnrichEventsFunction({ - services, - logger: ruleExecutionLogger, + const wrapHits = (events: Array>) => + wrapEsqlAlerts({ + events, + spaceId, + completeRule, + mergeStrategy, + isRuleAggregating, + alertTimestampOverride, + ruleExecutionLogger, + publicBaseUrl, + tuple, + }); + + const syntheticHits: Array> = results.map((document) => { + const { _id, _version, _index, ...source } = document; + + return { + _source: source as SignalSource, + fields: _id ? sourceDocuments[_id]?.fields : {}, + _id: _id ?? '', + _index: _index ?? '', + }; }); - const bulkCreateResult = await bulkCreate( - wrappedAlerts, - tuple.maxSignals - result.createdSignalsCount, - enrichAlerts - ); - addToSearchAfterReturn({ current: result, next: bulkCreateResult }); - ruleExecutionLogger.debug(`Created ${bulkCreateResult.createdItemsCount} alerts`); + if (isAlertSuppressionActive) { + const wrapSuppressedHits = (events: Array>) => + wrapSuppressedEsqlAlerts({ + events, + spaceId, + completeRule, + mergeStrategy, + isRuleAggregating, + alertTimestampOverride, + ruleExecutionLogger, + publicBaseUrl, + primaryTimestamp, + secondaryTimestamp, + tuple, + }); + + const bulkCreateResult = await bulkCreateSuppressedAlertsInMemory({ + enrichedEvents: syntheticHits, + toReturn: result, + wrapHits, + bulkCreate, + services, + ruleExecutionLogger, + tuple, + alertSuppression: completeRule.ruleParams.alertSuppression, + wrapSuppressedHits, + alertTimestampOverride, + alertWithSuppression, + experimentalFeatures, + buildReasonMessage: buildReasonMessageForEsqlAlert, + mergeSourceAndFields: true, + // passing 1 here since ES|QL does not support pagination + maxNumberOfAlertsMultiplier: 1, + }); + + addToSearchAfterReturn({ current: result, next: bulkCreateResult }); + ruleExecutionLogger.debug( + `Created ${bulkCreateResult.createdItemsCount} alerts. Suppressed ${bulkCreateResult.suppressedItemsCount} alerts` + ); + + if (bulkCreateResult.alertsWereTruncated) { + result.warningMessages.push(getSuppressionMaxSignalsWarning()); + break; + } + } else { + const wrappedAlerts = wrapHits(syntheticHits); + + const enrichAlerts = createEnrichEventsFunction({ + services, + logger: ruleExecutionLogger, + }); + const bulkCreateResult = await bulkCreate( + wrappedAlerts, + tuple.maxSignals - result.createdSignalsCount, + enrichAlerts + ); - if (bulkCreateResult.alertsWereTruncated) { - result.warningMessages.push(getMaxSignalsWarning()); - break; + addToSearchAfterReturn({ current: result, next: bulkCreateResult }); + ruleExecutionLogger.debug(`Created ${bulkCreateResult.createdItemsCount} alerts`); + + if (bulkCreateResult.alertsWereTruncated) { + result.warningMessages.push(getMaxSignalsWarning()); + break; + } } // no more results will be found diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/generate_alert_id.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/generate_alert_id.test.ts new file mode 100644 index 0000000000000..b3bc78e6b9478 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/generate_alert_id.test.ts @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { generateAlertId } from './generate_alert_id'; +import type * as estypes from '@elastic/elasticsearch/lib/api/types'; +import type { SignalSource } from '../../types'; +import type { CompleteRule, EsqlRuleParams } from '../../../rule_schema'; +import moment from 'moment'; +import { cloneDeep } from 'lodash'; + +const mockEvent: estypes.SearchHit = { + _id: 'test_id', + _version: 2, + _index: 'test_index', +}; + +const mockRule = { + alertId: 'test_alert_id', + ruleParams: { + query: 'from auditbeat*', + }, +} as CompleteRule; + +describe('generateAlertId', () => { + describe('aggregating query', () => { + const aggIdParams = { + event: mockEvent, + spaceId: 'default', + completeRule: mockRule, + tuple: { + to: moment('2010-10-20 04:43:12'), + from: moment('2010-10-20 04:10:12'), + maxSignals: 100, + }, + isRuleAggregating: true, + index: 10, + }; + + const id = generateAlertId(aggIdParams); + let modifiedIdParams: Parameters['0']; + + beforeEach(() => { + modifiedIdParams = cloneDeep(aggIdParams); + }); + + it('creates id dependant on time range tuple', () => { + modifiedIdParams.tuple.from = moment('2010-10-20 04:20:12'); + expect(id).not.toBe(generateAlertId(modifiedIdParams)); + }); + + it('creates id dependant on data row index', () => { + modifiedIdParams.index = 11; + expect(id).not.toBe(generateAlertId(modifiedIdParams)); + }); + + it('creates id dependant on spaceId', () => { + modifiedIdParams.spaceId = 'test-1'; + expect(id).not.toBe(generateAlertId(modifiedIdParams)); + }); + + it('creates id not dependant on event._id', () => { + modifiedIdParams.event._id = 'another-id'; + expect(id).toBe(generateAlertId(modifiedIdParams)); + }); + it('creates id not dependant on event._version', () => { + modifiedIdParams.event._version = 100; + expect(id).toBe(generateAlertId(modifiedIdParams)); + }); + it('creates id not dependant on event._index', () => { + modifiedIdParams.event._index = 'packetbeat-*'; + expect(id).toBe(generateAlertId(modifiedIdParams)); + }); + it('creates id dependant on rule alertId', () => { + modifiedIdParams.completeRule.alertId = 'another-alert-id'; + expect(id).not.toBe(generateAlertId(modifiedIdParams)); + }); + + it('creates id dependant on rule query', () => { + modifiedIdParams.completeRule.ruleParams.query = 'from packetbeat*'; + expect(id).not.toBe(generateAlertId(modifiedIdParams)); + }); + }); + + describe('non-aggregating query', () => { + const nonAggIdParams = { + event: mockEvent, + spaceId: 'default', + completeRule: mockRule, + tuple: { + to: moment('2010-10-20 04:43:12'), + from: moment('2010-10-20 04:10:12'), + maxSignals: 100, + }, + isRuleAggregating: false, + index: 10, + }; + + const id = generateAlertId(nonAggIdParams); + let modifiedIdParams: Parameters['0']; + + beforeEach(() => { + modifiedIdParams = cloneDeep(nonAggIdParams); + }); + + it('creates id not dependant on time range tuple', () => { + modifiedIdParams.tuple.from = moment('2010-10-20 04:20:12'); + expect(id).toBe(generateAlertId(modifiedIdParams)); + }); + + it('creates id not dependant on data row index', () => { + modifiedIdParams.index = 11; + expect(id).toBe(generateAlertId(modifiedIdParams)); + }); + + it('creates id dependant on spaceId', () => { + modifiedIdParams.spaceId = 'test-1'; + expect(id).not.toBe(generateAlertId(modifiedIdParams)); + }); + + it('creates id dependant on event._id', () => { + modifiedIdParams.event._id = 'another-id'; + expect(id).not.toBe(generateAlertId(modifiedIdParams)); + }); + it('creates id dependant on event._version', () => { + modifiedIdParams.event._version = 100; + expect(id).not.toBe(generateAlertId(modifiedIdParams)); + }); + it('creates id dependant on event._index', () => { + modifiedIdParams.event._index = 'packetbeat-*'; + expect(id).not.toBe(generateAlertId(modifiedIdParams)); + }); + it('creates id dependant on rule alertId', () => { + modifiedIdParams.completeRule.alertId = 'another-alert-id'; + expect(id).not.toBe(generateAlertId(modifiedIdParams)); + }); + + it('creates id not dependant on rule query', () => { + modifiedIdParams.completeRule.ruleParams.query = 'from packetbeat*'; + expect(id).toBe(generateAlertId(modifiedIdParams)); + }); + + it('creates id dependant on suppression terms', () => { + modifiedIdParams.suppressionTerms = [{ field: 'agent.name', value: ['test-1'] }]; + const id1 = generateAlertId(modifiedIdParams); + modifiedIdParams.suppressionTerms = [{ field: 'agent.name', value: ['test-2'] }]; + const id2 = generateAlertId(modifiedIdParams); + + expect(id).not.toBe(id1); + expect(id1).not.toBe(id2); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/generate_alert_id.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/generate_alert_id.ts new file mode 100644 index 0000000000000..0f549783f922e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/generate_alert_id.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import objectHash from 'object-hash'; +import type { Moment } from 'moment'; +import type * as estypes from '@elastic/elasticsearch/lib/api/types'; + +import type { CompleteRule, EsqlRuleParams } from '../../../rule_schema'; +import type { SignalSource } from '../../types'; +import type { SuppressionTerm } from '../../utils/suppression_utils'; +/** + * Generates id for ES|QL alert. + * Id is generated as hash of event properties and rule/space config identifiers. + * This would allow to deduplicate alerts, generated from the same event. + */ +export const generateAlertId = ({ + event, + spaceId, + completeRule, + tuple, + isRuleAggregating, + index, + suppressionTerms, +}: { + isRuleAggregating: boolean; + event: estypes.SearchHit; + spaceId: string | null | undefined; + completeRule: CompleteRule; + tuple: { + to: Moment; + from: Moment; + maxSignals: number; + }; + index: number; + suppressionTerms?: SuppressionTerm[]; +}) => { + const ruleRunId = tuple.from.toISOString() + tuple.to.toISOString(); + + return !isRuleAggregating && event._id + ? objectHash([ + event._id, + event._version, + event._index, + `${spaceId}:${completeRule.alertId}`, + ...(suppressionTerms ? [suppressionTerms] : []), + ]) + : objectHash([ + ruleRunId, + completeRule.ruleParams.query, + `${spaceId}:${completeRule.alertId}`, + index, + ]); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/index.ts index 061ddfda93174..4b2b842680e32 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/index.ts @@ -6,3 +6,4 @@ */ export * from './row_to_document'; +export * from './generate_alert_id'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.test.ts new file mode 100644 index 0000000000000..d54f91c088958 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.test.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { cloneDeep } from 'lodash'; +import moment from 'moment'; + +import { ALERT_UUID } from '@kbn/rule-data-utils'; +import { getCompleteRuleMock, getEsqlRuleParams } from '../../rule_schema/mocks'; +import { ruleExecutionLogMock } from '../../rule_monitoring/mocks'; +import { sampleDocNoSortIdWithTimestamp } from '../__mocks__/es_results'; +import { wrapEsqlAlerts } from './wrap_esql_alerts'; + +import * as esqlUtils from './utils/generate_alert_id'; + +const ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create(); + +const docId = 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71'; +const publicBaseUrl = 'http://somekibanabaseurl.com'; + +const alertSuppression = { + groupBy: ['source.ip'], +}; + +const completeRule = getCompleteRuleMock(getEsqlRuleParams()); +completeRule.ruleParams.alertSuppression = alertSuppression; + +describe('wrapSuppressedEsqlAlerts', () => { + test('should create an alert with the correct _id from a document', () => { + const doc = sampleDocNoSortIdWithTimestamp(docId); + const alerts = wrapEsqlAlerts({ + events: [doc], + isRuleAggregating: false, + spaceId: 'default', + mergeStrategy: 'missingFields', + completeRule, + alertTimestampOverride: undefined, + ruleExecutionLogger, + publicBaseUrl, + tuple: { + to: moment('2010-10-20 04:43:12'), + from: moment('2010-10-20 04:43:12'), + maxSignals: 100, + }, + }); + + expect(alerts[0]._id).toEqual('ed7fbf575371c898e0f0aea48cdf0bf1865939a9'); + expect(alerts[0]._source[ALERT_UUID]).toEqual('ed7fbf575371c898e0f0aea48cdf0bf1865939a9'); + }); + + test('should call generateAlertId for alert id', () => { + jest.spyOn(esqlUtils, 'generateAlertId').mockReturnValueOnce('mocked-alert-id'); + const completeRuleCloned = cloneDeep(completeRule); + completeRuleCloned.ruleParams.alertSuppression = { + groupBy: ['someKey'], + }; + const doc = sampleDocNoSortIdWithTimestamp(docId); + const alerts = wrapEsqlAlerts({ + events: [doc], + spaceId: 'default', + isRuleAggregating: true, + mergeStrategy: 'missingFields', + completeRule: completeRuleCloned, + alertTimestampOverride: undefined, + ruleExecutionLogger, + publicBaseUrl, + tuple: { + to: moment('2010-10-20 04:43:12'), + from: moment('2010-10-20 04:43:12'), + maxSignals: 100, + }, + }); + + expect(alerts[0]._id).toEqual('mocked-alert-id'); + expect(alerts[0]._source[ALERT_UUID]).toEqual('mocked-alert-id'); + + expect(esqlUtils.generateAlertId).toHaveBeenCalledWith( + expect.objectContaining({ + completeRule: expect.any(Object), + event: expect.any(Object), + index: 0, + isRuleAggregating: true, + spaceId: 'default', + tuple: { + from: expect.any(Object), + maxSignals: 100, + to: expect.any(Object), + }, + }) + ); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts index 6d3493e974668..b0fa2fd6638fa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts @@ -5,7 +5,6 @@ * 2.0. */ -import objectHash from 'object-hash'; import type { Moment } from 'moment'; import type * as estypes from '@elastic/elasticsearch/lib/api/types'; @@ -19,9 +18,10 @@ import { buildReasonMessageForNewTermsAlert } from '../utils/reason_formatters'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import { buildBulkBody } from '../factories/utils/build_bulk_body'; import type { SignalSource } from '../types'; +import { generateAlertId } from './utils'; export const wrapEsqlAlerts = ({ - results, + events, spaceId, completeRule, mergeStrategy, @@ -29,12 +29,10 @@ export const wrapEsqlAlerts = ({ ruleExecutionLogger, publicBaseUrl, tuple, - sourceDocuments, isRuleAggregating, }: { isRuleAggregating: boolean; - sourceDocuments: Record; - results: Array>; + events: Array>; spaceId: string | null | undefined; completeRule: CompleteRule; mergeStrategy: ConfigType['alertMergeStrategy']; @@ -47,37 +45,20 @@ export const wrapEsqlAlerts = ({ maxSignals: number; }; }): Array> => { - const wrapped = results.map>((document, i) => { - const ruleRunId = tuple.from.toISOString() + tuple.to.toISOString(); - - // for aggregating rules when metadata _id is present, generate alert based on ES document event id - const id = - !isRuleAggregating && document._id - ? objectHash([ - document._id, - document._version, - document._index, - `${spaceId}:${completeRule.alertId}`, - ]) - : objectHash([ - ruleRunId, - completeRule.ruleParams.query, - `${spaceId}:${completeRule.alertId}`, - i, - ]); - - // metadata fields need to be excluded from source, otherwise alerts creation fails - const { _id, _version, _index, ...source } = document; + const wrapped = events.map>((event, i) => { + const id = generateAlertId({ + event, + spaceId, + completeRule, + tuple, + isRuleAggregating, + index: i, + }); const baseAlert: BaseFieldsLatest = buildBulkBody( spaceId, completeRule, - { - _source: source as SignalSource, - fields: _id ? sourceDocuments[_id]?.fields : undefined, - _id: _id ?? '', - _index: _index ?? '', - }, + event, mergeStrategy, [], true, @@ -91,7 +72,7 @@ export const wrapEsqlAlerts = ({ return { _id: id, - _index: _index ?? '', + _index: event._index ?? '', _source: { ...baseAlert, }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.test.ts new file mode 100644 index 0000000000000..0c3c910efa056 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.test.ts @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { cloneDeep } from 'lodash'; +import moment from 'moment'; + +import { + ALERT_URL, + ALERT_UUID, + ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_INSTANCE_ID, + ALERT_SUPPRESSION_TERMS, + ALERT_SUPPRESSION_START, + ALERT_SUPPRESSION_END, +} from '@kbn/rule-data-utils'; +import { getCompleteRuleMock, getEsqlRuleParams } from '../../rule_schema/mocks'; +import { ruleExecutionLogMock } from '../../rule_monitoring/mocks'; +import { sampleDocNoSortIdWithTimestamp } from '../__mocks__/es_results'; +import { wrapSuppressedEsqlAlerts } from './wrap_suppressed_esql_alerts'; + +import * as esqlUtils from './utils/generate_alert_id'; + +const ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create(); + +const docId = 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71'; +const publicBaseUrl = 'http://somekibanabaseurl.com'; + +const alertSuppression = { + groupBy: ['source.ip'], +}; + +const completeRule = getCompleteRuleMock(getEsqlRuleParams()); +completeRule.ruleParams.alertSuppression = alertSuppression; + +describe('wrapSuppressedEsqlAlerts', () => { + test('should create an alert with the correct _id from a document and suppression fields', () => { + const doc = sampleDocNoSortIdWithTimestamp(docId); + const alerts = wrapSuppressedEsqlAlerts({ + events: [doc], + isRuleAggregating: false, + spaceId: 'default', + mergeStrategy: 'missingFields', + completeRule, + alertTimestampOverride: undefined, + ruleExecutionLogger, + publicBaseUrl, + primaryTimestamp: '@timestamp', + tuple: { + to: moment('2010-10-20 04:43:12'), + from: moment('2010-10-20 04:43:12'), + maxSignals: 100, + }, + }); + + expect(alerts[0]._id).toEqual('d94fb11e6062d7dce881ea07d952a1280398663a'); + expect(alerts[0]._source[ALERT_UUID]).toEqual('d94fb11e6062d7dce881ea07d952a1280398663a'); + expect(alerts[0]._source[ALERT_URL]).toContain( + 'http://somekibanabaseurl.com/app/security/alerts/redirect/d94fb11e6062d7dce881ea07d952a1280398663a?index=.alerts-security.alerts-default' + ); + expect(alerts[0]._source[ALERT_SUPPRESSION_DOCS_COUNT]).toEqual(0); + expect(alerts[0]._source[ALERT_INSTANCE_ID]).toEqual( + '1bf77f90e72d76d9335ad0ce356340a3d9833f96' + ); + expect(alerts[0]._source[ALERT_SUPPRESSION_TERMS]).toEqual([ + { field: 'source.ip', value: ['127.0.0.1'] }, + ]); + expect(alerts[0]._source[ALERT_SUPPRESSION_START]).toBeDefined(); + expect(alerts[0]._source[ALERT_SUPPRESSION_END]).toBeDefined(); + }); + + test('should create an alert with a different _id if suppression field is different', () => { + const completeRuleCloned = cloneDeep(completeRule); + completeRuleCloned.ruleParams.alertSuppression = { + groupBy: ['someKey'], + }; + const doc = sampleDocNoSortIdWithTimestamp(docId); + const alerts = wrapSuppressedEsqlAlerts({ + events: [doc], + spaceId: 'default', + isRuleAggregating: true, + mergeStrategy: 'missingFields', + completeRule: completeRuleCloned, + alertTimestampOverride: undefined, + ruleExecutionLogger, + publicBaseUrl, + primaryTimestamp: '@timestamp', + tuple: { + to: moment('2010-10-20 04:43:12'), + from: moment('2010-10-20 04:43:12'), + maxSignals: 100, + }, + }); + + expect(alerts[0]._source[ALERT_URL]).toContain( + 'http://somekibanabaseurl.com/app/security/alerts/redirect/' + ); + expect(alerts[0]._source[ALERT_SUPPRESSION_DOCS_COUNT]).toEqual(0); + expect(alerts[0]._source[ALERT_INSTANCE_ID]).toEqual( + 'c88edd552cb3501f040aea63ec68312e71af2ed2' + ); + expect(alerts[0]._source[ALERT_SUPPRESSION_TERMS]).toEqual([ + { field: 'someKey', value: 'someValue' }, + ]); + }); + + test('should call generateAlertId for alert id', () => { + jest.spyOn(esqlUtils, 'generateAlertId').mockReturnValueOnce('mocked-alert-id'); + const completeRuleCloned = cloneDeep(completeRule); + completeRuleCloned.ruleParams.alertSuppression = { + groupBy: ['someKey'], + }; + const doc = sampleDocNoSortIdWithTimestamp(docId); + const alerts = wrapSuppressedEsqlAlerts({ + events: [doc], + spaceId: 'default', + isRuleAggregating: false, + mergeStrategy: 'missingFields', + completeRule: completeRuleCloned, + alertTimestampOverride: undefined, + ruleExecutionLogger, + publicBaseUrl, + primaryTimestamp: '@timestamp', + tuple: { + to: moment('2010-10-20 04:43:12'), + from: moment('2010-10-20 04:43:12'), + maxSignals: 100, + }, + }); + + expect(alerts[0]._id).toEqual('mocked-alert-id'); + expect(alerts[0]._source[ALERT_UUID]).toEqual('mocked-alert-id'); + + expect(esqlUtils.generateAlertId).toHaveBeenCalledWith( + expect.objectContaining({ + completeRule: expect.any(Object), + event: expect.any(Object), + index: 0, + isRuleAggregating: false, + spaceId: 'default', + tuple: { + from: expect.any(Object), + maxSignals: 100, + to: expect.any(Object), + }, + }) + ); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts new file mode 100644 index 0000000000000..057cd5c906167 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import objectHash from 'object-hash'; +import type { Moment } from 'moment'; +import type * as estypes from '@elastic/elasticsearch/lib/api/types'; +import { TIMESTAMP } from '@kbn/rule-data-utils'; +import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; + +import type { + BaseFieldsLatest, + WrappedFieldsLatest, +} from '../../../../../common/api/detection_engine/model/alerts'; +import type { ConfigType } from '../../../../config'; +import type { CompleteRule, EsqlRuleParams } from '../../rule_schema'; +import { buildReasonMessageForNewTermsAlert } from '../utils/reason_formatters'; +import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; +import { buildBulkBody } from '../factories/utils/build_bulk_body'; +import type { SignalSource } from '../types'; +import { getSuppressionAlertFields, getSuppressionTerms } from '../utils'; +import { generateAlertId } from './utils'; + +export const wrapSuppressedEsqlAlerts = ({ + events, + spaceId, + completeRule, + mergeStrategy, + alertTimestampOverride, + ruleExecutionLogger, + publicBaseUrl, + tuple, + isRuleAggregating, + primaryTimestamp, + secondaryTimestamp, +}: { + isRuleAggregating: boolean; + events: Array>; + spaceId: string | null | undefined; + completeRule: CompleteRule; + mergeStrategy: ConfigType['alertMergeStrategy']; + alertTimestampOverride: Date | undefined; + ruleExecutionLogger: IRuleExecutionLogForExecutors; + publicBaseUrl: string | undefined; + tuple: { + to: Moment; + from: Moment; + maxSignals: number; + }; + primaryTimestamp: string; + secondaryTimestamp?: string; +}): Array> => { + const wrapped = events.map>( + (event, i) => { + const combinedFields = { ...event?.fields, ...event._source }; + + const suppressionTerms = getSuppressionTerms({ + alertSuppression: completeRule?.ruleParams?.alertSuppression, + fields: combinedFields, + }); + + const id = generateAlertId({ + event, + spaceId, + completeRule, + tuple, + isRuleAggregating, + index: i, + suppressionTerms, + }); + + const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); + + const baseAlert: BaseFieldsLatest = buildBulkBody( + spaceId, + completeRule, + event, + mergeStrategy, + [], + true, + buildReasonMessageForNewTermsAlert, + [], + alertTimestampOverride, + ruleExecutionLogger, + id, + publicBaseUrl + ); + + return { + _id: id, + _index: event._index ?? '', + _source: { + ...baseAlert, + ...getSuppressionAlertFields({ + primaryTimestamp, + secondaryTimestamp, + fields: combinedFields, + suppressionTerms, + fallbackTimestamp: baseAlert[TIMESTAMP], + instanceId, + }), + }, + }; + } + ); + + return wrapped; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/bulk_create_suppressed_alerts_in_memory.ts index 2fffa07f8d684..efa8e95c522a5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/bulk_create_suppressed_alerts_in_memory.ts @@ -87,7 +87,7 @@ export const bulkCreateSuppressedNewTermsAlertsInMemory = async ({ const partitionedEvents = partitionMissingFieldsEvents( eventsAndTerms, alertSuppression?.groupBy || [], - ['event'] + ['event', 'fields'] ); unsuppressibleWrappedDocs = wrapHits(partitionedEvents[1]); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index 85bfc76e964e2..030cb213d94dd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -51,6 +51,8 @@ export interface BulkCreateSuppressedAlertsParams enrichedEvents: SignalSourceHit[]; toReturn: SearchAfterAndBulkCreateReturnType; experimentalFeatures: ExperimentalFeatures; + mergeSourceAndFields?: boolean; + maxNumberOfAlertsMultiplier?: number; } /** * wraps, bulk create and suppress alerts in memory, also takes care of missing fields logic. @@ -70,6 +72,8 @@ export const bulkCreateSuppressedAlertsInMemory = async ({ alertWithSuppression, alertTimestampOverride, experimentalFeatures, + mergeSourceAndFields = false, + maxNumberOfAlertsMultiplier, }: BulkCreateSuppressedAlertsParams) => { const suppressOnMissingFields = (alertSuppression?.missingFieldsStrategy ?? DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY) === @@ -81,7 +85,9 @@ export const bulkCreateSuppressedAlertsInMemory = async ({ if (!suppressOnMissingFields) { const partitionedEvents = partitionMissingFieldsEvents( enrichedEvents, - alertSuppression?.groupBy || [] + alertSuppression?.groupBy || [], + ['fields'], + mergeSourceAndFields ); unsuppressibleWrappedDocs = wrapHits(partitionedEvents[1], buildReasonMessage); @@ -102,6 +108,7 @@ export const bulkCreateSuppressedAlertsInMemory = async ({ alertWithSuppression, alertTimestampOverride, experimentalFeatures, + maxNumberOfAlertsMultiplier, }); }; @@ -120,6 +127,7 @@ export interface ExecuteBulkCreateAlertsParams>; toReturn: SearchAfterAndBulkCreateReturnType; experimentalFeatures: ExperimentalFeatures; + maxNumberOfAlertsMultiplier?: number; } /** @@ -139,11 +147,12 @@ export const executeBulkCreateAlerts = async < alertWithSuppression, alertTimestampOverride, experimentalFeatures, + maxNumberOfAlertsMultiplier = MAX_SIGNALS_SUPPRESSION_MULTIPLIER, }: ExecuteBulkCreateAlertsParams) => { // max signals for suppression includes suppressed and created alerts // this allows to lift max signals limitation to higher value // and can detects events beyond default max_signals value - const suppressionMaxSignals = MAX_SIGNALS_SUPPRESSION_MULTIPLIER * tuple.maxSignals; + const suppressionMaxSignals = maxNumberOfAlertsMultiplier * tuple.maxSignals; const suppressionDuration = alertSuppression?.duration; const suppressionWindow = suppressionDuration diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.test.ts index dfee32a058ba6..7fad1d4f2b10c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.test.ts @@ -30,7 +30,8 @@ describe('partitionMissingFieldsEvents', () => { _index: 'index-0', }, ], - ['agent.host', 'agent.type', 'agent.version'] + ['agent.host', 'agent.type', 'agent.version'], + ['fields'] ) ).toEqual([ [ @@ -83,7 +84,7 @@ describe('partitionMissingFieldsEvents', () => { }, ], ['agent.host', 'agent.type', 'agent.version'], - ['event'] + ['event', 'fields'] ) ).toEqual([ [ @@ -113,6 +114,35 @@ describe('partitionMissingFieldsEvents', () => { ], ]); }); + it('should partition when fields located in root of event', () => { + expect( + partitionMissingFieldsEvents( + [ + { + 'agent.host': 'host-1', + 'agent.version': 2, + }, + { + 'agent.host': 'host-1', + }, + ], + ['agent.host', 'agent.version'], + [] + ) + ).toEqual([ + [ + { + 'agent.host': 'host-1', + 'agent.version': 2, + }, + ], + [ + { + 'agent.host': 'host-1', + }, + ], + ]); + }); it('should partition if two fields are empty', () => { expect( partitionMissingFieldsEvents( @@ -125,7 +155,8 @@ describe('partitionMissingFieldsEvents', () => { _index: 'index-0', }, ], - ['agent.host', 'agent.type', 'agent.version'] + ['agent.host', 'agent.type', 'agent.version'], + ['fields'] ) ).toEqual([ [], @@ -152,7 +183,8 @@ describe('partitionMissingFieldsEvents', () => { _index: 'index-0', }, ], - ['agent.host', 'agent.type', 'agent.version'] + ['agent.host', 'agent.type', 'agent.version'], + ['fields'] ) ).toEqual([ [], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts index f86a969dc60c3..901768fe5c773 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts @@ -17,20 +17,25 @@ import type { SignalSourceHit } from '../types'; * 2. where any of fields is empty */ export const partitionMissingFieldsEvents = < - T extends SignalSourceHit | { event: SignalSourceHit } + T extends SignalSourceHit | { event: SignalSourceHit } | Record >( events: T[], suppressedBy: string[] = [], // path to fields property within event object. At this point, it can be in root of event object or within event key - fieldsPath: ['event'] | [] = [] + fieldsPath: ['event', 'fields'] | ['fields'] | [] = [], + mergeSourceAndFields: boolean = false ): T[][] => { return partition(events, (event) => { if (suppressedBy.length === 0) { return true; } - const eventFields = get(event, [...fieldsPath, 'fields']); - const hasMissingFields = - Object.keys(pick(eventFields, suppressedBy)).length < suppressedBy.length; + const eventFields = fieldsPath.length ? get(event, fieldsPath) : event; + const sourceFields = + (event as SignalSourceHit)?._source || (event as { event: SignalSourceHit })?.event?._source; + + const fields = mergeSourceAndFields ? { ...sourceFields, ...eventFields } : eventFields; + + const hasMissingFields = Object.keys(pick(fields, suppressedBy)).length < suppressedBy.length; return !hasMissingFields; }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts index 167f22dc1d52b..44febba73e68e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts @@ -17,7 +17,8 @@ import { ALERT_SUPPRESSION_END, } from '@kbn/rule-data-utils'; import type { AlertSuppressionCamel } from '../../../../../common/api/detection_engine/model/rule_schema'; -interface SuppressionTerm { + +export interface SuppressionTerm { field: string; value: string[] | number[] | null; } @@ -33,7 +34,7 @@ export const getSuppressionAlertFields = ({ fallbackTimestamp, instanceId, }: { - fields: Record | undefined; + fields: Record | undefined; primaryTimestamp: string; secondaryTimestamp?: string; suppressionTerms: SuppressionTerm[]; diff --git a/x-pack/test/security_solution_api_integration/config/ess/config.base.ts b/x-pack/test/security_solution_api_integration/config/ess/config.base.ts index cc47b97377e6c..d134546c6f633 100644 --- a/x-pack/test/security_solution_api_integration/config/ess/config.base.ts +++ b/x-pack/test/security_solution_api_integration/config/ess/config.base.ts @@ -80,6 +80,7 @@ export function createTestConfig(options: CreateTestConfigOptions, testFiles?: s '--xpack.ruleRegistry.unsafe.legacyMultiTenancy.enabled=true', `--xpack.securitySolution.enableExperimental=${JSON.stringify([ 'previewTelemetryUrlEnabled', + 'alertSuppressionForEsqlRuleEnabled', 'riskScoringPersistence', 'riskScoringRoutesEnabled', 'bulkCustomHighlightedFieldsEnabled', diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts index f6ba7fa49895e..76c73ff71cc18 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts @@ -19,6 +19,7 @@ export default createTestConfig({ ])}`, // See tests within the file "ignore_fields.ts" which use these values in "alertIgnoreFields" `--xpack.securitySolution.enableExperimental=${JSON.stringify([ 'bulkCustomHighlightedFieldsEnabled', + 'alertSuppressionForEsqlRuleEnabled', ])}`, ], }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql_suppression.ts new file mode 100644 index 0000000000000..f7264e064bdba --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql_suppression.ts @@ -0,0 +1,2025 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import sortBy from 'lodash/sortBy'; +import expect from 'expect'; +import { v4 as uuidv4 } from 'uuid'; + +import { + ALERT_SUPPRESSION_START, + ALERT_SUPPRESSION_END, + ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_SUPPRESSION_TERMS, + ALERT_LAST_DETECTED, + TIMESTAMP, + ALERT_START, +} from '@kbn/rule-data-utils'; +import { EsqlRuleCreateProps } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema'; +import { getCreateEsqlRulesSchemaMock } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema/mocks'; +import { RuleExecutionStatusEnum } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_monitoring'; +import { ALERT_ORIGINAL_TIME } from '@kbn/security-solution-plugin/common/field_maps/field_names'; +import { DETECTION_ENGINE_SIGNALS_STATUS_URL as DETECTION_ENGINE_ALERTS_STATUS_URL } from '@kbn/security-solution-plugin/common/constants'; +import { getSuppressionMaxSignalsWarning as getSuppressionMaxAlertsWarning } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/utils/utils'; + +import { ENABLE_ASSET_CRITICALITY_SETTING } from '@kbn/security-solution-plugin/common/constants'; +import { + getPreviewAlerts, + previewRule, + getOpenAlerts, + dataGeneratorFactory, + previewRuleWithExceptionEntries, + setAlertStatus, + patchRule, +} from '../../../../utils'; +import { + deleteAllRules, + deleteAllAlerts, + createRule, +} from '../../../../../../../common/utils/security_solution'; +import { deleteAllExceptions } from '../../../../../lists_and_exception_lists/utils'; +import { FtrProviderContext } from '../../../../../../ftr_provider_context'; + +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const es = getService('es'); + const log = getService('log'); + const kibanaServer = getService('kibanaServer'); + const { indexEnhancedDocuments, indexListOfDocuments, indexGeneratedDocuments } = + dataGeneratorFactory({ + es, + index: 'ecs_compliant', + log, + }); + + /** + * to separate docs between rules runs + */ + const internalIdPipe = (id: string) => `| where id=="${id}"`; + + const getNonAggRuleQueryWithMetadata = (id: string) => + `from ecs_compliant metadata _id, _index, _version ${internalIdPipe(id)}`; + + // skipped in MKI as it depends on feature flag alertSuppressionForEsqlRuleEnabled + describe('@ess @serverless @skipInServerlessMKI ES|QL rule type, alert suppression', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/security_solution/ecs_compliant'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/security_solution/ecs_compliant'); + await deleteAllAlerts(supertest, log, es); + await deleteAllRules(supertest, log); + }); + + it('should suppress an alert during real rule executions', async () => { + const id = uuidv4(); + const firstTimestamp = new Date().toISOString(); + + const firstExecutionDocuments = [ + { + host: { name: 'host-0' }, + id, + '@timestamp': firstTimestamp, + }, + { + host: { name: 'host-0' }, + id, + '@timestamp': firstTimestamp, + }, + { + host: { name: 'host-0' }, + id, + '@timestamp': firstTimestamp, + }, + ]; + + await indexListOfDocuments(firstExecutionDocuments); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 300, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + }; + + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + + expect(alerts.hits.hits.length).toBe(1); + expect(alerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-0', + }, + ], + // suppression boundaries equal to original event time, since no alert been suppressed + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }) + ); + + const secondTimestamp = new Date().toISOString(); + const secondExecutionDocuments = [ + { + host: { name: 'host-0', ip: '127.0.0.5' }, + id, + '@timestamp': secondTimestamp, + }, + ]; + // Add a new document, then disable and re-enable to trigger another rule run. The second doc should + // trigger an update to the existing alert without changing the timestamp + await indexListOfDocuments(secondExecutionDocuments); + + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const afterTimestamp = new Date(); + const secondAlerts = await getOpenAlerts( + supertest, + log, + es, + createdRule, + RuleExecutionStatusEnum.succeeded, + undefined, + afterTimestamp + ); + expect(secondAlerts.hits.hits.length).toEqual(1); + expect(secondAlerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-0', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, // timestamp is the same + [ALERT_SUPPRESSION_START]: firstTimestamp, // suppression start is the same + [ALERT_SUPPRESSION_END]: secondTimestamp, // suppression end is updated + [ALERT_SUPPRESSION_DOCS_COUNT]: 3, + }) + ); + }); + + it('should NOT suppress and update an alert if the alert is closed', async () => { + const id = uuidv4(); + const firstTimestamp = new Date().toISOString(); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 300, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + }; + + const firstExecutionDocuments = [ + { + host: { name: 'host-0' }, + id, + '@timestamp': firstTimestamp, + }, + ]; + + await indexListOfDocuments(firstExecutionDocuments); + + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + expect(alerts.hits.hits).toHaveLength(1); + // Close the alert. Subsequent rule executions should ignore this closed alert + // for suppression purposes. + const alertIds = alerts.hits.hits.map((alert) => alert._id); + await supertest + .post(DETECTION_ENGINE_ALERTS_STATUS_URL) + .set('kbn-xsrf', 'true') + .send(setAlertStatus({ alertIds, status: 'closed' })) + .expect(200); + + const secondTimestamp = new Date().toISOString(); + const secondExecutionDocuments = [ + { + host: { name: 'host-0' }, + id, + '@timestamp': secondTimestamp, + }, + ]; + // Add new documents, then disable and re-enable to trigger another rule run. The second doc should + // trigger a new alert since the first one is now closed. + await indexListOfDocuments(secondExecutionDocuments); + + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const afterTimestamp = new Date(); + const secondAlerts = await getOpenAlerts( + supertest, + log, + es, + createdRule, + RuleExecutionStatusEnum.succeeded, + undefined, + afterTimestamp + ); + expect(secondAlerts.hits.hits.length).toEqual(1); + expect(alerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-0', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + }); + + it('should NOT suppress alerts when suppression period is less than rule interval', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:15:00.000Z'; + + const firstExecutionDocuments = [ + { + host: { name: 'host-0' }, + id, + '@timestamp': firstTimestamp, + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'host-0' }, + id, + '@timestamp': secondTimestamp, + }, + ]; + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 10, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toBe(2); + expect(previewAlerts[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-0', + }, + ], + [TIMESTAMP]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + + expect(previewAlerts[1]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-0', + }, + ], + [TIMESTAMP]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_START]: secondTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + }); + + it('should suppress alerts in the time window that covers 3 rule executions', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:15:00.000Z'; + const thirdTimestamp = '2020-10-28T06:45:00.000Z'; + + const firstExecutionDocuments = [ + { + host: { name: 'host-0' }, + id, + '@timestamp': firstTimestamp, + }, + { + host: { name: 'host-0' }, + id, + '@timestamp': firstTimestamp, + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'host-0' }, + id, + '@timestamp': secondTimestamp, + }, + ]; + const thirdExecutionDocuments = [ + { + host: { name: 'host-0' }, + id, + '@timestamp': thirdTimestamp, + }, + ]; + + await indexListOfDocuments([ + ...firstExecutionDocuments, + ...secondExecutionDocuments, + ...thirdExecutionDocuments, + ]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 2, + unit: 'h', + }, + missing_fields_strategy: 'suppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 3, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-0', + }, + ], + [TIMESTAMP]: '2020-10-28T06:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', // Note: ALERT_LAST_DETECTED gets updated, timestamp does not + [ALERT_START]: '2020-10-28T06:00:00.000Z', + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: thirdTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 3, // in total 3 alert got suppressed: 1 from the first run, 1 from the second, 1 from the third + }); + }); + + it('should suppress the correct alerts based on multi values group_by', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:15:00.000Z'; + + const firstExecutionDocuments = [ + { + host: { name: 'host-a' }, + id, + '@timestamp': firstTimestamp, + 'agent.version': 1, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': firstTimestamp, + 'agent.version': 2, + }, + { + host: { name: 'host-b' }, + id, + '@timestamp': firstTimestamp, + 'agent.version': 2, + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + 'agent.version': 1, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: `from ecs_compliant metadata _id, _index, _version ${internalIdPipe( + id + )} | where host.name=="host-a"`, + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name', 'agent.version'], + duration: { + value: 60, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', 'agent.version', ALERT_ORIGINAL_TIME], + }); + // 3 alerts should be generated: + // 1. for pair 'host-a', 1 - suppressed + // 2. for pair 'host-a', 2 - not suppressed + expect(previewAlerts.length).toEqual(2); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + { + field: 'agent.version', + value: '1', + }, + ], + [TIMESTAMP]: '2020-10-28T06:00:00.000Z', + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + expect(previewAlerts[1]._source).toEqual({ + ...previewAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + { + field: 'agent.version', + value: '2', + }, + ], + [TIMESTAMP]: '2020-10-28T06:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T06:00:00.000Z', + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, // no suppressed alerts + }); + }); + + it('should correctly suppress when using a timestamp override', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + const docWithoutOverride = { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }; + const docWithOverride = { + ...docWithoutOverride, + host: { name: 'host-a' }, + // This simulates a very late arriving doc + '@timestamp': '2020-10-28T03:00:00.000Z', + event: { + ingested: secondTimestamp, + }, + }; + + await indexListOfDocuments([docWithoutOverride, docWithOverride]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 60, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + timestamp_override: 'event.ingested', + }; + + // 1 alert should be suppressed, based on event.ingested value of a document + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('should deduplicate multiple alerts while suppressing new ones', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + const firstExecutionDocuments = [ + { + host: { name: 'host-a' }, + id, + '@timestamp': firstTimestamp, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': firstTimestamp, + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 60, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + // large look-back time covers all docs + from: 'now-1h', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 4, + }); + }); + + it('should suppress alerts with missing fields', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + const firstExecutionDocuments = [ + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a', ip: '127.0.0.3' }, + }, + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a', ip: '127.0.0.4' }, + }, + { + id, + '@timestamp': firstTimestamp, + host: { ip: '127.0.0.5' }, // doc 1 with missing host.name field + }, + { + id, + '@timestamp': firstTimestamp, + host: { ip: '127.0.0.6' }, // doc 2 with missing host.name field + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'host-a', ip: '127.0.0.10' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { ip: '127.0.0.11' }, // doc 3 with missing host.name field + id, + '@timestamp': secondTimestamp, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 60, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(2); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + + expect(previewAlerts[1]._source).toEqual({ + ...previewAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: null, + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + }); + + it('should not suppress alerts with missing fields if configured so', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + const firstExecutionDocuments = [ + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a', ip: '127.0.0.3' }, + }, + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a', ip: '127.0.0.4' }, + }, + { + id, + '@timestamp': firstTimestamp, + host: { ip: '127.0.0.5' }, // doc 1 with missing host.name field + }, + { + id, + '@timestamp': firstTimestamp, + host: { ip: '127.0.0.6' }, // doc 2 with missing host.name field + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'host-a', ip: '127.0.0.10' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { ip: '127.0.0.11' }, // doc 3 with missing host.name field + id, + '@timestamp': secondTimestamp, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 60, + unit: 'm', + }, + missing_fields_strategy: 'doNotSuppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(4); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + + // rest of alerts are not suppressed and do not have suppress properties + previewAlerts.slice(1).forEach((previewAlert) => { + const source = previewAlert._source; + expect(source).toHaveProperty('id', id); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_END); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_TERMS); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + }); + }); + + it('should suppress alerts for aggregating queries', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + const firstExecutionDocuments = [ + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }, + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: `from ecs_compliant ${internalIdPipe( + id + )} | stats counted_agents=count(host.name) by host.name`, + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 60, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', // since aggregation query results do not have timestamp properties suppression boundary start set as a first execution time + [ALERT_SUPPRESSION_END]: '2020-10-28T06:30:00.000Z', // since aggregation query results do not have timestamp properties suppression boundary end set as a second execution time + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, // only one suppressed alert, since aggregation query produces one alert per rule execution, no matter how many events aggregated + }); + expect(previewAlerts[0]._source).not.toHaveProperty(ALERT_ORIGINAL_TIME); + }); + + it('should suppress alerts by custom field, created in ES|QL query', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + const firstExecutionDocuments = [ + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }, + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-b' }, + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'test-c' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { name: 'host-d' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { name: 'test-s' }, + id, + '@timestamp': secondTimestamp, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + // ES|QL query creates new field custom_field - prefix_host, prefix_test + query: `from ecs_compliant metadata _id ${internalIdPipe( + id + )} | eval custom_field=concat("prefix_", left(host.name, 4))`, + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['custom_field'], + duration: { + value: 60, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + }); + // lodash sortBy is used here because custom_field is non ECS and not mapped in alerts index, so can't be sorted by + const sortedAlerts = sortBy(previewAlerts, 'custom_field'); + expect(previewAlerts.length).toEqual(2); + + expect(sortedAlerts[0]._source).toEqual({ + ...sortedAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'custom_field', + value: 'prefix_host', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + + expect(sortedAlerts[1]._source).toEqual({ + ...sortedAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'custom_field', + value: 'prefix_test', + }, + ], + [ALERT_ORIGINAL_TIME]: secondTimestamp, + [ALERT_SUPPRESSION_START]: secondTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('should suppress alerts by custom field, created in ES|QL query, when do not suppress missing fields configured', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + const firstExecutionDocuments = [ + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }, + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-b' }, + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'test-c' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { name: 'host-d' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { name: 'test-s' }, + id, + '@timestamp': secondTimestamp, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + // ES|QL query creates new field custom_field - prefix_host, prefix_test + query: `from ecs_compliant metadata _id ${internalIdPipe( + id + )} | eval custom_field=concat("prefix_", left(host.name, 4))`, + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['custom_field'], + duration: { + value: 60, + unit: 'm', + }, + missing_fields_strategy: 'doNotSuppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + }); + // lodash sortBy is used here because custom_field is non ECS and not mapped in alerts index, so can't be sorted by + const sortedAlerts = sortBy(previewAlerts, 'custom_field'); + expect(previewAlerts.length).toEqual(2); + + expect(sortedAlerts[0]._source).toEqual({ + ...sortedAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'custom_field', + value: 'prefix_host', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + + expect(sortedAlerts[1]._source).toEqual({ + ...sortedAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'custom_field', + value: 'prefix_test', + }, + ], + [ALERT_ORIGINAL_TIME]: secondTimestamp, + [ALERT_SUPPRESSION_START]: secondTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('should suppress by field, dropped in ES|QL query, but returned from source index', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + const firstExecutionDocuments = [ + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: `from ecs_compliant metadata _id ${internalIdPipe(id)} | drop host.name`, + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 60, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + // even when field is dropped in ES|QL query it is returned from source document + it('should not suppress alerts by field, dropped in ES|QL query, when do not suppress missing fields configured', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + const firstExecutionDocuments = [ + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }, + { + id, + '@timestamp': firstTimestamp, + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + { + id, + '@timestamp': firstTimestamp, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: `from ecs_compliant metadata _id ${internalIdPipe(id)} | drop host.name`, + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 60, + unit: 'm', + }, + missing_fields_strategy: 'doNotSuppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(3); + + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + // rest of alerts are not suppressed and do not have suppress properties + previewAlerts.slice(1).forEach((previewAlert) => { + const source = previewAlert._source; + expect(source).toHaveProperty('id', id); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_END); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_TERMS); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + }); + }); + + describe('rule execution only', () => { + it('should suppress alerts during rule execution only', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + const laterTimestamp = '2020-10-28T06:50:00.000Z'; + + const firstExecutionDocuments = [ + { + host: { name: 'host-a' }, + id, + '@timestamp': timestamp, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': timestamp, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': laterTimestamp, + }, + // does not generate alert + { + host: { name: 'host-b' }, + id, + '@timestamp': laterTimestamp, + }, + ]; + + await indexListOfDocuments(firstExecutionDocuments); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: `from ecs_compliant metadata _id, _index, _version ${internalIdPipe( + id + )} | where host.name=="host-a"`, + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: laterTimestamp, // suppression ends with later timestamp + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + }); + + it('should suppress alerts per rule execution for array field', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + + const firstExecutionDocuments = [ + { + host: { name: ['host-a', 'host-b'] }, + id, + '@timestamp': timestamp, + }, + { + host: { name: ['host-a', 'host-b'] }, + id, + '@timestamp': timestamp, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a', 'host-b'], + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: timestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('should suppress alerts with missing fields during rule execution only for multiple suppress by fields', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + + const firstExecutionDocuments = [ + // no missing fields + { + host: { name: 'host-a', ip: '127.0.0.11' }, + agent: { name: 'agent-a', version: 10 }, + id, + '@timestamp': timestamp, + }, + { + host: { name: 'host-a', ip: '127.0.0.12' }, + agent: { name: 'agent-a', version: 10 }, + id, + '@timestamp': timestamp, + }, + // missing agent.name + { + host: { name: 'host-a', ip: '127.0.0.21' }, + agent: { version: 10 }, + id, + '@timestamp': timestamp, + }, + { + host: { name: 'host-a', ip: '127.0.0.22' }, + agent: { version: 10 }, + id, + '@timestamp': timestamp, + }, + // missing agent.version + { + host: { name: 'host-a', ip: '127.0.0.31' }, + agent: { name: 'agent-a' }, + id, + '@timestamp': timestamp, + }, + { + host: { name: 'host-a', ip: '127.0.0.32' }, + agent: { name: 'agent-a' }, + id, + '@timestamp': timestamp, + }, + // missing both agent.* + { + host: { name: 'host-a', ip: '127.0.0.41' }, + id, + '@timestamp': timestamp, + }, + { + host: { name: 'host-a', ip: '127.0.0.42' }, + id, + '@timestamp': timestamp, + }, + ]; + + await indexListOfDocuments(firstExecutionDocuments); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + alert_suppression: { + group_by: ['agent.name', 'agent.version'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['agent.name', 'agent.version', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(4); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-a', + }, + { + field: 'agent.version', + value: '10', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + expect(previewAlerts[1]._source).toEqual({ + ...previewAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-a', + }, + { + field: 'agent.version', + value: null, + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + expect(previewAlerts[2]._source).toEqual({ + ...previewAlerts[2]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: null, + }, + { + field: 'agent.version', + value: '10', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + expect(previewAlerts[3]._source).toEqual({ + ...previewAlerts[3]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: null, + }, + { + field: 'agent.version', + value: null, + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('should not suppress alerts with missing fields during rule execution only if configured so for multiple suppress by fields', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + + const firstExecutionDocuments = [ + // no missing fields + { + host: { name: 'host-a', ip: '127.0.0.11' }, + agent: { name: 'agent-a', version: 10 }, + id, + '@timestamp': timestamp, + }, + { + host: { name: 'host-a', ip: '127.0.0.12' }, + agent: { name: 'agent-a', version: 10 }, + id, + '@timestamp': timestamp, + }, + // missing agent.name + { + host: { name: 'host-a', ip: '127.0.0.21' }, + agent: { version: 10 }, + id, + '@timestamp': timestamp, + }, + { + host: { name: 'host-a', ip: '127.0.0.22' }, + agent: { version: 10 }, + id, + '@timestamp': timestamp, + }, + // missing agent.version + { + host: { name: 'host-a', ip: '127.0.0.31' }, + agent: { name: 'agent-a' }, + id, + '@timestamp': timestamp, + }, + { + host: { name: 'host-a', ip: '127.0.0.32' }, + agent: { name: 'agent-a' }, + id, + '@timestamp': timestamp, + }, + // missing both agent.* + { + host: { name: 'host-a', ip: '127.0.0.41' }, + id, + '@timestamp': timestamp, + }, + { + host: { name: 'host-a', ip: '127.0.0.42' }, + id, + '@timestamp': timestamp, + }, + ]; + + await indexListOfDocuments(firstExecutionDocuments); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + alert_suppression: { + group_by: ['agent.name', 'agent.version'], + missing_fields_strategy: 'doNotSuppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['agent.name', 'agent.version', ALERT_ORIGINAL_TIME], + }); + // from 8 injected, only one should be suppressed + expect(previewAlerts.length).toEqual(7); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-a', + }, + { + field: 'agent.version', + value: '10', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + // rest of alerts are not suppressed and do not have suppress properties + previewAlerts.slice(1).forEach((previewAlert) => { + const source = previewAlert._source; + expect(source).toHaveProperty('id', id); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_END); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_TERMS); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + }); + }); + + it('should deduplicate alerts while suppressing new ones on rule execution', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + const firstExecutionDocuments = [ + { + host: { name: 'host-a' }, + id, + '@timestamp': firstTimestamp, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': firstTimestamp, + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + // large look-back time covers all docs + from: 'now-1h', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(2); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + expect(previewAlerts[1]._source).toEqual({ + ...previewAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_ORIGINAL_TIME]: secondTimestamp, + [ALERT_SUPPRESSION_START]: secondTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + }); + + it('should not suppress more than limited number of alerts (max_signals)', async () => { + const id = uuidv4(); + + await indexGeneratedDocuments({ + docsCount: 12000, + seed: (index) => ({ + id, + '@timestamp': `2020-10-28T06:50:00.${index}Z`, + host: { + name: `host-${index}`, + }, + agent: { name: 'agent-a' }, + }), + }); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: `from ecs_compliant metadata _id, _index, _version ${internalIdPipe( + id + )} | sort @timestamp asc`, + alert_suppression: { + group_by: ['agent.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + max_signals: 200, + }; + + const { previewId, logs } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + + expect(logs[0].warnings).toEqual( + expect.arrayContaining([getSuppressionMaxAlertsWarning()]) + ); + + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['agent.name', ALERT_ORIGINAL_TIME], + size: 1000, + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-a', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 200, + }); + }); + + it('should generate up to max_signals alerts', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:05:00.000Z'; + + await indexGeneratedDocuments({ + docsCount: 20000, + seed: (index) => ({ + id, + '@timestamp': timestamp, + host: { + name: `host-${index}`, + }, + 'agent.name': `agent-${index}`, + }), + }); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + alert_suppression: { + group_by: ['agent.name'], + duration: { + value: 300, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + max_signals: 150, + }; + + const { previewId, logs } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 1, + }); + expect(logs[0].warnings).toEqual( + expect.arrayContaining([getSuppressionMaxAlertsWarning()]) + ); + + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + size: 1000, + sort: ['agent.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(150); + }); + }); + + describe('with exceptions', async () => { + afterEach(async () => { + await deleteAllExceptions(supertest, log); + }); + + it('should apply exceptions', async () => { + const id = uuidv4(); + const interval: [string, string] = ['2020-10-28T06:00:00.000Z', '2020-10-28T06:10:00.000Z']; + const doc1 = { agent: { name: 'test-1' }, 'client.ip': '127.0.0.2' }; + const doc2 = { agent: { name: 'test-1' } }; + const doc3 = { agent: { name: 'test-1' }, 'client.ip': '127.0.0.1' }; + + await indexEnhancedDocuments({ documents: [doc1, doc2, doc3], interval, id }); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: `from ecs_compliant ${internalIdPipe(id)} | where agent.name=="test-1"`, + from: 'now-1h', + interval: '1h', + alert_suppression: { + group_by: ['agent.name'], + missing_fields_strategy: 'suppress', + }, + }; + + const { previewId } = await previewRuleWithExceptionEntries({ + supertest, + log, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + entries: [ + [ + { + field: 'client.ip', + operator: 'included', + type: 'match', + value: '127.0.0.1', + }, + ], + ], + }); + + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).toBe(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'test-1', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + }); + + describe('alerts enrichment', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/entity/risks'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/entity/risks'); + }); + + it('should be enriched with host risk score', async () => { + const id = uuidv4(); + const interval: [string, string] = ['2020-10-28T06:00:00.000Z', '2020-10-28T06:10:00.000Z']; + const doc1 = { host: { name: 'host-0' } }; + + await indexEnhancedDocuments({ documents: [doc1, doc1], interval, id }); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: `from ecs_compliant ${internalIdPipe(id)} | where host.name=="host-0"`, + from: 'now-1h', + interval: '1h', + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + }); + + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).toBe(1); + + expect(previewAlerts[0]._source).toHaveProperty('host.risk.calculated_level', 'Low'); + expect(previewAlerts[0]._source).toHaveProperty('host.risk.calculated_score_norm', 1); + }); + }); + + describe('with asset criticality', async () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/asset_criticality'); + await kibanaServer.uiSettings.update({ + [ENABLE_ASSET_CRITICALITY_SETTING]: true, + }); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/asset_criticality'); + }); + + it('should be enriched alert with criticality_level', async () => { + const id = uuidv4(); + const interval: [string, string] = ['2020-10-28T06:00:00.000Z', '2020-10-28T06:10:00.000Z']; + const doc1 = { host: { name: 'host-0' } }; + + await indexEnhancedDocuments({ documents: [doc1, doc1], interval, id }); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: `from ecs_compliant ${internalIdPipe(id)} | where host.name=="host-0"`, + from: 'now-1h', + interval: '1h', + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + }); + + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).toBe(1); + + expect(previewAlerts[0]?._source?.['host.asset.criticality']).toBe('extreme_impact'); + }); + }); + }); +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/index.ts index 329cef4ac7e8c..3ea2c4e6c9359 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/index.ts @@ -12,6 +12,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./eql')); loadTestFile(require.resolve('./eql_alert_suppression')); loadTestFile(require.resolve('./esql')); + loadTestFile(require.resolve('./esql_suppression')); loadTestFile(require.resolve('./machine_learning')); loadTestFile(require.resolve('./new_terms')); loadTestFile(require.resolve('./new_terms_alert_suppression')); diff --git a/x-pack/test/security_solution_cypress/config.ts b/x-pack/test/security_solution_cypress/config.ts index 588843b731f45..606df49c4f90e 100644 --- a/x-pack/test/security_solution_cypress/config.ts +++ b/x-pack/test/security_solution_cypress/config.ts @@ -45,6 +45,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { '--xpack.alerting.rules.minimumScheduleInterval.value=1s', '--xpack.ruleRegistry.unsafe.legacyMultiTenancy.enabled=true', `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForEsqlRuleEnabled', 'bulkCustomHighlightedFieldsEnabled', ])}`, // mock cloud to enable the guided onboarding tour in e2e tests diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_suppression_serverless_essentials.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_suppression_serverless_essentials.cy.ts index 62a406cd9d466..d6f23687cf418 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_suppression_serverless_essentials.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_suppression_serverless_essentials.cy.ts @@ -9,6 +9,7 @@ import { selectThresholdRuleType, selectIndicatorMatchType, selectNewTermsRuleType, + selectEsqlRuleType, } from '../../../../tasks/create_new_rule'; import { login } from '../../../../tasks/login'; import { visit } from '../../../../tasks/navigation'; @@ -21,6 +22,7 @@ import { CREATE_RULE_URL } from '../../../../urls/navigation'; describe( 'Detection rules, Alert Suppression for Essentials tier', { + // skipped in MKI as it depends on feature flag alertSuppressionForEsqlRuleEnabled tags: ['@serverless', '@skipInServerlessMKI'], env: { ftrConfig: { @@ -29,6 +31,12 @@ describe( { product_line: 'endpoint', product_tier: 'essentials' }, ], }, + // alertSuppressionForEsqlRuleEnabled feature flag is also enabled in a global config + kbnServerArgs: [ + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForEsqlRuleEnabled', + ])}`, + ], }, }, () => { @@ -49,6 +57,9 @@ describe( selectThresholdRuleType(); cy.get(THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX).should('be.enabled'); + + selectEsqlRuleType(); + cy.get(ALERT_SUPPRESSION_FIELDS_INPUT).should('be.enabled'); }); } ); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_supression_ess_basic.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_supression_ess_basic.cy.ts index fbcc43e4652ae..1f86d6d0dd789 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_supression_ess_basic.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_supression_ess_basic.cy.ts @@ -14,6 +14,7 @@ import { selectIndicatorMatchType, selectNewTermsRuleType, selectThresholdRuleType, + selectEsqlRuleType, openSuppressionFieldsTooltipAndCheckLicense, } from '../../../../tasks/create_new_rule'; import { startBasicLicense } from '../../../../tasks/api_calls/licensing'; @@ -48,6 +49,9 @@ describe( selectNewTermsRuleType(); openSuppressionFieldsTooltipAndCheckLicense(); + selectEsqlRuleType(); + openSuppressionFieldsTooltipAndCheckLicense(); + selectThresholdRuleType(); cy.get(THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX).should('be.disabled'); cy.get(THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX).parent().trigger('mouseover'); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule.cy.ts new file mode 100644 index 0000000000000..b8cebae392d38 --- /dev/null +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule.cy.ts @@ -0,0 +1,292 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getEsqlRule } from '../../../../objects/rule'; + +import { + RULES_MANAGEMENT_TABLE, + RULE_NAME, + INVESTIGATION_FIELDS_VALUE_ITEM, +} from '../../../../screens/alerts_detection_rules'; +import { + RULE_NAME_HEADER, + RULE_TYPE_DETAILS, + RULE_NAME_OVERRIDE_DETAILS, + DEFINITION_DETAILS, + SUPPRESS_BY_DETAILS, + SUPPRESS_FOR_DETAILS, + SUPPRESS_MISSING_FIELD, +} from '../../../../screens/rule_details'; + +import { ESQL_QUERY_BAR } from '../../../../screens/create_new_rule'; + +import { getDetails, goBackToRulesTable } from '../../../../tasks/rule_details'; +import { expectNumberOfRules } from '../../../../tasks/alerts_detection_rules'; +import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; +import { + expandEsqlQueryBar, + fillAboutRuleAndContinue, + fillDefineEsqlRuleAndContinue, + fillScheduleRuleAndContinue, + selectEsqlRuleType, + getDefineContinueButton, + fillEsqlQueryBar, + fillAboutSpecificEsqlRuleAndContinue, + createRuleWithoutEnabling, + expandAdvancedSettings, + fillCustomInvestigationFields, + fillRuleName, + fillDescription, + getAboutContinueButton, + fillAlertSuppressionFields, + selectAlertSuppressionPerInterval, + setAlertSuppressionDuration, + selectDoNotSuppressForMissingFields, + continueFromDefineStep, + fillAboutRuleMinimumAndContinue, + skipScheduleRuleAction, + interceptEsqlQueryFieldsRequest, +} from '../../../../tasks/create_new_rule'; +import { login } from '../../../../tasks/login'; +import { visit } from '../../../../tasks/navigation'; + +import { CREATE_RULE_URL } from '../../../../urls/navigation'; + +// https://github.com/cypress-io/cypress/issues/22113 +// issue is inside monaco editor, used in ES|QL query input +// calling it after visiting page in each tests, seems fixes the issue +// the only other alternative is patching ResizeObserver, which is something I would like to avoid +const workaroundForResizeObserver = () => + cy.on('uncaught:exception', (err) => { + if (err.message.includes('ResizeObserver loop limit exceeded')) { + return false; + } + }); + +describe( + 'Detection ES|QL rules, creation', + { + // skipped in MKI as it depends on feature flag alertSuppressionForEsqlRuleEnabled + // alertSuppressionForEsqlRuleEnabled feature flag is also enabled in a global config + tags: ['@ess', '@serverless', '@skipInServerlessMKI'], + env: { + kbnServerArgs: [ + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForEsqlRuleEnabled', + ])}`, + ], + }, + }, + () => { + const rule = getEsqlRule(); + const expectedNumberOfRules = 1; + + describe('creation', () => { + beforeEach(() => { + deleteAlertsAndRules(); + login(); + + visit(CREATE_RULE_URL); + workaroundForResizeObserver(); + }); + + it('creates an ES|QL rule', function () { + selectEsqlRuleType(); + expandEsqlQueryBar(); + + fillDefineEsqlRuleAndContinue(rule); + fillAboutRuleAndContinue(rule); + fillScheduleRuleAndContinue(rule); + createRuleWithoutEnabling(); + + // ensures after rule save ES|QL rule is displayed + cy.get(RULE_NAME_HEADER).should('contain', `${rule.name}`); + getDetails(RULE_TYPE_DETAILS).contains('ES|QL'); + + // ensures newly created rule is displayed in table + goBackToRulesTable(); + + expectNumberOfRules(RULES_MANAGEMENT_TABLE, expectedNumberOfRules); + + cy.get(RULE_NAME).should('have.text', rule.name); + }); + + // this test case is important, since field shown in rule override component are coming from ES|QL query, not data view fields API + it('creates an ES|QL rule and overrides its name', function () { + selectEsqlRuleType(); + expandEsqlQueryBar(); + + fillDefineEsqlRuleAndContinue(rule); + fillAboutSpecificEsqlRuleAndContinue({ ...rule, rule_name_override: 'test_id' }); + fillScheduleRuleAndContinue(rule); + createRuleWithoutEnabling(); + + // ensure rule name override is displayed on details page + getDetails(RULE_NAME_OVERRIDE_DETAILS).should('have.text', 'test_id'); + }); + }); + + describe('ES|QL query validation', () => { + beforeEach(() => { + login(); + visit(CREATE_RULE_URL); + + workaroundForResizeObserver(); + }); + it('shows error when ES|QL query is empty', function () { + selectEsqlRuleType(); + expandEsqlQueryBar(); + getDefineContinueButton().click(); + + cy.get(ESQL_QUERY_BAR).contains('ES|QL query is required'); + }); + + it('proceeds further once invalid query is fixed', function () { + selectEsqlRuleType(); + expandEsqlQueryBar(); + getDefineContinueButton().click(); + + cy.get(ESQL_QUERY_BAR).contains('required'); + + // once correct query typed, we can proceed ot the next step + fillEsqlQueryBar(rule.query); + getDefineContinueButton().click(); + + cy.get(ESQL_QUERY_BAR).should('not.be.visible'); + }); + + it('shows error when non-aggregating ES|QL query does not have metadata operator', function () { + const invalidNonAggregatingQuery = 'from auditbeat* | limit 5'; + selectEsqlRuleType(); + expandEsqlQueryBar(); + fillEsqlQueryBar(invalidNonAggregatingQuery); + getDefineContinueButton().click(); + + cy.get(ESQL_QUERY_BAR).contains( + 'must include the "metadata _id, _version, _index" operator after the source command' + ); + }); + + it('shows error when non-aggregating ES|QL query does not return _id field', function () { + const invalidNonAggregatingQuery = + 'from auditbeat* metadata _id, _version, _index | keep agent.* | limit 5'; + + selectEsqlRuleType(); + expandEsqlQueryBar(); + fillEsqlQueryBar(invalidNonAggregatingQuery); + getDefineContinueButton().click(); + + cy.get(ESQL_QUERY_BAR).contains( + 'must include the "metadata _id, _version, _index" operator after the source command' + ); + }); + + it('shows error when ES|QL query is invalid', function () { + const invalidEsqlQuery = + 'from auditbeat* metadata _id, _version, _index | not_existing_operator'; + visit(CREATE_RULE_URL); + + selectEsqlRuleType(); + expandEsqlQueryBar(); + fillEsqlQueryBar(invalidEsqlQuery); + getDefineContinueButton().click(); + + cy.get(ESQL_QUERY_BAR).contains('Error validating ES|QL'); + }); + }); + + describe('ES|QL investigation fields', () => { + beforeEach(() => { + login(); + visit(CREATE_RULE_URL); + }); + it('shows custom ES|QL field in investigation fields autocomplete and saves it in rule', function () { + const CUSTOM_ESQL_FIELD = '_custom_agent_name'; + const queryWithCustomFields = [ + `from auditbeat* metadata _id, _version, _index`, + `eval ${CUSTOM_ESQL_FIELD} = agent.name`, + `keep _id, _custom_agent_name`, + `limit 5`, + ].join(' | '); + + workaroundForResizeObserver(); + + selectEsqlRuleType(); + expandEsqlQueryBar(); + fillEsqlQueryBar(queryWithCustomFields); + getDefineContinueButton().click(); + + expandAdvancedSettings(); + fillRuleName(); + fillDescription(); + fillCustomInvestigationFields([CUSTOM_ESQL_FIELD]); + getAboutContinueButton().click(); + + fillScheduleRuleAndContinue(rule); + createRuleWithoutEnabling(); + + cy.get(INVESTIGATION_FIELDS_VALUE_ITEM).should('have.text', CUSTOM_ESQL_FIELD); + }); + }); + + describe('Alert suppression', () => { + beforeEach(() => { + login(); + visit(CREATE_RULE_URL); + }); + it('shows custom ES|QL field in investigation fields autocomplete and saves it in rule', function () { + const CUSTOM_ESQL_FIELD = '_custom_agent_name'; + const SUPPRESS_BY_FIELDS = [CUSTOM_ESQL_FIELD, 'agent.type']; + + const queryWithCustomFields = [ + `from auditbeat* metadata _id, _version, _index`, + `eval ${CUSTOM_ESQL_FIELD} = agent.name`, + `drop agent.*`, + ].join(' | '); + + workaroundForResizeObserver(); + + selectEsqlRuleType(); + expandEsqlQueryBar(); + + interceptEsqlQueryFieldsRequest(queryWithCustomFields, 'esqlSuppressionFieldsRequest'); + fillEsqlQueryBar(queryWithCustomFields); + + cy.wait('@esqlSuppressionFieldsRequest'); + fillAlertSuppressionFields(SUPPRESS_BY_FIELDS); + selectAlertSuppressionPerInterval(); + setAlertSuppressionDuration(2, 'h'); + selectDoNotSuppressForMissingFields(); + continueFromDefineStep(); + + // ensures details preview works correctly + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', SUPPRESS_BY_FIELDS.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', '2h'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Do not suppress alerts for events with missing fields' + ); + }); + + fillAboutRuleMinimumAndContinue(rule); + skipScheduleRuleAction(); + createRuleWithoutEnabling(); + + // ensures rule details displayed correctly after rule created + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', SUPPRESS_BY_FIELDS.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', '2h'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Do not suppress alerts for events with missing fields' + ); + }); + }); + }); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule_ess.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule_ess.cy.ts deleted file mode 100644 index 2e95bb19a0477..0000000000000 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule_ess.cy.ts +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { getEsqlRule } from '../../../../objects/rule'; - -import { - RULES_MANAGEMENT_TABLE, - RULE_NAME, - INVESTIGATION_FIELDS_VALUE_ITEM, -} from '../../../../screens/alerts_detection_rules'; -import { - RULE_NAME_HEADER, - RULE_TYPE_DETAILS, - RULE_NAME_OVERRIDE_DETAILS, -} from '../../../../screens/rule_details'; - -import { ESQL_QUERY_BAR } from '../../../../screens/create_new_rule'; - -import { getDetails, goBackToRulesTable } from '../../../../tasks/rule_details'; -import { expectNumberOfRules } from '../../../../tasks/alerts_detection_rules'; -import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; -import { - expandEsqlQueryBar, - fillAboutRuleAndContinue, - fillDefineEsqlRuleAndContinue, - fillScheduleRuleAndContinue, - selectEsqlRuleType, - getDefineContinueButton, - fillEsqlQueryBar, - fillAboutSpecificEsqlRuleAndContinue, - createRuleWithoutEnabling, - expandAdvancedSettings, - fillCustomInvestigationFields, - fillRuleName, - fillDescription, - getAboutContinueButton, -} from '../../../../tasks/create_new_rule'; -import { login } from '../../../../tasks/login'; -import { visit } from '../../../../tasks/navigation'; - -import { CREATE_RULE_URL } from '../../../../urls/navigation'; - -// https://github.com/cypress-io/cypress/issues/22113 -// issue is inside monaco editor, used in ES|QL query input -// calling it after visiting page in each tests, seems fixes the issue -// the only other alternative is patching ResizeObserver, which is something I would like to avoid -const workaroundForResizeObserver = () => - cy.on('uncaught:exception', (err) => { - if (err.message.includes('ResizeObserver loop limit exceeded')) { - return false; - } - }); - -describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => { - const rule = getEsqlRule(); - const expectedNumberOfRules = 1; - - describe('creation', () => { - beforeEach(() => { - deleteAlertsAndRules(); - login(); - }); - - it('creates an ES|QL rule', function () { - visit(CREATE_RULE_URL); - workaroundForResizeObserver(); - - selectEsqlRuleType(); - expandEsqlQueryBar(); - - fillDefineEsqlRuleAndContinue(rule); - fillAboutRuleAndContinue(rule); - fillScheduleRuleAndContinue(rule); - createRuleWithoutEnabling(); - - // ensures after rule save ES|QL rule is displayed - cy.get(RULE_NAME_HEADER).should('contain', `${rule.name}`); - getDetails(RULE_TYPE_DETAILS).contains('ES|QL'); - - // ensures newly created rule is displayed in table - goBackToRulesTable(); - - expectNumberOfRules(RULES_MANAGEMENT_TABLE, expectedNumberOfRules); - - cy.get(RULE_NAME).should('have.text', rule.name); - }); - - // this test case is important, since field shown in rule override component are coming from ES|QL query, not data view fields API - it('creates an ES|QL rule and overrides its name', function () { - visit(CREATE_RULE_URL); - workaroundForResizeObserver(); - - selectEsqlRuleType(); - expandEsqlQueryBar(); - - fillDefineEsqlRuleAndContinue(rule); - fillAboutSpecificEsqlRuleAndContinue({ ...rule, rule_name_override: 'test_id' }); - fillScheduleRuleAndContinue(rule); - createRuleWithoutEnabling(); - - // ensure rule name override is displayed on details page - getDetails(RULE_NAME_OVERRIDE_DETAILS).should('have.text', 'test_id'); - }); - }); - - describe('ES|QL query validation', () => { - beforeEach(() => { - login(); - visit(CREATE_RULE_URL); - }); - it('shows error when ES|QL query is empty', function () { - workaroundForResizeObserver(); - - selectEsqlRuleType(); - expandEsqlQueryBar(); - getDefineContinueButton().click(); - - cy.get(ESQL_QUERY_BAR).contains('ES|QL query is required'); - }); - - it('proceeds further once invalid query is fixed', function () { - workaroundForResizeObserver(); - - selectEsqlRuleType(); - expandEsqlQueryBar(); - getDefineContinueButton().click(); - - cy.get(ESQL_QUERY_BAR).contains('required'); - - // once correct query typed, we can proceed ot the next step - fillEsqlQueryBar(rule.query); - getDefineContinueButton().click(); - - cy.get(ESQL_QUERY_BAR).should('not.be.visible'); - }); - - it('shows error when non-aggregating ES|QL query does not have metadata operator', function () { - workaroundForResizeObserver(); - - const invalidNonAggregatingQuery = 'from auditbeat* | limit 5'; - selectEsqlRuleType(); - expandEsqlQueryBar(); - fillEsqlQueryBar(invalidNonAggregatingQuery); - getDefineContinueButton().click(); - - cy.get(ESQL_QUERY_BAR).contains( - 'must include the "metadata _id, _version, _index" operator after the source command' - ); - }); - - it('shows error when non-aggregating ES|QL query does not return _id field', function () { - workaroundForResizeObserver(); - - const invalidNonAggregatingQuery = - 'from auditbeat* metadata _id, _version, _index | keep agent.* | limit 5'; - - selectEsqlRuleType(); - expandEsqlQueryBar(); - fillEsqlQueryBar(invalidNonAggregatingQuery); - getDefineContinueButton().click(); - - cy.get(ESQL_QUERY_BAR).contains( - 'must include the "metadata _id, _version, _index" operator after the source command' - ); - }); - - it('shows error when ES|QL query is invalid', function () { - workaroundForResizeObserver(); - const invalidEsqlQuery = - 'from auditbeat* metadata _id, _version, _index | not_existing_operator'; - visit(CREATE_RULE_URL); - - selectEsqlRuleType(); - expandEsqlQueryBar(); - fillEsqlQueryBar(invalidEsqlQuery); - getDefineContinueButton().click(); - - cy.get(ESQL_QUERY_BAR).contains('Error validating ES|QL'); - }); - }); - - describe('ES|QL investigation fields', () => { - beforeEach(() => { - login(); - visit(CREATE_RULE_URL); - }); - it('shows custom ES|QL field in investigation fields autocomplete and saves it in rule', function () { - const CUSTOM_ESQL_FIELD = '_custom_agent_name'; - const queryWithCustomFields = [ - `from auditbeat* metadata _id, _version, _index`, - `eval ${CUSTOM_ESQL_FIELD} = agent.name`, - `keep _id, _custom_agent_name`, - `limit 5`, - ].join(' | '); - - workaroundForResizeObserver(); - - selectEsqlRuleType(); - expandEsqlQueryBar(); - fillEsqlQueryBar(queryWithCustomFields); - getDefineContinueButton().click(); - - expandAdvancedSettings(); - fillRuleName(); - fillDescription(); - fillCustomInvestigationFields([CUSTOM_ESQL_FIELD]); - getAboutContinueButton().click(); - - fillScheduleRuleAndContinue(rule); - createRuleWithoutEnabling(); - - cy.get(INVESTIGATION_FIELDS_VALUE_ITEM).should('have.text', CUSTOM_ESQL_FIELD); - }); - }); -}); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/esql_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/esql_rule.cy.ts index 265263ba495c6..a255ce289b1a7 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/esql_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/esql_rule.cy.ts @@ -7,9 +7,22 @@ import { getEsqlRule } from '../../../../objects/rule'; -import { ESQL_QUERY_DETAILS, RULE_NAME_OVERRIDE_DETAILS } from '../../../../screens/rule_details'; +import { + ESQL_QUERY_DETAILS, + RULE_NAME_OVERRIDE_DETAILS, + SUPPRESS_FOR_DETAILS, + DEFINITION_DETAILS, + SUPPRESS_MISSING_FIELD, + SUPPRESS_BY_DETAILS, + DETAILS_TITLE, +} from '../../../../screens/rule_details'; -import { ESQL_QUERY_BAR } from '../../../../screens/create_new_rule'; +import { + ESQL_QUERY_BAR, + ALERT_SUPPRESSION_DURATION_INPUT, + ALERT_SUPPRESSION_FIELDS, + ALERT_SUPPRESSION_MISSING_FIELDS_SUPPRESS, +} from '../../../../screens/create_new_rule'; import { createRule } from '../../../../tasks/api_calls/rules'; @@ -17,12 +30,15 @@ import { RULES_MANAGEMENT_URL } from '../../../../urls/rules_management'; import { getDetails } from '../../../../tasks/rule_details'; import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; import { - clearEsqlQueryBar, expandEsqlQueryBar, fillEsqlQueryBar, fillOverrideEsqlRuleName, goToAboutStepTab, expandAdvancedSettings, + selectAlertSuppressionPerRuleExecution, + selectDoNotSuppressForMissingFields, + fillAlertSuppressionFields, + interceptEsqlQueryFieldsRequest, } from '../../../../tasks/create_new_rule'; import { login } from '../../../../tasks/login'; @@ -33,63 +49,160 @@ import { visit } from '../../../../tasks/navigation'; const rule = getEsqlRule(); -const expectedValidEsqlQuery = 'from auditbeat* | stats count(event.category) by event.category'; - -describe('Detection ES|QL rules, edit', { tags: ['@ess'] }, () => { - beforeEach(() => { - login(); - deleteAlertsAndRules(); - createRule(rule); - }); - - it('edits ES|QL rule and checks details page', () => { - visit(RULES_MANAGEMENT_URL); - editFirstRule(); - expandEsqlQueryBar(); - // ensure once edit form opened, correct query is displayed in ES|QL input - cy.get(ESQL_QUERY_BAR).contains(rule.query); - - clearEsqlQueryBar(); - fillEsqlQueryBar(expectedValidEsqlQuery); - - saveEditedRule(); - - // ensure updated query is displayed on details page - getDetails(ESQL_QUERY_DETAILS).should('have.text', expectedValidEsqlQuery); - }); - - it('edits ES|QL rule query and override rule name with new property', () => { - visit(RULES_MANAGEMENT_URL); - editFirstRule(); - clearEsqlQueryBar(); - fillEsqlQueryBar(expectedValidEsqlQuery); - - goToAboutStepTab(); - expandAdvancedSettings(); - fillOverrideEsqlRuleName('event.category'); - - saveEditedRule(); - - // ensure rule name override is displayed on details page - getDetails(RULE_NAME_OVERRIDE_DETAILS).should('have.text', 'event.category'); - }); - - it('adds ES|QL override rule name on edit', () => { - visit(RULES_MANAGEMENT_URL); - editFirstRule(); - - expandEsqlQueryBar(); - // ensure once edit form opened, correct query is displayed in ES|QL input - cy.get(ESQL_QUERY_BAR).contains(rule.query); - - goToAboutStepTab(); - expandAdvancedSettings(); - // this field defined to be returned in rule query - fillOverrideEsqlRuleName('test_id'); - - saveEditedRule(); - - // ensure rule name override is displayed on details page - getDetails(RULE_NAME_OVERRIDE_DETAILS).should('have.text', 'test_id'); - }); -}); +const expectedValidEsqlQuery = + 'from auditbeat* | stats _count=count(event.category) by event.category'; + +// skipped in MKI as it depends on feature flag alertSuppressionForEsqlRuleEnabled +// alertSuppressionForEsqlRuleEnabled feature flag is also enabled in a global config +describe( + 'Detection ES|QL rules, edit', + { + tags: ['@ess', '@serverless', '@skipInServerlessMKI'], + env: { + kbnServerArgs: [ + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForEsqlRuleEnabled', + ])}`, + ], + }, + }, + () => { + beforeEach(() => { + login(); + deleteAlertsAndRules(); + createRule(rule); + + visit(RULES_MANAGEMENT_URL); + editFirstRule(); + }); + + it('edits ES|QL rule and checks details page', () => { + expandEsqlQueryBar(); + // ensure once edit form opened, correct query is displayed in ES|QL input + cy.get(ESQL_QUERY_BAR).contains(rule.query); + + fillEsqlQueryBar(expectedValidEsqlQuery); + + saveEditedRule(); + + // ensure updated query is displayed on details page + getDetails(ESQL_QUERY_DETAILS).should('have.text', expectedValidEsqlQuery); + }); + + it('edits ES|QL rule query and override rule name with new property', () => { + fillEsqlQueryBar(expectedValidEsqlQuery); + + goToAboutStepTab(); + expandAdvancedSettings(); + fillOverrideEsqlRuleName('event.category'); + + saveEditedRule(); + + // ensure rule name override is displayed on details page + getDetails(RULE_NAME_OVERRIDE_DETAILS).should('have.text', 'event.category'); + }); + + it('adds ES|QL override rule name on edit', () => { + expandEsqlQueryBar(); + // ensure once edit form opened, correct query is displayed in ES|QL input + cy.get(ESQL_QUERY_BAR).contains(rule.query); + + goToAboutStepTab(); + expandAdvancedSettings(); + // this field defined to be returned in rule query + fillOverrideEsqlRuleName('test_id'); + + saveEditedRule(); + + // ensure rule name override is displayed on details page + getDetails(RULE_NAME_OVERRIDE_DETAILS).should('have.text', 'test_id'); + }); + + describe('with configured suppression', () => { + const SUPPRESS_BY_FIELDS = ['event.category']; + const NEW_SUPPRESS_BY_FIELDS = ['event.category', '_count']; + + beforeEach(() => { + deleteAlertsAndRules(); + createRule({ + ...rule, + query: expectedValidEsqlQuery, + alert_suppression: { + group_by: SUPPRESS_BY_FIELDS, + duration: { value: 3, unit: 'h' }, + missing_fields_strategy: 'suppress', + }, + }); + }); + + it('displays suppress options correctly on edit form and allows its editing', () => { + visit(RULES_MANAGEMENT_URL); + + interceptEsqlQueryFieldsRequest(expectedValidEsqlQuery, 'esqlSuppressionFieldsRequest'); + editFirstRule(); + + // check saved suppression settings + cy.get(ALERT_SUPPRESSION_DURATION_INPUT).eq(0).should('be.enabled').should('have.value', 3); + cy.get(ALERT_SUPPRESSION_DURATION_INPUT) + .eq(1) + .should('be.enabled') + .should('have.value', 'h'); + cy.get(ALERT_SUPPRESSION_FIELDS).should('contain', SUPPRESS_BY_FIELDS.join('')); + cy.get(ALERT_SUPPRESSION_MISSING_FIELDS_SUPPRESS).should('be.checked'); + + selectAlertSuppressionPerRuleExecution(); + selectDoNotSuppressForMissingFields(); + + cy.wait('@esqlSuppressionFieldsRequest'); + fillAlertSuppressionFields(['_count']); + + saveEditedRule(); + + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', NEW_SUPPRESS_BY_FIELDS.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', 'One rule execution'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Do not suppress alerts for events with missing fields' + ); + }); + }); + }); + + describe('without suppression', () => { + const SUPPRESS_BY_FIELDS = ['event.category']; + + beforeEach(() => { + deleteAlertsAndRules(); + createRule({ + ...rule, + query: expectedValidEsqlQuery, + }); + }); + + it('enables suppression on time interval', () => { + visit(RULES_MANAGEMENT_URL); + + interceptEsqlQueryFieldsRequest(expectedValidEsqlQuery, 'esqlSuppressionFieldsRequest'); + editFirstRule(); + + cy.wait('@esqlSuppressionFieldsRequest'); + fillAlertSuppressionFields(SUPPRESS_BY_FIELDS); + + saveEditedRule(); + + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', SUPPRESS_BY_FIELDS.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', 'One rule execution'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Suppress and group alerts for events with missing fields' + ); + + // suppression functionality should be under Tech Preview + cy.contains(DETAILS_TITLE, SUPPRESS_FOR_DETAILS).contains('Technical Preview'); + }); + }); + }); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts index 15229445e54f0..13af60594d103 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts @@ -337,6 +337,14 @@ describe('Detection rules, Prebuilt Rules Installation and Update workflow', () type: 'esql', language: 'esql', query: 'FROM .alerts-security.alerts-default | STATS count = COUNT(@timestamp) BY @timestamp', + alert_suppression: { + group_by: [ + 'Endpoint.policy.applied.artifacts.global.identifiers.name', + 'Endpoint.policy.applied.id', + ], + duration: { unit: 'm', value: 5 }, + missing_fields_strategy: 'suppress', + }, }); const RULE_WITHOUT_INVESTIGATION_AND_SETUP_GUIDES = createRuleAssetSavedObject({ @@ -621,25 +629,23 @@ describe('Detection rules, Prebuilt Rules Installation and Update workflow', () const { query } = NEW_TERMS_INDEX_PATTERN_RULE['security-rule'] as { query: string }; assertCustomQueryPropertyShown(query); }); - }); - describe( - 'Skip in Serverless environment', - { tags: TEST_ENV_TAGS.filter((tag) => tag !== '@serverless') }, - () => { - /* Serverless environment doesn't support ESQL rules just yet */ - it('ESQL rule properties', () => { - clickAddElasticRulesButton(); + it('ESQL rule properties', () => { + clickAddElasticRulesButton(); - openRuleInstallPreview(ESQL_RULE['security-rule'].name); + openRuleInstallPreview(ESQL_RULE['security-rule'].name); - assertCommonPropertiesShown(commonProperties); + assertCommonPropertiesShown(commonProperties); - const { query } = ESQL_RULE['security-rule'] as { query: string }; - assertEsqlQueryPropertyShown(query); - }); - } - ); + const { query } = ESQL_RULE['security-rule'] as { query: string }; + assertEsqlQueryPropertyShown(query); + + const { alert_suppression: alertSuppression } = ESQL_RULE['security-rule'] as { + alert_suppression: AlertSuppression; + }; + assertAlertSuppressionPropertiesShown(alertSuppression); + }); + }); }); }); @@ -1049,26 +1055,24 @@ describe('Detection rules, Prebuilt Rules Installation and Update workflow', () }; assertCustomQueryPropertyShown(query); }); - }); - describe( - 'Skip in Serverless environment', - { tags: TEST_ENV_TAGS.filter((tag) => tag !== '@serverless') }, - () => { - /* Serverless environment doesn't support ESQL rules just yet */ - it('ESQL rule properties', () => { - clickRuleUpdatesTab(); + it('ESQL rule properties', () => { + clickRuleUpdatesTab(); + + openRuleUpdatePreview(UPDATED_ESQL_RULE['security-rule'].name); + selectPreviewTab(PREVIEW_TABS.OVERVIEW); - openRuleUpdatePreview(UPDATED_ESQL_RULE['security-rule'].name); - selectPreviewTab(PREVIEW_TABS.OVERVIEW); + assertCommonPropertiesShown(commonProperties); - assertCommonPropertiesShown(commonProperties); + const { query } = UPDATED_ESQL_RULE['security-rule'] as { query: string }; + assertEsqlQueryPropertyShown(query); - const { query } = UPDATED_ESQL_RULE['security-rule'] as { query: string }; - assertEsqlQueryPropertyShown(query); - }); - } - ); + const { alert_suppression: alertSuppression } = UPDATED_ESQL_RULE['security-rule'] as { + alert_suppression: AlertSuppression; + }; + assertAlertSuppressionPropertiesShown(alertSuppression); + }); + }); }); describe('Viewing rule changes in JSON diff view', { tags: TEST_ENV_TAGS }, () => { diff --git a/x-pack/test/security_solution_cypress/cypress/objects/rule.ts b/x-pack/test/security_solution_cypress/cypress/objects/rule.ts index 0b9a4ddedf04d..fcbaaca3d1dde 100644 --- a/x-pack/test/security_solution_cypress/cypress/objects/rule.ts +++ b/x-pack/test/security_solution_cypress/cypress/objects/rule.ts @@ -406,7 +406,7 @@ export const getEsqlRule = ( ): EsqlRuleCreateProps => ({ type: 'esql', language: 'esql', - query: 'from auditbeat-* [metadata _id, _version, _index] | keep agent.*,_id | eval test_id=_id', + query: 'from auditbeat-* metadata _id, _version, _index | keep agent.*,_id | eval test_id=_id', name: 'ES|QL Rule', description: 'The new rule description.', severity: 'high', diff --git a/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts b/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts index a71d990ec31a9..181f5dfa22eb1 100644 --- a/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts +++ b/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts @@ -613,16 +613,23 @@ export const fillDefineNewTermsRuleAndContinue = (rule: NewTermsRuleCreateProps) cy.get(DEFINE_CONTINUE_BUTTON).should('exist').click({ force: true }); }; -export const fillEsqlQueryBar = (query: string) => { +const typeEsqlQueryBar = (query: string) => { // eslint-disable-next-line cypress/no-force cy.get(ESQL_QUERY_BAR_INPUT_AREA).should('not.be.disabled').type(query, { force: true }); }; -export const clearEsqlQueryBar = () => { - // monaco editor under the hood is quite complex in matter to clear it - // underlying textarea holds just the last character of query displayed in search bar - // in order to clear it - it requires to select all text within editor and type in it - fillEsqlQueryBar(Cypress.platform === 'darwin' ? '{cmd}a' : '{ctrl}a'); +/** + * clears ES|QL search bar first + * types new query + */ +export const fillEsqlQueryBar = (query: string) => { + // before typing anything in query bar, we need to clear it + // Since first click on ES|QL query bar trigger re-render. We need to clear search bar during second attempt + typeEsqlQueryBar(' '); + typeEsqlQueryBar(Cypress.platform === 'darwin' ? '{cmd}a{del}' : '{ctrl}a{del}'); + + // only after this query can be safely typed + typeEsqlQueryBar(query); }; /** @@ -939,6 +946,20 @@ export const openSuppressionFieldsTooltipAndCheckLicense = () => { cy.get(TOOLTIP).contains('Platinum license'); }; +/** + * intercepts /internal/bsearch request that contains esqlQuery and adds alias to it + */ +export const interceptEsqlQueryFieldsRequest = ( + esqlQuery: string, + alias: string = 'esqlQueryFields' +) => { + cy.intercept('POST', '/internal/bsearch?*', (req) => { + if (req.body?.batch?.[0]?.request?.params?.query?.includes?.(esqlQuery)) { + req.alias = alias; + } + }); +}; + export const checkLoadQueryDynamically = () => { cy.get(LOAD_QUERY_DYNAMICALLY_CHECKBOX).click({ force: true }); cy.get(LOAD_QUERY_DYNAMICALLY_CHECKBOX).should('be.checked'); diff --git a/x-pack/test/security_solution_cypress/serverless_config.ts b/x-pack/test/security_solution_cypress/serverless_config.ts index 51462be717fc8..04a15e49d070a 100644 --- a/x-pack/test/security_solution_cypress/serverless_config.ts +++ b/x-pack/test/security_solution_cypress/serverless_config.ts @@ -35,6 +35,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { { product_line: 'cloud', product_tier: 'complete' }, ])}`, `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForEsqlRuleEnabled', 'bulkCustomHighlightedFieldsEnabled', ])}`, ],