From 7478b45ee6694ee64713a95291662990775470d5 Mon Sep 17 00:00:00 2001 From: Marshall Main <55718608+marshallmain@users.noreply.github.com> Date: Thu, 4 Feb 2021 12:45:29 -0500 Subject: [PATCH] [Security Solution][Detections] Reduce detection engine reliance on _source (#89371) (#90287) * First pass at switching rules to depend on fields instead of _source * Fix tests * Change operator: excluded logic so missing fields are allowlisted Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../signals/__mocks__/es_results.ts | 11 +++++ .../signals/build_events_query.test.ts | 41 ++++++++++++++++--- .../signals/build_events_query.ts | 6 +++ .../bulk_create_threshold_signals.test.ts | 2 +- .../signals/bulk_create_threshold_signals.ts | 6 +-- .../create_field_and_set_tuples.test.ts | 8 ++-- .../create_set_to_filter_against.test.ts | 8 ++-- .../filters/create_set_to_filter_against.ts | 3 +- .../signals/filters/filter_events.test.ts | 10 ++--- .../signals/filters/filter_events.ts | 13 +++--- .../filter_events_against_list.test.ts | 16 ++++---- .../signals/find_threshold_signals.ts | 6 +++ .../signals/search_after_bulk_create.test.ts | 14 +++---- .../signals/single_bulk_create.test.ts | 14 +------ .../build_threat_mapping_filter.mock.ts | 11 +++++ .../build_threat_mapping_filter.test.ts | 18 ++++++-- .../build_threat_mapping_filter.ts | 9 ++-- .../signals/threat_mapping/get_threat_list.ts | 6 +++ .../detection_engine/signals/utils.test.ts | 25 +++++++---- .../server/lib/machine_learning/index.ts | 6 +++ 20 files changed, 162 insertions(+), 71 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index eb38c58d82ea1..6011c67376973 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -166,6 +166,12 @@ export const sampleDocWithSortId = ( ip: destIp ?? '127.0.0.1', }, }, + fields: { + someKey: ['someValue'], + '@timestamp': ['2020-04-20T21:27:45+0000'], + 'source.ip': ip ? (Array.isArray(ip) ? ip : [ip]) : ['127.0.0.1'], + 'destination.ip': destIp ? (Array.isArray(destIp) ? destIp : [destIp]) : ['127.0.0.1'], + }, sort: ['1234567891111'], }); @@ -185,6 +191,11 @@ export const sampleDocNoSortId = ( ip: ip ?? '127.0.0.1', }, }, + fields: { + someKey: ['someValue'], + '@timestamp': ['2020-04-20T21:27:45+0000'], + 'source.ip': [ip ?? '127.0.0.1'], + }, sort: [], }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts index dc8ed156d8dea..8597667f64657 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts @@ -56,7 +56,12 @@ describe('create_signals', () => { ], }, }, - + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], sort: [ { '@timestamp': { @@ -115,7 +120,12 @@ describe('create_signals', () => { ], }, }, - + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], sort: [ { '@timestamp': { @@ -175,7 +185,12 @@ describe('create_signals', () => { ], }, }, - + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], sort: [ { '@timestamp': { @@ -236,7 +251,12 @@ describe('create_signals', () => { ], }, }, - + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], sort: [ { '@timestamp': { @@ -296,7 +316,12 @@ describe('create_signals', () => { ], }, }, - + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], sort: [ { '@timestamp': { @@ -358,6 +383,12 @@ describe('create_signals', () => { ], }, }, + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], aggregations: { tags: { terms: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts index dde284ed3beab..f8fd4ed30d6ee 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts @@ -89,6 +89,12 @@ export const buildEventsSearchQuery = ({ ], }, }, + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], ...(aggregations ? { aggregations } : {}), sort: [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts index f3da37c198ac2..713178345361d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts @@ -67,7 +67,7 @@ describe('transformThresholdResultsToEcs', () => { _id, _index: 'test', _source: { - '@timestamp': '2020-04-20T21:27:45+0000', + '@timestamp': ['2020-04-20T21:27:45+0000'], threshold_result: { count: 1, value: '127.0.0.1', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index e0494c2e92b1c..dd9e1e97a2b73 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -75,7 +75,7 @@ const getTransformedHits = ( } const source = { - '@timestamp': get(timestampOverride ?? '@timestamp', hit._source), + '@timestamp': get(timestampOverride ?? '@timestamp', hit.fields), threshold_result: { count: totalResults, value: ruleId, @@ -104,10 +104,10 @@ const getTransformedHits = ( } const source = { - '@timestamp': get(timestampOverride ?? '@timestamp', hit._source), + '@timestamp': get(timestampOverride ?? '@timestamp', hit.fields), threshold_result: { count: docCount, - value: get(threshold.field, hit._source), + value: key, }, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.test.ts index 6f744de469d5c..aac0f47c28295 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.test.ts @@ -120,7 +120,7 @@ describe('filterEventsAgainstList', () => { exceptionItem, buildRuleMessage, }); - expect([...matchedSet]).toEqual([JSON.stringify('1.1.1.1')]); + expect([...matchedSet]).toEqual([JSON.stringify(['1.1.1.1'])]); }); test('it returns two matched sets as a JSON.stringify() set from the "events"', async () => { @@ -133,7 +133,7 @@ describe('filterEventsAgainstList', () => { exceptionItem, buildRuleMessage, }); - expect([...matchedSet]).toEqual([JSON.stringify('1.1.1.1'), JSON.stringify('2.2.2.2')]); + expect([...matchedSet]).toEqual([JSON.stringify(['1.1.1.1']), JSON.stringify(['2.2.2.2'])]); }); test('it returns an array as a set as a JSON.stringify() array from the "events"', async () => { @@ -282,7 +282,7 @@ describe('filterEventsAgainstList', () => { exceptionItem, buildRuleMessage, }); - expect([...matchedSet1]).toEqual([JSON.stringify('1.1.1.1'), JSON.stringify('2.2.2.2')]); - expect([...matchedSet2]).toEqual([JSON.stringify('3.3.3.3'), JSON.stringify('5.5.5.5')]); + expect([...matchedSet1]).toEqual([JSON.stringify(['1.1.1.1']), JSON.stringify(['2.2.2.2'])]); + expect([...matchedSet2]).toEqual([JSON.stringify(['3.3.3.3']), JSON.stringify(['5.5.5.5'])]); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.test.ts index aff372dc5bf3b..aae4a7aae2b9e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.test.ts @@ -62,9 +62,9 @@ describe('createSetToFilterAgainst', () => { expect(listClient.searchListItemByValues).toHaveBeenCalledWith({ listId: 'list-123', type: 'ip', - value: ['1.1.1.1'], + value: [['1.1.1.1']], }); - expect([...field]).toEqual([JSON.stringify('1.1.1.1')]); + expect([...field]).toEqual([JSON.stringify(['1.1.1.1'])]); }); test('it returns 2 fields if the list returns 2 items', async () => { @@ -81,9 +81,9 @@ describe('createSetToFilterAgainst', () => { expect(listClient.searchListItemByValues).toHaveBeenCalledWith({ listId: 'list-123', type: 'ip', - value: ['1.1.1.1', '2.2.2.2'], + value: [['1.1.1.1'], ['2.2.2.2']], }); - expect([...field]).toEqual([JSON.stringify('1.1.1.1'), JSON.stringify('2.2.2.2')]); + expect([...field]).toEqual([JSON.stringify(['1.1.1.1']), JSON.stringify(['2.2.2.2'])]); }); test('it returns 0 fields if the field does not match up to a valid field within the event', async () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.ts index c9f98e1b1e4e3..d400cc901a3ed 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { get } from 'lodash/fp'; import { CreateSetToFilterAgainstOptions } from './types'; /** @@ -31,7 +30,7 @@ export const createSetToFilterAgainst = async ({ buildRuleMessage, }: CreateSetToFilterAgainstOptions): Promise> => { const valuesFromSearchResultField = events.reduce((acc, searchResultItem) => { - const valueField = get(field, searchResultItem._source); + const valueField = searchResultItem.fields ? searchResultItem.fields[field] : undefined; if (valueField != null) { acc.add(valueField); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.test.ts index 092a684756ea3..eb5c69e8abfe8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.test.ts @@ -40,7 +40,7 @@ describe('filterEvents', () => { { field: 'source.ip', operator: 'included', - matchedSet: new Set([JSON.stringify('1.1.1.1')]), + matchedSet: new Set([JSON.stringify(['1.1.1.1'])]), }, ]; const field = filterEvents({ @@ -56,7 +56,7 @@ describe('filterEvents', () => { { field: 'source.ip', operator: 'excluded', - matchedSet: new Set([JSON.stringify('1.1.1.1')]), + matchedSet: new Set([JSON.stringify(['1.1.1.1'])]), }, ]; const field = filterEvents({ @@ -72,7 +72,7 @@ describe('filterEvents', () => { { field: 'madeup.nonexistent', // field does not exist operator: 'included', - matchedSet: new Set([JSON.stringify('1.1.1.1')]), + matchedSet: new Set([JSON.stringify(['1.1.1.1'])]), }, ]; const field = filterEvents({ @@ -88,12 +88,12 @@ describe('filterEvents', () => { { field: 'source.ip', operator: 'included', - matchedSet: new Set([JSON.stringify('1.1.1.1')]), + matchedSet: new Set([JSON.stringify(['1.1.1.1'])]), }, { field: 'source.ip', operator: 'excluded', - matchedSet: new Set([JSON.stringify('1.1.1.1')]), + matchedSet: new Set([JSON.stringify(['1.1.1.1'])]), }, ]; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.ts index 316ef5eb74f41..421ed91278f4c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { get } from 'lodash/fp'; import { SearchResponse } from '../../../types'; import { FilterEventsOptions } from './types'; @@ -22,13 +21,17 @@ export const filterEvents = ({ return events.filter((item) => { return fieldAndSetTuples .map((tuple) => { - const eventItem = get(tuple.field, item._source); - if (eventItem == null) { - return true; - } else if (tuple.operator === 'included') { + const eventItem = item.fields ? item.fields[tuple.field] : undefined; + if (tuple.operator === 'included') { + if (eventItem == null) { + return true; + } // only create a signal if the event is not in the value list return !tuple.matchedSet.has(JSON.stringify(eventItem)); } else if (tuple.operator === 'excluded') { + if (eventItem == null) { + return false; + } // only create a signal if the event is in the value list return tuple.matchedSet.has(JSON.stringify(eventItem)); } else { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events_against_list.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events_against_list.test.ts index c1ba8eabf7110..5b2f3426cd8aa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events_against_list.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events_against_list.test.ts @@ -162,12 +162,12 @@ describe('filterEventsAgainstList', () => { // this call represents an exception list with a value list containing ['2.2.2.2', '4.4.4.4'] (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, - { ...getSearchListItemResponseMock(), value: '4.4.4.4' }, + { ...getSearchListItemResponseMock(), value: ['2.2.2.2'] }, + { ...getSearchListItemResponseMock(), value: ['4.4.4.4'] }, ]); // this call represents an exception list with a value list containing ['6.6.6.6'] (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getSearchListItemResponseMock(), value: '6.6.6.6' }, + { ...getSearchListItemResponseMock(), value: ['6.6.6.6'] }, ]); const res = await filterEventsAgainstList({ @@ -224,11 +224,11 @@ describe('filterEventsAgainstList', () => { // this call represents an exception list with a value list containing ['2.2.2.2', '4.4.4.4'] (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, + { ...getSearchListItemResponseMock(), value: ['2.2.2.2'] }, ]); // this call represents an exception list with a value list containing ['6.6.6.6'] (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getSearchListItemResponseMock(), value: '6.6.6.6' }, + { ...getSearchListItemResponseMock(), value: ['6.6.6.6'] }, ]); const res = await filterEventsAgainstList({ @@ -283,11 +283,11 @@ describe('filterEventsAgainstList', () => { // this call represents an exception list with a value list containing ['2.2.2.2'] (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, + { ...getSearchListItemResponseMock(), value: ['2.2.2.2'] }, ]); // this call represents an exception list with a value list containing ['4.4.4.4'] (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getSearchListItemResponseMock(), value: '4.4.4.4' }, + { ...getSearchListItemResponseMock(), value: ['4.4.4.4'] }, ]); const res = await filterEventsAgainstList({ @@ -365,7 +365,7 @@ describe('filterEventsAgainstList', () => { // this call represents an exception list with a value list containing ['2.2.2.2', '4.4.4.4'] (listClient.searchListItemByValues as jest.Mock).mockResolvedValue([ - { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, + { ...getSearchListItemResponseMock(), value: ['2.2.2.2'] }, ]); const res = await filterEventsAgainstList({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts index 7d32ac6873eb2..6144f1f4b3823 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts @@ -69,6 +69,12 @@ export const findThresholdSignals = async ({ }, }, ], + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], size: 1, }, }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index 418d30711169e..b506a2463a311 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -310,9 +310,9 @@ describe('searchAfterAndBulkCreate', () => { test('should return success when all search results are in the allowlist and with sortId present', async () => { const searchListItems: SearchListItemArraySchema = [ - { ...getSearchListItemResponseMock(), value: '1.1.1.1' }, - { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, - { ...getSearchListItemResponseMock(), value: '3.3.3.3' }, + { ...getSearchListItemResponseMock(), value: ['1.1.1.1'] }, + { ...getSearchListItemResponseMock(), value: ['2.2.2.2'] }, + { ...getSearchListItemResponseMock(), value: ['3.3.3.3'] }, ]; listClient.searchListItemByValues = jest.fn().mockResolvedValue(searchListItems); const sampleParams = sampleRuleAlertParams(30); @@ -374,10 +374,10 @@ describe('searchAfterAndBulkCreate', () => { test('should return success when all search results are in the allowlist and no sortId present', async () => { const searchListItems: SearchListItemArraySchema = [ - { ...getSearchListItemResponseMock(), value: '1.1.1.1' }, - { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, - { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, - { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, + { ...getSearchListItemResponseMock(), value: ['1.1.1.1'] }, + { ...getSearchListItemResponseMock(), value: ['2.2.2.2'] }, + { ...getSearchListItemResponseMock(), value: ['2.2.2.2'] }, + { ...getSearchListItemResponseMock(), value: ['2.2.2.2'] }, ]; listClient.searchListItemByValues = jest.fn().mockResolvedValue(searchListItems); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts index 942db1e3b1aaa..19aba907f0c84 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts @@ -316,19 +316,9 @@ describe('singleBulkCreate', () => { }); test('filter duplicate rules will return back search responses if they do not have a signal and will NOT filter the source out', () => { - const ancestors = sampleDocWithAncestors(); - ancestors.hits.hits[0]._source = { '@timestamp': '2020-04-20T21:27:45+0000' }; + const ancestors = sampleDocSearchResultsNoSortId(); const filtered = filterDuplicateRules('04128c15-0d1b-4716-a4c5-46997ac7f3bd', ancestors); - expect(filtered).toEqual([ - { - _index: 'myFakeSignalIndex', - _type: 'doc', - _score: 100, - _version: 1, - _id: 'e1e08ddc-5e37-49ff-a258-5393aa44435a', - _source: { '@timestamp': '2020-04-20T21:27:45+0000' }, - }, - ]); + expect(filtered).toEqual(ancestors.hits.hits); }); test('filter duplicate rules does not attempt filters when the signal is not an event type of signal but rather a "clash" from the source index having its own numeric signal type', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts index a88d9061f7a1f..12865e4dd47a9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts @@ -81,6 +81,7 @@ export const getThreatListSearchResponseMock = (): SearchResponse ({ }, }); +export const getThreatListItemFieldsMock = () => ({ + '@timestamp': ['2020-09-09T21:59:13Z'], + 'host.name': ['host-1'], + 'host.ip': ['192.168.0.0.1'], + 'source.ip': ['127.0.0.1'], + 'source.port': [1], + 'destination.ip': ['127.0.0.1'], + 'destination.port': [1], +}); + export const getFilterThreatMapping = (): ThreatMapping => [ { entries: [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts index 792fa889e395d..7a9c4b43b8f7a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts @@ -133,10 +133,16 @@ describe('build_threat_mapping_filter', () => { }, ], threatListItem: { - '@timestamp': '2020-09-09T21:59:13Z', - host: { - name: 'host-1', - // since ip is missing this entire AND clause should be dropped + _source: { + '@timestamp': '2020-09-09T21:59:13Z', + host: { + name: 'host-1', + // since ip is missing this entire AND clause should be dropped + }, + }, + fields: { + '@timestamp': ['2020-09-09T21:59:13Z'], + 'host.name': ['host-1'], }, }, }); @@ -177,6 +183,10 @@ describe('build_threat_mapping_filter', () => { name: 'host-1', }, }, + fields: { + '@timestamp': ['2020-09-09T21:59:13Z'], + 'host.name': ['host-1'], + }, }, }); expect(item).toEqual([ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts index 180895877bdd2..cab01a602b8a9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts @@ -55,7 +55,8 @@ export const filterThreatMapping = ({ threatMapping .map((threatMap) => { const atLeastOneItemMissingInThreatList = threatMap.entries.some((entry) => { - return get(entry.value, threatListItem._source) == null; + const itemValue = get(entry.value, threatListItem.fields); + return itemValue == null || itemValue.length !== 1; }); if (atLeastOneItemMissingInThreatList) { return { ...threatMap, entries: [] }; @@ -70,15 +71,15 @@ export const createInnerAndClauses = ({ threatListItem, }: CreateInnerAndClausesOptions): BooleanFilter[] => { return threatMappingEntries.reduce((accum, threatMappingEntry) => { - const value = get(threatMappingEntry.value, threatListItem._source); - if (value != null) { + const value = get(threatMappingEntry.value, threatListItem.fields); + if (value != null && value.length === 1) { // These values could be potentially 10k+ large so mutating the array intentionally accum.push({ bool: { should: [ { match: { - [threatMappingEntry.field]: value, + [threatMappingEntry.field]: value[0], }, }, ], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts index c646fee81f1b1..92d4e5cf8a93b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts @@ -55,6 +55,12 @@ export const getThreatList = async ({ const response: SearchResponse = await callCluster('search', { body: { query: queryFilter, + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], search_after: searchAfter, sort: getSortWithTieBreaker({ sortField, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index 5444f08474053..75bd9f593a6ac 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -1166,6 +1166,9 @@ describe('utils', () => { test('It will not set an invalid date time stamp from a non-existent @timestamp when the index is not 100% ECS compliant', () => { const searchResult = sampleDocSearchResultsNoSortId(); (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = undefined; + if (searchResult.hits.hits[0].fields != null) { + (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = undefined; + } const { lastLookBackDate } = createSearchAfterReturnTypeFromResponse({ searchResult, timestampOverride: undefined, @@ -1176,6 +1179,9 @@ describe('utils', () => { test('It will not set an invalid date time stamp from a null @timestamp when the index is not 100% ECS compliant', () => { const searchResult = sampleDocSearchResultsNoSortId(); (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = null; + if (searchResult.hits.hits[0].fields != null) { + (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = null; + } const { lastLookBackDate } = createSearchAfterReturnTypeFromResponse({ searchResult, timestampOverride: undefined, @@ -1186,6 +1192,9 @@ describe('utils', () => { test('It will not set an invalid date time stamp from an invalid @timestamp string', () => { const searchResult = sampleDocSearchResultsNoSortId(); (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = 'invalid'; + if (searchResult.hits.hits[0].fields != null) { + (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = ['invalid']; + } const { lastLookBackDate } = createSearchAfterReturnTypeFromResponse({ searchResult, timestampOverride: undefined, @@ -1198,6 +1207,9 @@ describe('utils', () => { test('It returns undefined if the search result contains a null timestamp', () => { const searchResult = sampleDocSearchResultsNoSortId(); (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = null; + if (searchResult.hits.hits[0].fields != null) { + (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = null; + } const date = lastValidDate({ searchResult, timestampOverride: undefined }); expect(date).toEqual(undefined); }); @@ -1205,6 +1217,9 @@ describe('utils', () => { test('It returns undefined if the search result contains a undefined timestamp', () => { const searchResult = sampleDocSearchResultsNoSortId(); (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = undefined; + if (searchResult.hits.hits[0].fields != null) { + (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = undefined; + } const date = lastValidDate({ searchResult, timestampOverride: undefined }); expect(date).toEqual(undefined); }); @@ -1212,13 +1227,9 @@ describe('utils', () => { test('It returns undefined if the search result contains an invalid string value', () => { const searchResult = sampleDocSearchResultsNoSortId(); (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = 'invalid value'; - const date = lastValidDate({ searchResult, timestampOverride: undefined }); - expect(date).toEqual(undefined); - }); - - test('It returns correct date time stamp if the search result contains an invalid string value', () => { - const searchResult = sampleDocSearchResultsNoSortId(); - (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = 'invalid value'; + if (searchResult.hits.hits[0].fields != null) { + (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = ['invalid value']; + } const date = lastValidDate({ searchResult, timestampOverride: undefined }); expect(date).toEqual(undefined); }); diff --git a/x-pack/plugins/security_solution/server/lib/machine_learning/index.ts b/x-pack/plugins/security_solution/server/lib/machine_learning/index.ts index a6f4c2086e47b..962c44174d891 100644 --- a/x-pack/plugins/security_solution/server/lib/machine_learning/index.ts +++ b/x-pack/plugins/security_solution/server/lib/machine_learning/index.ts @@ -60,6 +60,12 @@ export const getAnomalies = async ( })?.query, }, }, + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], sort: [{ record_score: { order: 'desc' } }], }, },