Skip to content

Commit

Permalink
Deduplication of entries and items before sending to endpoint (#71297)
Browse files Browse the repository at this point in the history 
* Deduplication of entries and items before sending to endpoint

* Renaming
  • Loading branch information
Alex Kahan authored Jul 9, 2020
1 parent d570ab1 commit c9e8650
Show file tree
Hide file tree
Showing 2 changed files with 153 additions and 5 deletions.
139 changes: 137 additions & 2 deletions x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,139 @@ describe('buildEventTypeSignal', () => {
});
});

test('it should deduplicate exception entries', async () => {
const testEntries: EntriesArray = [
{ field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' },
{ field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' },
{ field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' },
{ field: 'server.ip', operator: 'included', type: 'match', value: '192.168.1.1' },
{
field: 'host.hostname.text',
operator: 'included',
type: 'match_any',
value: ['estc', 'kibana'],
},
];

const expectedEndpointExceptions = {
type: 'simple',
entries: [
{
field: 'server.domain',
operator: 'included',
type: 'exact_caseless',
value: 'DOMAIN',
},
{
field: 'server.ip',
operator: 'included',
type: 'exact_cased',
value: '192.168.1.1',
},
{
field: 'host.hostname',
operator: 'included',
type: 'exact_caseless_any',
value: ['estc', 'kibana'],
},
],
};

const first = getFoundExceptionListItemSchemaMock();
first.data[0].entries = testEntries;
mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);

const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
expect(resp).toEqual({
entries: [expectedEndpointExceptions],
});
});

test('it should not deduplicate exception entries across nested boundaries', async () => {
const testEntries: EntriesArray = [
{
entries: [
{ field: 'nested.field', operator: 'included', type: 'match', value: 'some value' },
],
field: 'some.parentField',
type: 'nested',
},
// Same as above but not inside the nest
{ field: 'nested.field', operator: 'included', type: 'match', value: 'some value' },
];

const expectedEndpointExceptions = {
type: 'simple',
entries: [
{
entries: [
{
field: 'nested.field',
operator: 'included',
type: 'exact_cased',
value: 'some value',
},
],
field: 'some.parentField',
type: 'nested',
},
{
field: 'nested.field',
operator: 'included',
type: 'exact_cased',
value: 'some value',
},
],
};

const first = getFoundExceptionListItemSchemaMock();
first.data[0].entries = testEntries;
mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);

const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
expect(resp).toEqual({
entries: [expectedEndpointExceptions],
});
});

test('it should deduplicate exception items', async () => {
const testEntries: EntriesArray = [
{ field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' },
{ field: 'server.ip', operator: 'included', type: 'match', value: '192.168.1.1' },
];

const expectedEndpointExceptions = {
type: 'simple',
entries: [
{
field: 'server.domain',
operator: 'included',
type: 'exact_caseless',
value: 'DOMAIN',
},
{
field: 'server.ip',
operator: 'included',
type: 'exact_cased',
value: '192.168.1.1',
},
],
};

const first = getFoundExceptionListItemSchemaMock();
first.data[0].entries = testEntries;

// Create a second exception item with the same entries
first.data[1] = getExceptionListItemSchemaMock();
first.data[1].entries = testEntries;
mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);

const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
expect(resp).toEqual({
entries: [expectedEndpointExceptions],
});
});

test('it should ignore unsupported entries', async () => {
// Lists and exists are not supported by the Endpoint
const testEntries: EntriesArray = [
Expand Down Expand Up @@ -178,8 +311,9 @@ describe('buildEventTypeSignal', () => {
});

test('it should convert the exception lists response to the proper endpoint format while paging', async () => {
// The first call returns one exception
// The first call returns two exceptions
const first = getFoundExceptionListItemSchemaMock();
first.data.push(getExceptionListItemSchemaMock());

// The second call returns two exceptions
const second = getFoundExceptionListItemSchemaMock();
Expand All @@ -194,7 +328,8 @@ describe('buildEventTypeSignal', () => {
.mockReturnValueOnce(second)
.mockReturnValueOnce(third);
const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
expect(resp.entries.length).toEqual(3);
// Expect 2 exceptions, the first two calls returned the same exception list items
expect(resp.entries.length).toEqual(2);
});

test('it should handle no exceptions', async () => {
Expand Down
19 changes: 16 additions & 3 deletions x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,10 +97,18 @@ export function translateToEndpointExceptions(
exc: FoundExceptionListItemSchema,
schemaVersion: string
): TranslatedExceptionListItem[] {
const entrySet = new Set();
const entriesFiltered: TranslatedExceptionListItem[] = [];
if (schemaVersion === 'v1') {
return exc.data.map((item) => {
return translateItem(schemaVersion, item);
exc.data.forEach((entry) => {
const translatedItem = translateItem(schemaVersion, entry);
const entryHash = createHash('sha256').update(JSON.stringify(translatedItem)).digest('hex');
if (!entrySet.has(entryHash)) {
entriesFiltered.push(translatedItem);
entrySet.add(entryHash);
}
});
return entriesFiltered;
} else {
throw new Error('unsupported schemaVersion');
}
Expand All @@ -124,12 +132,17 @@ function translateItem(
schemaVersion: string,
item: ExceptionListItemSchema
): TranslatedExceptionListItem {
const itemSet = new Set();
return {
type: item.type,
entries: item.entries.reduce((translatedEntries: TranslatedEntry[], entry) => {
const translatedEntry = translateEntry(schemaVersion, entry);
if (translatedEntry !== undefined && translatedEntryType.is(translatedEntry)) {
translatedEntries.push(translatedEntry);
const itemHash = createHash('sha256').update(JSON.stringify(translatedEntry)).digest('hex');
if (!itemSet.has(itemHash)) {
translatedEntries.push(translatedEntry);
itemSet.add(itemHash);
}
}
return translatedEntries;
}, []),
Expand Down

0 comments on commit c9e8650

Please sign in to comment.