-
Notifications
You must be signed in to change notification settings - Fork 8.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[SIEM] [Detection Engine] Incorporate large lists to rule execution. (#…
…65372) * introduce lists plugin for use by executor * adds getListClient function on setup * refactors searchAfterBulkCreate to integrate with the lists plugin so we only generate signals from events not in the list * fixes type check issues * fixes unit tests, adds field and other parameters for using lists in executor. * cleaning up types and exports, updates to match new contracts with lists client from master * prior to this commit the refactored while loop was doing more search after loops than it needed to and this fixes two bugs in the list filter function where we were returning the wrong count, and we were not accessing the right field on the event * exception lists are optional * use exceptions list format, this works with given sample query in scripts * updates tests and fixes type issues * updates README doc in detection engine with example for rule with list exception * adds one test and removes commented out code * fix sample rule json from 30s to 5m * fix sample rule json from 30s to 5m * remove unused import * more cleanup * e2e test for prepackaged rules was failing because lists was undefined in the siem plugin and was preventing the registration of the rule alert type. I removed this but once lists is ready for prime time we should consider adding the null check back * can't reuse the same env var since the tests are setting the ELASTIC_XPACK_SIEM_LISTS_FEATURE env var to true without enabling the lists plugin * fixes from pr review, still needs more TLC * exports listspluginsetup type from top-level in lists plugin, fixes logic for empty exceptions list, updates types * utilize type.is to remove as casting, also do null checks and throw an error when exceptionItem is malformed. This will change in the very near future once the new json format for exception lists is incorporated * fix type issues after merging master into branch * update mock * remove bad null check for ml plugin before registering rule alert type in siem plugin * prettier linting * adds test for filter events with list * pr comments * adds logic for included vs excluded and updates tests * update test cases for search after bulk create to default to included for exception lists * filter out non-list exception items from the loop
- Loading branch information
Showing
19 changed files
with
770 additions
and
288 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
24 changes: 24 additions & 0 deletions
24
.../siem/server/lib/detection_engine/scripts/rules/queries/lists/query_with_list_plugin.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| { | ||
| "name": "Query with a list", | ||
| "description": "Query with a list only generate signals if source.ip is not in list", | ||
| "rule_id": "query-with-list", | ||
| "risk_score": 2, | ||
| "severity": "high", | ||
| "type": "query", | ||
| "query": "host.name: *", | ||
| "interval": "30s", | ||
| "language": "kuery", | ||
| "exceptions_list": [ | ||
| { | ||
| "field": "source.ip", | ||
| "values_operator": "excluded", | ||
| "values_type": "list", | ||
| "values": [ | ||
| { | ||
| "id": "ci-badguys.txt", | ||
| "name": "ip" | ||
| } | ||
| ] | ||
| } | ||
| ] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
242 changes: 242 additions & 0 deletions
242
x-pack/plugins/siem/server/lib/detection_engine/signals/filter_events_with_list.test.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,242 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License; | ||
| * you may not use this file except in compliance with the Elastic License. | ||
| */ | ||
|
|
||
| import uuid from 'uuid'; | ||
| import { filterEventsAgainstList } from './filter_events_with_list'; | ||
| import { mockLogger, repeatedSearchResultsWithSortId } from './__mocks__/es_results'; | ||
|
|
||
| import { ListClient } from '../../../../../lists/server'; | ||
|
|
||
| const someGuids = Array.from({ length: 13 }).map((x) => uuid.v4()); | ||
|
|
||
| describe('filterEventsAgainstList', () => { | ||
| it('should respond with eventSearchResult if exceptionList is empty', async () => { | ||
| const res = await filterEventsAgainstList({ | ||
| logger: mockLogger, | ||
| listClient: ({ | ||
| getListItemByValues: async () => [], | ||
| } as unknown) as ListClient, | ||
| exceptionsList: undefined, | ||
| eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [ | ||
| '1.1.1.1', | ||
| '2.2.2.2', | ||
| '3.3.3.3', | ||
| '7.7.7.7', | ||
| ]), | ||
| }); | ||
| expect(res.hits.hits.length).toEqual(4); | ||
| }); | ||
|
|
||
| it('should throw an error if malformed exception list present', async () => { | ||
| let message = ''; | ||
| try { | ||
| await filterEventsAgainstList({ | ||
| logger: mockLogger, | ||
| listClient: ({ | ||
| getListItemByValues: async () => [], | ||
| } as unknown) as ListClient, | ||
| exceptionsList: [ | ||
| { | ||
| field: 'source.ip', | ||
| values_operator: 'excluded', | ||
| values_type: 'list', | ||
| values: undefined, | ||
| }, | ||
| ], | ||
| eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [ | ||
| '1.1.1.1', | ||
| '2.2.2.2', | ||
| '3.3.3.3', | ||
| '7.7.7.7', | ||
| ]), | ||
| }); | ||
| } catch (exc) { | ||
| message = exc.message; | ||
| } | ||
| expect(message).toEqual( | ||
| 'Failed to query lists index. Reason: Malformed exception list provided' | ||
| ); | ||
| }); | ||
|
|
||
| it('should throw an error if unsupported exception type', async () => { | ||
| let message = ''; | ||
| try { | ||
| await filterEventsAgainstList({ | ||
| logger: mockLogger, | ||
| listClient: ({ | ||
| getListItemByValues: async () => [], | ||
| } as unknown) as ListClient, | ||
| exceptionsList: [ | ||
| { | ||
| field: 'source.ip', | ||
| values_operator: 'excluded', | ||
| values_type: 'list', | ||
| values: [ | ||
| { | ||
| id: 'ci-badguys.txt', | ||
| name: 'unsupportedListPluginType', | ||
| }, | ||
| ], | ||
| }, | ||
| ], | ||
| eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [ | ||
| '1.1.1.1', | ||
| '2.2.2.2', | ||
| '3.3.3.3', | ||
| '7.7.7.7', | ||
| ]), | ||
| }); | ||
| } catch (exc) { | ||
| message = exc.message; | ||
| } | ||
| expect(message).toEqual( | ||
| 'Failed to query lists index. Reason: Unsupported list type used, please use one of ip,keyword' | ||
| ); | ||
| }); | ||
|
|
||
| describe('operator_type is includes', () => { | ||
| it('should respond with same list if no items match value list', async () => { | ||
| const res = await filterEventsAgainstList({ | ||
| logger: mockLogger, | ||
| listClient: ({ | ||
| getListItemByValues: async () => [], | ||
| } as unknown) as ListClient, | ||
| exceptionsList: [ | ||
| { | ||
| field: 'source.ip', | ||
| values_operator: 'included', | ||
| values_type: 'list', | ||
| values: [ | ||
| { | ||
| id: 'ci-badguys.txt', | ||
| name: 'ip', | ||
| }, | ||
| ], | ||
| }, | ||
| ], | ||
| eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3)), | ||
| }); | ||
| expect(res.hits.hits.length).toEqual(4); | ||
| }); | ||
| it('should respond with less items in the list if some values match', async () => { | ||
| let outerType = ''; | ||
| let outerListId = ''; | ||
| const res = await filterEventsAgainstList({ | ||
| logger: mockLogger, | ||
| listClient: ({ | ||
| getListItemByValues: async ({ | ||
| value, | ||
| type, | ||
| listId, | ||
| }: { | ||
| type: string; | ||
| listId: string; | ||
| value: string[]; | ||
| }) => { | ||
| outerType = type; | ||
| outerListId = listId; | ||
| return value.slice(0, 2).map((item) => ({ | ||
| value: item, | ||
| })); | ||
| }, | ||
| } as unknown) as ListClient, | ||
| exceptionsList: [ | ||
| { | ||
| field: 'source.ip', | ||
| values_operator: 'included', | ||
| values_type: 'list', | ||
| values: [ | ||
| { | ||
| id: 'ci-badguys.txt', | ||
| name: 'ip', | ||
| }, | ||
| ], | ||
| }, | ||
| ], | ||
| eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [ | ||
| '1.1.1.1', | ||
| '2.2.2.2', | ||
| '3.3.3.3', | ||
| '7.7.7.7', | ||
| ]), | ||
| }); | ||
| expect(outerType).toEqual('ip'); | ||
| expect(outerListId).toEqual('ci-badguys.txt'); | ||
| expect(res.hits.hits.length).toEqual(2); | ||
| }); | ||
| }); | ||
| describe('operator type is excluded', () => { | ||
| it('should respond with empty list if no items match value list', async () => { | ||
| const res = await filterEventsAgainstList({ | ||
| logger: mockLogger, | ||
| listClient: ({ | ||
| getListItemByValues: async () => [], | ||
| } as unknown) as ListClient, | ||
| exceptionsList: [ | ||
| { | ||
| field: 'source.ip', | ||
| values_operator: 'excluded', | ||
| values_type: 'list', | ||
| values: [ | ||
| { | ||
| id: 'ci-badguys.txt', | ||
| name: 'ip', | ||
| }, | ||
| ], | ||
| }, | ||
| ], | ||
| eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3)), | ||
| }); | ||
| expect(res.hits.hits.length).toEqual(0); | ||
| }); | ||
| it('should respond with less items in the list if some values match', async () => { | ||
| let outerType = ''; | ||
| let outerListId = ''; | ||
| const res = await filterEventsAgainstList({ | ||
| logger: mockLogger, | ||
| listClient: ({ | ||
| getListItemByValues: async ({ | ||
| value, | ||
| type, | ||
| listId, | ||
| }: { | ||
| type: string; | ||
| listId: string; | ||
| value: string[]; | ||
| }) => { | ||
| outerType = type; | ||
| outerListId = listId; | ||
| return value.slice(0, 2).map((item) => ({ | ||
| value: item, | ||
| })); | ||
| }, | ||
| } as unknown) as ListClient, | ||
| exceptionsList: [ | ||
| { | ||
| field: 'source.ip', | ||
| values_operator: 'excluded', | ||
| values_type: 'list', | ||
| values: [ | ||
| { | ||
| id: 'ci-badguys.txt', | ||
| name: 'ip', | ||
| }, | ||
| ], | ||
| }, | ||
| ], | ||
| eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [ | ||
| '1.1.1.1', | ||
| '2.2.2.2', | ||
| '3.3.3.3', | ||
| '7.7.7.7', | ||
| ]), | ||
| }); | ||
| expect(outerType).toEqual('ip'); | ||
| expect(outerListId).toEqual('ci-badguys.txt'); | ||
| expect(res.hits.hits.length).toEqual(2); | ||
| }); | ||
| }); | ||
| }); |
Oops, something went wrong.