From 20a0d82e372cfda14a9e2d58eb97ff4d838e4075 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 1 Mar 2021 17:18:42 -0600 Subject: [PATCH] [Security Solution][Detecttions] Indicator enrichment tweaks (#92989) (#93121) * Update copy of rule config * Encode threat index as part of our named query * Add index to named query, and enrich both id and index We still need mappings and to fix integration tests, but this generates the correct data. * Update integration tests with new enrichment fields Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../rules/step_about_rule/schema.tsx | 4 +- .../build_threat_mapping_filter.ts | 1 + .../enrich_signal_threat_matches.mock.ts | 1 + .../enrich_signal_threat_matches.test.ts | 94 +++++++++++++++++-- .../enrich_signal_threat_matches.ts | 2 +- .../signals/threat_mapping/types.ts | 1 + .../signals/threat_mapping/utils.test.ts | 5 +- .../signals/threat_mapping/utils.ts | 11 ++- .../tests/create_threat_matching.ts | 46 ++++++--- 9 files changed, 136 insertions(+), 29 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/schema.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/schema.tsx index 07012af17e7343..3467b34d47135c 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/schema.tsx @@ -198,14 +198,14 @@ export const schema: FormSchema = { label: i18n.translate( 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThreatIndicatorPathLabel', { - defaultMessage: 'Threat Indicator Path', + defaultMessage: 'Indicator prefix override', } ), helpText: i18n.translate( 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThreatIndicatorPathHelpText', { defaultMessage: - 'Specify the document path containing your threat indicator fields. Used for enrichment of indicator match alerts.', + 'Specify the document prefix containing your indicator fields. Used for enrichment of indicator match alerts.', } ), labelAppend: OptionalFieldLabel, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts index 0a2789ec2f1d0d..18204bb678a477 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts @@ -84,6 +84,7 @@ export const createInnerAndClauses = ({ query: value[0], _name: encodeThreatMatchNamedQuery({ id: threatListItem._id, + index: threatListItem._index, field: threatMappingEntry.field, value: threatMappingEntry.value, }), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.mock.ts index a3ff932e97886e..f66dc461c6d408 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.mock.ts @@ -12,6 +12,7 @@ export const getNamedQueryMock = ( overrides: Partial = {} ): ThreatMatchNamedQuery => ({ id: 'id', + index: 'index', field: 'field', value: 'value', ...overrides, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts index b77e8228e72d89..7b3ca099cc93c7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts @@ -88,7 +88,12 @@ describe('buildMatchedIndicator', () => { }), ]; queries = [ - getNamedQueryMock({ id: '123', field: 'event.field', value: 'threat.indicator.domain' }), + getNamedQueryMock({ + id: '123', + index: 'threat-index', + field: 'event.field', + value: 'threat.indicator.domain', + }), ]; }); @@ -112,6 +117,26 @@ describe('buildMatchedIndicator', () => { expect(get(indicator, 'matched.atomic')).toEqual('domain_1'); }); + it('returns the _id of the matched indicator as matched.id', () => { + const [indicator] = buildMatchedIndicator({ + queries, + threats, + indicatorPath, + }); + + expect(get(indicator, 'matched.id')).toEqual('123'); + }); + + it('returns the _index of the matched indicator as matched.index', () => { + const [indicator] = buildMatchedIndicator({ + queries, + threats, + indicatorPath, + }); + + expect(get(indicator, 'matched.index')).toEqual('threat-index'); + }); + it('returns the field of the matched indicator as matched.field', () => { const [indicator] = buildMatchedIndicator({ queries, @@ -173,6 +198,8 @@ describe('buildMatchedIndicator', () => { domain: 'domain_1', matched: { atomic: 'domain_1', + id: '123', + index: 'threat-index', field: 'event.field', type: 'type_1', }, @@ -211,6 +238,8 @@ describe('buildMatchedIndicator', () => { indicator_field: 'indicator_field_1', matched: { atomic: 'domain_1', + id: '123', + index: 'threat-index', field: 'event.field', type: 'indicator_type', }, @@ -237,6 +266,8 @@ describe('buildMatchedIndicator', () => { { matched: { atomic: undefined, + id: '123', + index: 'threat-index', field: 'event.field', type: undefined, }, @@ -262,6 +293,8 @@ describe('buildMatchedIndicator', () => { { matched: { atomic: undefined, + id: '123', + index: 'threat-index', field: 'event.field', type: undefined, }, @@ -295,6 +328,8 @@ describe('buildMatchedIndicator', () => { domain: 'foo', matched: { atomic: undefined, + id: '123', + index: 'threat-index', field: 'event.field', type: 'first', }, @@ -362,7 +397,12 @@ describe('enrichSignalThreatMatches', () => { }), ]; matchedQuery = encodeThreatMatchNamedQuery( - getNamedQueryMock({ id: '123', field: 'event.field', value: 'threat.indicator.domain' }) + getNamedQueryMock({ + id: '123', + index: 'indicator_index', + field: 'event.field', + value: 'threat.indicator.domain', + }) ); }); @@ -395,7 +435,13 @@ describe('enrichSignalThreatMatches', () => { { existing: 'indicator' }, { domain: 'domain_1', - matched: { atomic: 'domain_1', field: 'event.field', type: 'type_1' }, + matched: { + atomic: 'domain_1', + id: '123', + index: 'indicator_index', + field: 'event.field', + type: 'type_1', + }, other: 'other_1', type: 'type_1', }, @@ -418,7 +464,13 @@ describe('enrichSignalThreatMatches', () => { expect(indicators).toEqual([ { - matched: { atomic: undefined, field: 'event.field', type: undefined }, + matched: { + atomic: undefined, + id: '123', + index: 'indicator_index', + field: 'event.field', + type: undefined, + }, }, ]); }); @@ -441,7 +493,13 @@ describe('enrichSignalThreatMatches', () => { { existing: 'indicator' }, { domain: 'domain_1', - matched: { atomic: 'domain_1', field: 'event.field', type: 'type_1' }, + matched: { + atomic: 'domain_1', + id: '123', + index: 'indicator_index', + field: 'event.field', + type: 'type_1', + }, other: 'other_1', type: 'type_1', }, @@ -477,6 +535,7 @@ describe('enrichSignalThreatMatches', () => { matchedQuery = encodeThreatMatchNamedQuery( getNamedQueryMock({ id: '123', + index: 'custom_index', field: 'event.field', value: 'custom_threat.custom_indicator.domain', }) @@ -496,7 +555,13 @@ describe('enrichSignalThreatMatches', () => { expect(indicators).toEqual([ { domain: 'custom_domain', - matched: { atomic: 'custom_domain', field: 'event.field', type: 'custom_type' }, + matched: { + atomic: 'custom_domain', + id: '123', + index: 'custom_index', + field: 'event.field', + type: 'custom_type', + }, other: 'custom_other', type: 'custom_type', }, @@ -526,7 +591,12 @@ describe('enrichSignalThreatMatches', () => { _id: 'signal123', matched_queries: [ encodeThreatMatchNamedQuery( - getNamedQueryMock({ id: '456', field: 'event.other', value: 'threat.indicator.domain' }) + getNamedQueryMock({ + id: '456', + index: 'other_custom_index', + field: 'event.other', + value: 'threat.indicator.domain', + }) ), ], }); @@ -545,7 +615,13 @@ describe('enrichSignalThreatMatches', () => { expect(indicators).toEqual([ { domain: 'domain_1', - matched: { atomic: 'domain_1', field: 'event.field', type: 'type_1' }, + matched: { + atomic: 'domain_1', + id: '123', + index: 'indicator_index', + field: 'event.field', + type: 'type_1', + }, other: 'other_1', type: 'type_1', }, @@ -553,6 +629,8 @@ describe('enrichSignalThreatMatches', () => { domain: 'domain_2', matched: { atomic: 'domain_2', + id: '456', + index: 'other_custom_index', field: 'event.other', type: 'type_2', }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts index 3c8b80886cabeb..bc2be4ecaab329 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts @@ -60,7 +60,7 @@ export const buildMatchedIndicator = ({ return { ...indicator, - matched: { atomic, field: query.field, type }, + matched: { atomic, field: query.field, id: query.id, index: query.index, type }, }; }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index a022cbbdd40428..38a8840550e7b0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -202,6 +202,7 @@ export interface SortWithTieBreaker { export interface ThreatMatchNamedQuery { id: string; + index: string; field: string; value: string; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts index 897143f9ae5744..6219da93036ee8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts @@ -589,6 +589,7 @@ describe('utils', () => { it('generates a string that can be later decoded', () => { const encoded = encodeThreatMatchNamedQuery({ id: 'id', + index: 'index', field: 'field', value: 'value', }); @@ -601,6 +602,7 @@ describe('utils', () => { it('can decode an encoded query', () => { const query: ThreatMatchNamedQuery = { id: 'my_id', + index: 'index', field: 'threat.indicator.domain', value: 'host.name', }; @@ -623,6 +625,7 @@ describe('utils', () => { it('raises an error if the query is missing a value', () => { const badQuery: ThreatMatchNamedQuery = { id: 'my_id', + index: 'index', // @ts-expect-error field intentionally undefined field: undefined, value: 'host.name', @@ -630,7 +633,7 @@ describe('utils', () => { const badInput = encodeThreatMatchNamedQuery(badQuery); expect(() => decodeThreatMatchNamedQuery(badInput)).toThrowError( - 'Decoded query is invalid. Decoded value: {"id":"my_id","field":"","value":"host.name"}' + 'Decoded query is invalid. Decoded value: {"id":"my_id","index":"index","field":"","value":"host.name"}' ); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts index 72d9257798e1c9..805aca563701c6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts @@ -115,21 +115,22 @@ export const combineConcurrentResults = ( return combineResults(currentResult, maxedNewResult); }; -const separator = '___SEPARATOR___'; +const separator = '__SEP__'; export const encodeThreatMatchNamedQuery = ({ id, + index, field, value, }: ThreatMatchNamedQuery): string => { - return [id, field, value].join(separator); + return [id, index, field, value].join(separator); }; export const decodeThreatMatchNamedQuery = (encoded: string): ThreatMatchNamedQuery => { const queryValues = encoded.split(separator); - const [id, field, value] = queryValues; - const query = { id, field, value }; + const [id, index, field, value] = queryValues; + const query = { id, index, field, value }; - if (queryValues.length !== 3 || !queryValues.every(Boolean)) { + if (queryValues.length !== 4 || !queryValues.every(Boolean)) { const queryString = JSON.stringify(query); throw new Error(`Decoded query is invalid. Decoded value: ${queryString}`); } diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts index 20d2b107dc2cc1..9da98e316315db 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts @@ -307,6 +307,8 @@ export default ({ getService }: FtrProviderContext) => { first_seen: '2021-01-26T11:09:04.000Z', matched: { atomic: '159.89.119.67', + id: '978783', + index: 'filebeat-8.0.0-2021.01.26-000001', field: 'destination.ip', type: 'url', }, @@ -327,6 +329,8 @@ export default ({ getService }: FtrProviderContext) => { first_seen: '2021-01-26T11:09:04.000Z', matched: { atomic: '159.89.119.67', + id: '978783', + index: 'filebeat-8.0.0-2021.01.26-000001', field: 'destination.ip', type: 'url', }, @@ -388,6 +392,8 @@ export default ({ getService }: FtrProviderContext) => { ip: '45.115.45.3', matched: { atomic: '45.115.45.3', + id: '978785', + index: 'filebeat-8.0.0-2021.01.26-000001', field: 'source.ip', type: 'url', }, @@ -401,6 +407,8 @@ export default ({ getService }: FtrProviderContext) => { ip: '45.115.45.3', matched: { atomic: '45.115.45.3', + id: '978787', + index: 'filebeat-8.0.0-2021.01.26-000001', field: 'source.ip', type: 'ip', }, @@ -468,6 +476,8 @@ export default ({ getService }: FtrProviderContext) => { ip: '45.115.45.3', matched: { atomic: '45.115.45.3', + id: '978785', + index: 'filebeat-8.0.0-2021.01.26-000001', field: 'source.ip', type: 'url', }, @@ -475,18 +485,6 @@ export default ({ getService }: FtrProviderContext) => { provider: 'geenensp', type: 'url', }, - { - description: 'this should match auditbeat/hosts on ip', - first_seen: '2021-01-26T11:06:03.000Z', - ip: '45.115.45.3', - matched: { - atomic: '45.115.45.3', - field: 'source.ip', - type: 'ip', - }, - provider: 'other_provider', - type: 'ip', - }, // We do not merge matched indicators during enrichment, so in // certain circumstances a given indicator document could appear // multiple times in an enriched alert (albeit with different @@ -498,6 +496,8 @@ export default ({ getService }: FtrProviderContext) => { ip: '45.115.45.3', matched: { atomic: 57324, + id: '978785', + index: 'filebeat-8.0.0-2021.01.26-000001', field: 'source.port', type: 'url', }, @@ -505,6 +505,20 @@ export default ({ getService }: FtrProviderContext) => { provider: 'geenensp', type: 'url', }, + { + description: 'this should match auditbeat/hosts on ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + matched: { + atomic: '45.115.45.3', + id: '978787', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'source.ip', + type: 'ip', + }, + provider: 'other_provider', + type: 'ip', + }, ], }, ]); @@ -570,6 +584,8 @@ export default ({ getService }: FtrProviderContext) => { first_seen: '2021-01-26T11:09:04.000Z', matched: { atomic: '159.89.119.67', + id: '978783', + index: 'filebeat-8.0.0-2021.01.26-000001', field: 'destination.ip', type: 'url', }, @@ -590,6 +606,8 @@ export default ({ getService }: FtrProviderContext) => { first_seen: '2021-01-26T11:09:04.000Z', matched: { atomic: '159.89.119.67', + id: '978783', + index: 'filebeat-8.0.0-2021.01.26-000001', field: 'destination.ip', type: 'url', }, @@ -606,6 +624,8 @@ export default ({ getService }: FtrProviderContext) => { ip: '45.115.45.3', matched: { atomic: '45.115.45.3', + id: '978785', + index: 'filebeat-8.0.0-2021.01.26-000001', field: 'source.ip', type: 'url', }, @@ -619,6 +639,8 @@ export default ({ getService }: FtrProviderContext) => { ip: '45.115.45.3', matched: { atomic: 57324, + id: '978785', + index: 'filebeat-8.0.0-2021.01.26-000001', field: 'source.port', type: 'url', },