From fdeb172d8c1a7c7e3711b51eb9002db0c4d5e680 Mon Sep 17 00:00:00 2001 From: Georgii Gorbachev Date: Tue, 1 Dec 2020 14:57:53 +0100 Subject: [PATCH] [Security Solution][Detections] Support arrays in event fields for Severity/Risk overrides (#83723) This PR changes the behavior of severity and risk score overrides in two ways: - adds support for arrays in the mapped event fields (so a rule can be triggered by an event where e.g. `event.custom_severity` has a value like `[45, 70, 90]`) - makes the logic of overrides more flexible, resilient to the incoming values (filters out junk, extracts meaningful values, does its best to find a value that would fit the mapping) --- .../signals/__mocks__/es_results.ts | 25 +- .../build_risk_score_from_mapping.test.ts | 213 +++++++++++++++++- .../mappings/build_risk_score_from_mapping.ts | 73 ++++-- .../build_severity_from_mapping.test.ts | 184 +++++++++++---- .../mappings/build_severity_from_mapping.ts | 113 +++++++--- .../tests/generating_signals.ts | 153 +++++++++++++ .../signals/severity_risk_overrides/data.json | 55 +++++ .../severity_risk_overrides/mappings.json | 26 +++ 8 files changed, 743 insertions(+), 99 deletions(-) create mode 100644 x-pack/test/functional/es_archives/signals/severity_risk_overrides/data.json create mode 100644 x-pack/test/functional/es_archives/signals/severity_risk_overrides/mappings.json diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 92e6b9562d9706..0a38bdc790b412 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { set } from '@elastic/safer-lodash-set'; import { SignalSourceHit, SignalSearchResponse, @@ -189,9 +190,25 @@ export const sampleDocNoSortId = ( sort: [], }); -export const sampleDocSeverity = ( - severity?: Array | string | number | null -): SignalSourceHit => ({ +export const sampleDocSeverity = (severity?: unknown, fieldName?: string): SignalSourceHit => { + const doc = { + _index: 'myFakeSignalIndex', + _type: 'doc', + _score: 100, + _version: 1, + _id: sampleIdGuid, + _source: { + someKey: 'someValue', + '@timestamp': '2020-04-20T21:27:45+0000', + }, + sort: [], + }; + + set(doc._source, fieldName ?? 'event.severity', severity); + return doc; +}; + +export const sampleDocRiskScore = (riskScore?: unknown): SignalSourceHit => ({ _index: 'myFakeSignalIndex', _type: 'doc', _score: 100, @@ -201,7 +218,7 @@ export const sampleDocSeverity = ( someKey: 'someValue', '@timestamp': '2020-04-20T21:27:45+0000', event: { - severity: severity ?? 100, + risk: riskScore, }, }, sort: [], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_risk_score_from_mapping.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_risk_score_from_mapping.test.ts index ff50c2634dfd16..9395085dd6e99a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_risk_score_from_mapping.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_risk_score_from_mapping.test.ts @@ -4,23 +4,218 @@ * you may not use this file except in compliance with the Elastic License. */ -import { sampleDocNoSortId } from '../__mocks__/es_results'; -import { buildRiskScoreFromMapping } from './build_risk_score_from_mapping'; +import { + RiskScore, + RiskScoreMappingOrUndefined, +} from '../../../../../common/detection_engine/schemas/common/schemas'; +import { sampleDocRiskScore } from '../__mocks__/es_results'; +import { + buildRiskScoreFromMapping, + BuildRiskScoreFromMappingReturn, +} from './build_risk_score_from_mapping'; describe('buildRiskScoreFromMapping', () => { beforeEach(() => { jest.clearAllMocks(); }); - test('risk score defaults to provided if mapping is incomplete', () => { - const riskScore = buildRiskScoreFromMapping({ - eventSource: sampleDocNoSortId()._source, - riskScore: 57, - riskScoreMapping: undefined, + describe('base cases: when mapping is undefined', () => { + test('returns the provided default score', () => { + testIt({ + fieldValue: 42, + scoreDefault: 57, + scoreMapping: undefined, + expected: scoreOf(57), + }); }); + }); + + describe('base cases: when mapping to a field of type number', () => { + test(`returns that number if it's integer and within the range [0;100]`, () => { + testIt({ + fieldValue: 42, + scoreDefault: 57, + scoreMapping: mappingToSingleField(), + expected: overriddenScoreOf(42), + }); + }); + + test(`returns that number if it's float and within the range [0;100]`, () => { + testIt({ + fieldValue: 3.14, + scoreDefault: 57, + scoreMapping: mappingToSingleField(), + expected: overriddenScoreOf(3.14), + }); + }); + + test(`returns default score if the number is < 0`, () => { + testIt({ + fieldValue: -0.0000000000001, + scoreDefault: 57, + scoreMapping: mappingToSingleField(), + expected: scoreOf(57), + }); + }); + + test(`returns default score if the number is > 100`, () => { + testIt({ + fieldValue: 100.0000000000001, + scoreDefault: 57, + scoreMapping: mappingToSingleField(), + expected: scoreOf(57), + }); + }); + }); + + describe('base cases: when mapping to a field of type string', () => { + test(`returns the number casted from string if it's integer and within the range [0;100]`, () => { + testIt({ + fieldValue: '42', + scoreDefault: 57, + scoreMapping: mappingToSingleField(), + expected: overriddenScoreOf(42), + }); + }); + + test(`returns the number casted from string if it's float and within the range [0;100]`, () => { + testIt({ + fieldValue: '3.14', + scoreDefault: 57, + scoreMapping: mappingToSingleField(), + expected: overriddenScoreOf(3.14), + }); + }); + + test(`returns default score if the "number" is < 0`, () => { + testIt({ + fieldValue: '-1', + scoreDefault: 57, + scoreMapping: mappingToSingleField(), + expected: scoreOf(57), + }); + }); + + test(`returns default score if the "number" is > 100`, () => { + testIt({ + fieldValue: '101', + scoreDefault: 57, + scoreMapping: mappingToSingleField(), + expected: scoreOf(57), + }); + }); + }); - expect(riskScore).toEqual({ riskScore: 57, riskScoreMeta: {} }); + describe('base cases: when mapping to an array of numbers or strings', () => { + test(`returns that number if it's a single element and it's within the range [0;100]`, () => { + testIt({ + fieldValue: [3.14], + scoreDefault: 57, + scoreMapping: mappingToSingleField(), + expected: overriddenScoreOf(3.14), + }); + }); + + test(`returns the max number of those that are within the range [0;100]`, () => { + testIt({ + fieldValue: [42, -42, 17, 87, 87.5, '86.5', 110, 66], + scoreDefault: 57, + scoreMapping: mappingToSingleField(), + expected: overriddenScoreOf(87.5), + }); + }); + + test(`supports casting strings to numbers`, () => { + testIt({ + fieldValue: [-1, 1, '3', '1.5', '3.14', 2], + scoreDefault: 57, + scoreMapping: mappingToSingleField(), + expected: overriddenScoreOf(3.14), + }); + }); }); - // TODO: Enhance... + describe('edge cases: when mapping to a single junk value', () => { + describe('ignores it and returns the default score', () => { + const cases = [ + undefined, + null, + NaN, + Infinity, + -Infinity, + Number.MAX_VALUE, + -Number.MAX_VALUE, + -Number.MIN_VALUE, + 'string', + [], + {}, + new Date(), + ]; + + test.each(cases)('%p', (value) => { + testIt({ + fieldValue: value, + scoreDefault: 57, + scoreMapping: mappingToSingleField(), + expected: scoreOf(57), + }); + }); + }); + }); + + describe('edge cases: when mapping to an array of junk values', () => { + describe('ignores junk, extracts valid numbers and returns the max number within the range [0;100]', () => { + type Case = [unknown[], number]; + const cases: Case[] = [ + [[undefined, null, 1.5, 1, -Infinity], 1.5], + [['42', NaN, '44', '43', 42, {}], 44], + [[Infinity, '101', 100, 99, Number.MIN_VALUE], 100], + [[Number.MIN_VALUE, -0], Number.MIN_VALUE], + ]; + + test.each(cases)('%p', (value, expectedScore) => { + testIt({ + fieldValue: value, + scoreDefault: 57, + scoreMapping: mappingToSingleField(), + expected: overriddenScoreOf(expectedScore), + }); + }); + }); + }); }); + +interface TestCase { + fieldValue: unknown; + scoreDefault: RiskScore; + scoreMapping: RiskScoreMappingOrUndefined; + expected: BuildRiskScoreFromMappingReturn; +} + +function testIt({ fieldValue, scoreDefault, scoreMapping, expected }: TestCase) { + const result = buildRiskScoreFromMapping({ + eventSource: sampleDocRiskScore(fieldValue)._source, + riskScore: scoreDefault, + riskScoreMapping: scoreMapping, + }); + + expect(result).toEqual(expected); +} + +function mappingToSingleField() { + return [{ field: 'event.risk', operator: 'equals' as const, value: '', risk_score: undefined }]; +} + +function scoreOf(value: number) { + return { + riskScore: value, + riskScoreMeta: {}, + }; +} + +function overriddenScoreOf(value: number) { + return { + riskScore: value, + riskScoreMeta: { riskScoreOverridden: true }, + }; +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_risk_score_from_mapping.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_risk_score_from_mapping.ts index c358339e66cd92..cb3fcba1023507 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_risk_score_from_mapping.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_risk_score_from_mapping.ts @@ -11,35 +11,78 @@ import { } from '../../../../../common/detection_engine/schemas/common/schemas'; import { SignalSource } from '../types'; -interface BuildRiskScoreFromMappingProps { +export interface BuildRiskScoreFromMappingProps { eventSource: SignalSource; riskScore: RiskScore; riskScoreMapping: RiskScoreMappingOrUndefined; } -interface BuildRiskScoreFromMappingReturn { +export interface BuildRiskScoreFromMappingReturn { riskScore: RiskScore; riskScoreMeta: Meta; // TODO: Stricter types } +/** + * Calculates the final risk score for a detection alert based on: + * - source event object that can potentially contain fields representing risk score + * - the default score specified by the user + * - (optional) score mapping specified by the user ("map this field to the score") + * + * NOTE: Current MVP support is for mapping from a single field. + */ export const buildRiskScoreFromMapping = ({ eventSource, riskScore, riskScoreMapping, }: BuildRiskScoreFromMappingProps): BuildRiskScoreFromMappingReturn => { - // MVP support is for mapping from a single field - if (riskScoreMapping != null && riskScoreMapping.length > 0) { - const mappedField = riskScoreMapping[0].field; - // TODO: Expand by verifying fieldType from index via doc._index - const mappedValue = get(mappedField, eventSource); - if ( - typeof mappedValue === 'number' && - Number.isSafeInteger(mappedValue) && - mappedValue >= 0 && - mappedValue <= 100 - ) { - return { riskScore: mappedValue, riskScoreMeta: { riskScoreOverridden: true } }; + if (!riskScoreMapping || !riskScoreMapping.length) { + return defaultScore(riskScore); + } + + // TODO: Expand by verifying fieldType from index via doc._index + const eventField = riskScoreMapping[0].field; + const eventValue = get(eventField, eventSource); + const eventValues = Array.isArray(eventValue) ? eventValue : [eventValue]; + + const validNumbers = eventValues.map(toValidNumberOrMinusOne).filter((n) => n > -1); + + if (validNumbers.length > 0) { + const maxNumber = getMaxOf(validNumbers); + return overriddenScore(maxNumber); + } + + return defaultScore(riskScore); +}; + +function toValidNumberOrMinusOne(value: unknown): number { + if (typeof value === 'number' && isValidNumber(value)) { + return value; + } + + if (typeof value === 'string') { + const num = Number(value); + if (isValidNumber(num)) { + return num; } } + + return -1; +} + +function isValidNumber(value: number): boolean { + return Number.isFinite(value) && value >= 0 && value <= 100; +} + +function getMaxOf(array: number[]) { + // NOTE: It's safer to use reduce rather than Math.max(...array). The latter won't handle large input. + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/max + return array.reduce((a, b) => Math.max(a, b)); +} + +function defaultScore(riskScore: RiskScore): BuildRiskScoreFromMappingReturn { return { riskScore, riskScoreMeta: {} }; -}; +} + +function overriddenScore(riskScore: RiskScore): BuildRiskScoreFromMappingReturn { + return { riskScore, riskScoreMeta: { riskScoreOverridden: true } }; +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_severity_from_mapping.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_severity_from_mapping.test.ts index 430564cd985c2e..cfb5c56d7cd23d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_severity_from_mapping.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_severity_from_mapping.test.ts @@ -4,63 +4,169 @@ * you may not use this file except in compliance with the Elastic License. */ -import { sampleDocNoSortId, sampleDocSeverity } from '../__mocks__/es_results'; -import { buildSeverityFromMapping } from './build_severity_from_mapping'; +import { + Severity, + SeverityMappingOrUndefined, +} from '../../../../../common/detection_engine/schemas/common/schemas'; +import { sampleDocSeverity } from '../__mocks__/es_results'; +import { + buildSeverityFromMapping, + BuildSeverityFromMappingReturn, +} from './build_severity_from_mapping'; + +const ECS_FIELD = 'event.severity'; +const ANY_FIELD = 'event.my_custom_severity'; describe('buildSeverityFromMapping', () => { beforeEach(() => { jest.clearAllMocks(); }); - test('severity defaults to provided if mapping is undefined', () => { - const severity = buildSeverityFromMapping({ - eventSource: sampleDocNoSortId()._source, - severity: 'low', - severityMapping: undefined, + describe('base cases: when mapping is undefined', () => { + test('returns the provided default severity', () => { + testIt({ + fieldValue: 23, + severityDefault: 'low', + severityMapping: undefined, + expected: severityOf('low'), + }); + }); + }); + + describe('base cases: when mapping to the "event.severity" field from ECS', () => { + test(`severity is overridden if there's a match to a number`, () => { + testIt({ + fieldValue: 23, + severityDefault: 'low', + severityMapping: [ + { field: ECS_FIELD, operator: 'equals', value: '13', severity: 'low' }, + { field: ECS_FIELD, operator: 'equals', value: '23', severity: 'medium' }, + { field: ECS_FIELD, operator: 'equals', value: '33', severity: 'high' }, + { field: ECS_FIELD, operator: 'equals', value: '43', severity: 'critical' }, + ], + expected: overriddenSeverityOf('medium'), + }); }); - expect(severity).toEqual({ severity: 'low', severityMeta: {} }); + test(`returns the default severity if there's a match to a string (ignores strings)`, () => { + testIt({ + fieldValue: 'hackerman', + severityDefault: 'low', + severityMapping: [ + { field: ECS_FIELD, operator: 'equals', value: 'hackerman', severity: 'critical' }, + ], + expected: severityOf('low'), + }); + }); }); - test('severity is overridden to highest matched mapping', () => { - const severity = buildSeverityFromMapping({ - eventSource: sampleDocSeverity(23)._source, - severity: 'low', - severityMapping: [ - { field: 'event.severity', operator: 'equals', value: '23', severity: 'critical' }, - { field: 'event.severity', operator: 'equals', value: '23', severity: 'low' }, - { field: 'event.severity', operator: 'equals', value: '11', severity: 'critical' }, - { field: 'event.severity', operator: 'equals', value: '23', severity: 'medium' }, - ], + describe('base cases: when mapping to any other field containing a single value', () => { + test(`severity is overridden if there's a match to a number`, () => { + testIt({ + fieldName: ANY_FIELD, + fieldValue: 23, + severityDefault: 'low', + severityMapping: [ + { field: ANY_FIELD, operator: 'equals', value: '13', severity: 'low' }, + { field: ANY_FIELD, operator: 'equals', value: '23', severity: 'medium' }, + { field: ANY_FIELD, operator: 'equals', value: '33', severity: 'high' }, + { field: ANY_FIELD, operator: 'equals', value: '43', severity: 'critical' }, + ], + expected: overriddenSeverityOf('medium', ANY_FIELD), + }); }); - expect(severity).toEqual({ - severity: 'critical', - severityMeta: { - severityOverrideField: 'event.severity', - }, + test(`severity is overridden if there's a match to a string`, () => { + testIt({ + fieldName: ANY_FIELD, + fieldValue: 'hackerman', + severityDefault: 'low', + severityMapping: [ + { field: ANY_FIELD, operator: 'equals', value: 'anything', severity: 'medium' }, + { field: ANY_FIELD, operator: 'equals', value: 'hackerman', severity: 'critical' }, + ], + expected: overriddenSeverityOf('critical', ANY_FIELD), + }); }); }); - test('severity is overridden when field is event.severity and source value is number', () => { - const severity = buildSeverityFromMapping({ - eventSource: sampleDocSeverity(23)._source, - severity: 'low', - severityMapping: [ - { field: 'event.severity', operator: 'equals', value: '13', severity: 'low' }, - { field: 'event.severity', operator: 'equals', value: '23', severity: 'medium' }, - { field: 'event.severity', operator: 'equals', value: '33', severity: 'high' }, - { field: 'event.severity', operator: 'equals', value: '43', severity: 'critical' }, - ], + describe('base cases: when mapping to an array', () => { + test(`severity is overridden to highest matched mapping (works for "event.severity" field)`, () => { + testIt({ + fieldValue: [23, 'some string', 43, 33], + severityDefault: 'low', + severityMapping: [ + { field: ECS_FIELD, operator: 'equals', value: '13', severity: 'low' }, + { field: ECS_FIELD, operator: 'equals', value: '23', severity: 'medium' }, + { field: ECS_FIELD, operator: 'equals', value: '33', severity: 'high' }, + { field: ECS_FIELD, operator: 'equals', value: '43', severity: 'critical' }, + ], + expected: overriddenSeverityOf('critical'), + }); }); - expect(severity).toEqual({ - severity: 'medium', - severityMeta: { - severityOverrideField: 'event.severity', - }, + test(`severity is overridden to highest matched mapping (works for any custom field)`, () => { + testIt({ + fieldName: ANY_FIELD, + fieldValue: ['foo', 'bar', 'baz', 'boo'], + severityDefault: 'low', + severityMapping: [ + { field: ANY_FIELD, operator: 'equals', value: 'bar', severity: 'high' }, + { field: ANY_FIELD, operator: 'equals', value: 'baz', severity: 'critical' }, + { field: ANY_FIELD, operator: 'equals', value: 'foo', severity: 'low' }, + { field: ANY_FIELD, operator: 'equals', value: 'boo', severity: 'medium' }, + ], + expected: overriddenSeverityOf('critical', ANY_FIELD), + }); }); }); - // TODO: Enhance... + describe('edge cases: when mapping the same numerical value to different severities multiple times', () => { + test('severity is overridden to highest matched mapping', () => { + testIt({ + fieldValue: 23, + severityDefault: 'low', + severityMapping: [ + { field: ECS_FIELD, operator: 'equals', value: '23', severity: 'medium' }, + { field: ECS_FIELD, operator: 'equals', value: '23', severity: 'critical' }, + { field: ECS_FIELD, operator: 'equals', value: '23', severity: 'high' }, + ], + expected: overriddenSeverityOf('critical'), + }); + }); + }); }); + +interface TestCase { + fieldName?: string; + fieldValue: unknown; + severityDefault: Severity; + severityMapping: SeverityMappingOrUndefined; + expected: BuildSeverityFromMappingReturn; +} + +function testIt({ fieldName, fieldValue, severityDefault, severityMapping, expected }: TestCase) { + const result = buildSeverityFromMapping({ + eventSource: sampleDocSeverity(fieldValue, fieldName)._source, + severity: severityDefault, + severityMapping, + }); + + expect(result).toEqual(expected); +} + +function severityOf(value: Severity) { + return { + severity: value, + severityMeta: {}, + }; +} + +function overriddenSeverityOf(value: Severity, field = ECS_FIELD) { + return { + severity: value, + severityMeta: { + severityOverrideField: field, + }, + }; +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_severity_from_mapping.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_severity_from_mapping.ts index 52ebd67f257af6..1560bbb48f0bab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_severity_from_mapping.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_severity_from_mapping.ts @@ -11,15 +11,16 @@ import { severity as SeverityIOTS, SeverityMappingOrUndefined, } from '../../../../../common/detection_engine/schemas/common/schemas'; +import { SearchTypes } from '../../../../../common/detection_engine/types'; import { SignalSource } from '../types'; -interface BuildSeverityFromMappingProps { +export interface BuildSeverityFromMappingProps { eventSource: SignalSource; severity: Severity; severityMapping: SeverityMappingOrUndefined; } -interface BuildSeverityFromMappingReturn { +export interface BuildSeverityFromMappingReturn { severity: Severity; severityMeta: Meta; // TODO: Stricter types } @@ -31,41 +32,89 @@ const severitySortMapping = { critical: 3, }; +const ECS_SEVERITY_FIELD = 'event.severity'; + export const buildSeverityFromMapping = ({ eventSource, severity, severityMapping, }: BuildSeverityFromMappingProps): BuildSeverityFromMappingReturn => { - if (severityMapping != null && severityMapping.length > 0) { - let severityMatch: SeverityMappingItem | undefined; - - // Sort the SeverityMapping from low to high, so last match (highest severity) is used - const severityMappingSorted = severityMapping.sort( - (a, b) => severitySortMapping[a.severity] - severitySortMapping[b.severity] - ); - - severityMappingSorted.forEach((mapping) => { - const docValue = get(mapping.field, eventSource); - // TODO: Expand by verifying fieldType from index via doc._index - // Till then, explicit parsing of event.severity (long) to number. If not ECS, this could be - // another datatype, but until we can lookup datatype we must assume number for the Elastic - // Endpoint Security rule to function correctly - let parsedMappingValue: string | number = mapping.value; - if (mapping.field === 'event.severity') { - parsedMappingValue = Math.floor(Number(parsedMappingValue)); - } - - if (parsedMappingValue === docValue) { - severityMatch = { ...mapping }; - } - }); - - if (severityMatch != null && SeverityIOTS.is(severityMatch.severity)) { - return { - severity: severityMatch.severity, - severityMeta: { severityOverrideField: severityMatch.field }, - }; + if (!severityMapping || !severityMapping.length) { + return defaultSeverity(severity); + } + + let severityMatch: SeverityMappingItem | undefined; + + // Sort the SeverityMapping from low to high, so last match (highest severity) is used + const severityMappingSorted = severityMapping.sort( + (a, b) => severitySortMapping[a.severity] - severitySortMapping[b.severity] + ); + + severityMappingSorted.forEach((mapping) => { + const mappingField = mapping.field; + const mappingValue = mapping.value; + const eventValue = get(mappingField, eventSource); + + const normalizedEventValues = normalizeEventValue(mappingField, eventValue); + const normalizedMappingValue = normalizeMappingValue(mappingField, mappingValue); + + if (normalizedEventValues.has(normalizedMappingValue)) { + severityMatch = { ...mapping }; } + }); + + if (severityMatch != null && SeverityIOTS.is(severityMatch.severity)) { + return overriddenSeverity(severityMatch.severity, severityMatch.field); } - return { severity, severityMeta: {} }; + + return defaultSeverity(severity); }; + +function normalizeMappingValue(eventField: string, mappingValue: string): string | number { + // TODO: Expand by verifying fieldType from index via doc._index + // Till then, explicit parsing of event.severity (long) to number. If not ECS, this could be + // another datatype, but until we can lookup datatype we must assume number for the Elastic + // Endpoint Security rule to function correctly + if (eventField === ECS_SEVERITY_FIELD) { + return Math.floor(Number(mappingValue)); + } + + return mappingValue; +} + +function normalizeEventValue(eventField: string, eventValue: SearchTypes): Set { + const eventValues = Array.isArray(eventValue) ? eventValue : [eventValue]; + const validValues = eventValues.filter((v): v is string | number => isValidValue(eventField, v)); + const finalValues = eventField === ECS_SEVERITY_FIELD ? validValues : validValues.map(String); + return new Set(finalValues); +} + +function isValidValue(eventField: string, value: unknown): value is string | number { + return eventField === ECS_SEVERITY_FIELD + ? isValidNumber(value) + : isValidNumber(value) || isValidString(value); +} + +function isValidString(value: unknown): value is string { + return typeof value === 'string'; +} + +function isValidNumber(value: unknown): value is number { + return typeof value === 'number' && Number.isSafeInteger(value); +} + +function defaultSeverity(value: Severity): BuildSeverityFromMappingReturn { + return { + severity: value, + severityMeta: {}, + }; +} + +function overriddenSeverity(value: Severity, field: string): BuildSeverityFromMappingReturn { + return { + severity: value, + severityMeta: { + severityOverrideField: field, + }, + }; +} diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts index 0db3013503a33f..9442d911c3fd95 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts @@ -5,6 +5,7 @@ */ import expect from '@kbn/expect'; +import { orderBy } from 'lodash'; import { EqlCreateSchema, @@ -617,5 +618,157 @@ export default ({ getService }: FtrProviderContext) => { }); }); }); + + /** + * Here we test the functionality of Severity and Risk Score overrides (also called "mappings" + * in the code). If the rule specifies a mapping, then the final Severity or Risk Score + * value of the signal will be taken from the mapped field of the source event. + */ + describe('Signals generated from events with custom severity and risk score fields', () => { + beforeEach(async () => { + await esArchiver.load('signals/severity_risk_overrides'); + }); + + afterEach(async () => { + await esArchiver.unload('signals/severity_risk_overrides'); + }); + + const executeRuleAndGetSignals = async (rule: QueryCreateSchema) => { + const { id } = await createRule(supertest, rule); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 4, [id]); + const signalsResponse = await getSignalsByIds(supertest, [id]); + const signals = signalsResponse.hits.hits.map((hit) => hit._source); + const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); + return signalsOrderedByEventId; + }; + + it('should get default severity and risk score if there is no mapping', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['signal_overrides']), + severity: 'medium', + risk_score: 75, + }; + + const signals = await executeRuleAndGetSignals(rule); + + expect(signals.length).equal(4); + signals.forEach((s) => { + expect(s.signal.rule.severity).equal('medium'); + expect(s.signal.rule.severity_mapping).eql([]); + + expect(s.signal.rule.risk_score).equal(75); + expect(s.signal.rule.risk_score_mapping).eql([]); + }); + }); + + it('should get overridden severity if the rule has a mapping for it', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['signal_overrides']), + severity: 'medium', + severity_mapping: [ + { field: 'my_severity', operator: 'equals', value: 'sev_900', severity: 'high' }, + { field: 'my_severity', operator: 'equals', value: 'sev_max', severity: 'critical' }, + ], + risk_score: 75, + }; + + const signals = await executeRuleAndGetSignals(rule); + const severities = signals.map((s) => ({ + id: s.signal.parent?.id, + value: s.signal.rule.severity, + })); + + expect(signals.length).equal(4); + expect(severities).eql([ + { id: '1', value: 'high' }, + { id: '2', value: 'critical' }, + { id: '3', value: 'critical' }, + { id: '4', value: 'critical' }, + ]); + + signals.forEach((s) => { + expect(s.signal.rule.risk_score).equal(75); + expect(s.signal.rule.risk_score_mapping).eql([]); + expect(s.signal.rule.severity_mapping).eql([ + { field: 'my_severity', operator: 'equals', value: 'sev_900', severity: 'high' }, + { field: 'my_severity', operator: 'equals', value: 'sev_max', severity: 'critical' }, + ]); + }); + }); + + it('should get overridden risk score if the rule has a mapping for it', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['signal_overrides']), + severity: 'medium', + risk_score: 75, + risk_score_mapping: [ + { field: 'my_risk', operator: 'equals', value: '', risk_score: undefined }, + ], + }; + + const signals = await executeRuleAndGetSignals(rule); + const riskScores = signals.map((s) => ({ + id: s.signal.parent?.id, + value: s.signal.rule.risk_score, + })); + + expect(signals.length).equal(4); + expect(riskScores).eql([ + { id: '1', value: 31.14 }, + { id: '2', value: 32.14 }, + { id: '3', value: 33.14 }, + { id: '4', value: 34.14 }, + ]); + + signals.forEach((s) => { + expect(s.signal.rule.severity).equal('medium'); + expect(s.signal.rule.severity_mapping).eql([]); + expect(s.signal.rule.risk_score_mapping).eql([ + { field: 'my_risk', operator: 'equals', value: '' }, + ]); + }); + }); + + it('should get overridden severity and risk score if the rule has both mappings', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['signal_overrides']), + severity: 'medium', + severity_mapping: [ + { field: 'my_severity', operator: 'equals', value: 'sev_900', severity: 'high' }, + { field: 'my_severity', operator: 'equals', value: 'sev_max', severity: 'critical' }, + ], + risk_score: 75, + risk_score_mapping: [ + { field: 'my_risk', operator: 'equals', value: '', risk_score: undefined }, + ], + }; + + const signals = await executeRuleAndGetSignals(rule); + const values = signals.map((s) => ({ + id: s.signal.parent?.id, + severity: s.signal.rule.severity, + risk: s.signal.rule.risk_score, + })); + + expect(signals.length).equal(4); + expect(values).eql([ + { id: '1', severity: 'high', risk: 31.14 }, + { id: '2', severity: 'critical', risk: 32.14 }, + { id: '3', severity: 'critical', risk: 33.14 }, + { id: '4', severity: 'critical', risk: 34.14 }, + ]); + + signals.forEach((s) => { + expect(s.signal.rule.severity_mapping).eql([ + { field: 'my_severity', operator: 'equals', value: 'sev_900', severity: 'high' }, + { field: 'my_severity', operator: 'equals', value: 'sev_max', severity: 'critical' }, + ]); + expect(s.signal.rule.risk_score_mapping).eql([ + { field: 'my_risk', operator: 'equals', value: '' }, + ]); + }); + }); + }); }); }; diff --git a/x-pack/test/functional/es_archives/signals/severity_risk_overrides/data.json b/x-pack/test/functional/es_archives/signals/severity_risk_overrides/data.json new file mode 100644 index 00000000000000..1f541dc1ef0a57 --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/severity_risk_overrides/data.json @@ -0,0 +1,55 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": "signal_overrides", + "source": { + "@timestamp": "2020-11-24T13:00:01.000Z", + "my_severity" : "sev_900", + "my_risk": 31.14 + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "2", + "index": "signal_overrides", + "source": { + "@timestamp": "2020-11-24T13:00:02.000Z", + "my_severity": ["sev_900", "sev_max"], + "my_risk": [32.14] + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "3", + "index": "signal_overrides", + "source": { + "@timestamp": "2020-11-24T13:00:03.000Z", + "my_severity": ["sev_max", "sev_900"], + "my_risk": "33.14" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "4", + "index": "signal_overrides", + "source": { + "@timestamp": "2020-11-24T13:00:04.000Z", + "my_severity": "sev_max", + "my_risk": [3.14, "34.14"] + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/signals/severity_risk_overrides/mappings.json b/x-pack/test/functional/es_archives/signals/severity_risk_overrides/mappings.json new file mode 100644 index 00000000000000..8a67be50e05fe5 --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/severity_risk_overrides/mappings.json @@ -0,0 +1,26 @@ +{ + "type": "index", + "value": { + "index": "signal_overrides", + "mappings": { + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date" + }, + "my_severity": { + "type": "keyword" + }, + "my_risk": { + "type": "integer" + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +}