diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx index f06d4bdef74cb7..1fe1b809d4f30a 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx @@ -52,7 +52,7 @@ import { } from '../../../../../common/detection_engine/utils'; import { EqlQueryBar } from '../eql_query_bar'; import { ThreatMatchInput } from '../threatmatch_input'; -import { useFetchIndex } from '../../../../common/containers/source'; +import { BrowserField, BrowserFields, useFetchIndex } from '../../../../common/containers/source'; import { PreviewQuery, Threshold } from '../query_preview'; const CommonUseField = getUseField({ component: Field }); @@ -168,6 +168,26 @@ const StepDefineRuleComponent: FC = ({ const queryBarQuery = formQuery != null ? formQuery.query.query : '' || initialState.queryBar.query.query; const [indexPatternsLoading, { browserFields, indexPatterns }] = useFetchIndex(index); + const aggregatableFields = Object.entries(browserFields).reduce( + (groupAcc, [groupName, groupValue]) => { + return { + ...groupAcc, + [groupName]: { + fields: Object.entries(groupValue.fields ?? {}).reduce>( + (fieldAcc, [fieldName, fieldValue]) => { + if (fieldValue.aggregatable === true) { + return { ...fieldAcc, [fieldName]: fieldValue }; + } + return fieldAcc; + }, + {} + ), + } as Partial, + }; + }, + {} + ); + const [ threatIndexPatternsLoading, { browserFields: threatBrowserFields, indexPatterns: threatIndexPatterns }, @@ -262,12 +282,12 @@ const StepDefineRuleComponent: FC = ({ const ThresholdInputChildren = useCallback( ({ thresholdField, thresholdValue }) => ( ), - [browserFields] + [aggregatableFields] ); const ThreatMatchInputChildren = useCallback( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts index 6a75d0655cf59d..022c07defc9c19 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts @@ -4,329 +4,77 @@ * you may not use this file except in compliance with the Elastic License. */ -import { sampleDocNoSortIdNoVersion } from './__mocks__/es_results'; -import { getThresholdSignalQueryFields } from './bulk_create_threshold_signals'; +import { loggingSystemMock } from '../../../../../../../src/core/server/mocks'; +import { sampleDocNoSortId, sampleDocSearchResultsNoSortId } from './__mocks__/es_results'; +import { transformThresholdResultsToEcs } from './bulk_create_threshold_signals'; +import { calculateThresholdSignalUuid } from './utils'; -describe('getThresholdSignalQueryFields', () => { - it('should return proper fields for match_phrase filters', () => { - const mockHit = { - ...sampleDocNoSortIdNoVersion(), - _source: { - '@timestamp': '2020-11-03T02:31:47.431Z', - event: { - dataset: 'traefik.access', - module: 'traefik', - }, - traefik: { - access: { - entryPointName: 'web-secure', - }, - }, - url: { - domain: 'kibana.siem.estc.dev', - }, - }, - }; - const mockFilters = { - bool: { - must: [], - filter: [ - { - bool: { - filter: [ - { - bool: { - should: [ - { - match_phrase: { - 'event.module': 'traefik', - }, - }, - ], - minimum_should_match: 1, - }, - }, - { - bool: { - filter: [ - { - bool: { - should: [ - { - match_phrase: { - 'event.dataset': 'traefik.access', - }, - }, - ], - minimum_should_match: 1, - }, - }, - { - bool: { - should: [ - { - match_phrase: { - 'traefik.access.entryPointName': 'web-secure', - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - }, - }, - ], - }, - }, - { - match_phrase: { - 'url.domain': 'kibana.siem.estc.dev', - }, - }, - ], - should: [], - must_not: [], - }, - }; - - expect(getThresholdSignalQueryFields(mockHit, mockFilters)).toEqual({ - 'event.dataset': 'traefik.access', - 'event.module': 'traefik', - 'traefik.access.entryPointName': 'web-secure', - 'url.domain': 'kibana.siem.estc.dev', - }); - }); - - it('should return proper fields object for nested match filters', () => { - const mockHit = { - ...sampleDocNoSortIdNoVersion(), - _source: { - '@timestamp': '2020-11-03T02:31:47.431Z', - event: { - dataset: 'traefik.access', - module: 'traefik', - }, - url: { - domain: 'kibana.siem.estc.dev', - }, - }, - }; - const filters = { - bool: { - must: [], - filter: [ - { - bool: { - filter: [ - { - bool: { - should: [ - { - match_phrase: { - 'event.module': 'traefik', - }, - }, - ], - minimum_should_match: 1, - }, - }, - { - bool: { - should: [ - { - match: { - 'event.dataset': 'traefik.*', - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - }, - }, - ], - should: [], - must_not: [], - }, - }; - - expect(getThresholdSignalQueryFields(mockHit, filters)).toEqual({ - 'event.dataset': 'traefik.access', - 'event.module': 'traefik', - }); - }); - - it('should return proper object for simple match filters', () => { - const mockHit = { - ...sampleDocNoSortIdNoVersion(), - _source: { - '@timestamp': '2020-11-03T02:31:47.431Z', - event: { - dataset: 'traefik.access', - module: 'traefik', - }, - }, - }; - const filters = { - bool: { - must: [], - filter: [ - { - bool: { - should: [ - { - match: { - 'event.module': 'traefik', - }, - }, - ], - minimum_should_match: 1, - }, - }, - { - match_phrase: { - 'event.dataset': 'traefik.access', - }, - }, - ], - should: [], - must_not: [], - }, - }; - - expect(getThresholdSignalQueryFields(mockHit, filters)).toEqual({ - 'event.dataset': 'traefik.access', - 'event.module': 'traefik', - }); - }); - - it('should return proper object for simple match_phrase filters', () => { - const mockHit = { - ...sampleDocNoSortIdNoVersion(), - _source: { - '@timestamp': '2020-11-03T02:31:47.431Z', - event: { - dataset: 'traefik.access', - module: 'traefik', - }, - }, +describe('transformThresholdResultsToEcs', () => { + it('should return transformed threshold results', () => { + const threshold = { + field: 'source.ip', + value: 1, }; - const filters = { - bool: { - must: [], - filter: [ - { - bool: { - should: [ - { - match_phrase: { - 'event.module': 'traefik', + const startedAt = new Date('2020-12-17T16:27:00Z'); + const transformedResults = transformThresholdResultsToEcs( + { + ...sampleDocSearchResultsNoSortId('abcd'), + aggregations: { + threshold: { + buckets: [ + { + key: '127.0.0.1', + doc_count: 1, + top_threshold_hits: { + hits: { + hits: [sampleDocNoSortId('abcd')], }, }, - ], - minimum_should_match: 1, - }, - }, - { - match_phrase: { - 'event.dataset': 'traefik.access', - }, + }, + ], }, - ], - should: [], - must_not: [], - }, - }; - - expect(getThresholdSignalQueryFields(mockHit, filters)).toEqual({ - 'event.module': 'traefik', - 'event.dataset': 'traefik.access', - }); - }); - - it('should return proper object for exists filters', () => { - const mockHit = { - ...sampleDocNoSortIdNoVersion(), - _source: { - '@timestamp': '2020-11-03T02:31:47.431Z', - event: { - module: 'traefik', }, }, - }; - const filters = { - bool: { - should: [ - { - bool: { - should: [ - { - exists: { - field: 'process.name', - }, - }, - ], - minimum_should_match: 1, - }, - }, - { - bool: { - should: [ - { - exists: { - field: 'event.type', - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - minimum_should_match: 1, + 'test', + startedAt, + undefined, + loggingSystemMock.createLogger(), + threshold, + '1234', + undefined + ); + const _id = calculateThresholdSignalUuid('1234', startedAt, 'source.ip', '127.0.0.1'); + expect(transformedResults).toEqual({ + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, }, - }; - expect(getThresholdSignalQueryFields(mockHit, filters)).toEqual({}); - }); - - it('should NOT add invalid characters from CIDR such as the "/" proper object for simple match_phrase filters', () => { - const mockHit = { - ...sampleDocNoSortIdNoVersion(), - _source: { - '@timestamp': '2020-11-03T02:31:47.431Z', - destination: { - ip: '192.168.0.16', - }, - event: { - module: 'traefik', + results: { + hits: { + total: 1, }, }, - }; - const filters = { - bool: { - must: [], - filter: [ + hits: { + total: 100, + max_score: 100, + hits: [ { - bool: { - should: [ - { - match: { - 'destination.ip': '192.168.0.0/16', - }, - }, - ], - minimum_should_match: 1, + _id, + _index: 'test', + _source: { + '@timestamp': '2020-04-20T21:27:45+0000', + threshold_result: { + count: 1, + value: '127.0.0.1', + }, }, }, ], - should: [], - must_not: [], }, - }; - - expect(getThresholdSignalQueryFields(mockHit, filters)).toEqual({ - 'destination.ip': '192.168.0.16', }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index a98aae4ec8107b..3cad33b2787495 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -4,8 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import uuidv5 from 'uuid/v5'; -import { reduce, get, isEmpty } from 'lodash/fp'; +import { get, isEmpty } from 'lodash/fp'; import set from 'set-value'; import { @@ -17,12 +16,10 @@ import { AlertServices } from '../../../../../alerts/server'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams, RefreshTypes } from '../types'; import { singleBulkCreate, SingleBulkCreateResponse } from './single_bulk_create'; -import { SignalSearchResponse, SignalSourceHit, ThresholdAggregationBucket } from './types'; +import { SignalSearchResponse, ThresholdAggregationBucket } from './types'; +import { calculateThresholdSignalUuid } from './utils'; import { BuildRuleMessage } from './rule_messages'; -// used to generate constant Threshold Signals ID when run with the same params -const NAMESPACE_ID = '0684ec03-7201-4ee0-8ee0-3a3f6b2479b2'; - interface BulkCreateThresholdSignalsParams { actions: RuleAlertAction[]; someResult: SignalSearchResponse; @@ -48,81 +45,6 @@ interface BulkCreateThresholdSignalsParams { buildRuleMessage: BuildRuleMessage; } -interface FilterObject { - bool?: { - filter?: FilterObject | FilterObject[]; - should?: Array>>; - }; -} - -const injectFirstMatch = ( - hit: SignalSourceHit, - match: object | Record -): Record | undefined => { - if (match != null) { - for (const key of Object.keys(match)) { - return { [key]: get(key, hit._source) } as Record; - } - } -}; - -const getNestedQueryFilters = ( - hit: SignalSourceHit, - filtersObj: FilterObject -): Record => { - if (Array.isArray(filtersObj.bool?.filter)) { - return reduce( - (acc, filterItem) => { - const nestedFilter = getNestedQueryFilters(hit, filterItem); - - if (nestedFilter) { - return { ...acc, ...nestedFilter }; - } - - return acc; - }, - {}, - filtersObj.bool?.filter - ); - } else { - return ( - (filtersObj.bool?.should && - filtersObj.bool?.should[0] && - (injectFirstMatch(hit, filtersObj.bool.should[0].match) || - injectFirstMatch(hit, filtersObj.bool.should[0].match_phrase))) ?? - {} - ); - } -}; - -export const getThresholdSignalQueryFields = (hit: SignalSourceHit, filter: unknown) => { - const filters = get('bool.filter', filter); - - return reduce( - (acc, item) => { - if (item.match_phrase) { - return { ...acc, ...injectFirstMatch(hit, item.match_phrase) }; - } - - if (item.bool?.should && (item.bool.should[0].match || item.bool.should[0].match_phrase)) { - return { - ...acc, - ...(injectFirstMatch(hit, item.bool.should[0].match) || - injectFirstMatch(hit, item.bool.should[0].match_phrase)), - }; - } - - if (item.bool?.filter) { - return { ...acc, ...getNestedQueryFilters(hit, item) }; - } - - return acc; - }, - {}, - filters - ); -}; - const getTransformedHits = ( results: SignalSearchResponse, inputIndex: string, @@ -153,13 +75,12 @@ const getTransformedHits = ( count: totalResults, value: ruleId, }, - ...getThresholdSignalQueryFields(hit, filter), }; return [ { _index: inputIndex, - _id: uuidv5(`${ruleId}${startedAt}${threshold.field}`, NAMESPACE_ID), + _id: calculateThresholdSignalUuid(ruleId, startedAt, threshold.field), _source: source, }, ]; @@ -183,14 +104,11 @@ const getTransformedHits = ( count: docCount, value: get(threshold.field, hit._source), }, - ...getThresholdSignalQueryFields(hit, filter), }; - set(source, threshold.field, key); - return { _index: inputIndex, - _id: uuidv5(`${ruleId}${startedAt}${threshold.field}${key}`, NAMESPACE_ID), + _id: calculateThresholdSignalUuid(ruleId, startedAt, threshold.field, key), _source: source, }; } @@ -226,6 +144,8 @@ export const transformThresholdResultsToEcs = ( }, }; + delete thresholdResults.aggregations; // no longer needed + set(thresholdResults, 'results.hits.total', transformedHits.length); return thresholdResults; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 7fd99a17598ae8..3928228357d4cb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -8,7 +8,6 @@ import { Logger, KibanaRequest } from 'src/core/server'; -import { Filter } from 'src/plugins/data/common'; import { SIGNALS_ID, DEFAULT_SEARCH_AFTER_PAGE_SIZE, @@ -29,7 +28,6 @@ import { SignalRuleAlertTypeDefinition, RuleAlertAttributes, EqlSignalSearchResponse, - ThresholdQueryBucket, WrappedSignalHit, } from './types'; import { @@ -48,9 +46,9 @@ import { signalParamsSchema } from './signal_params_schema'; import { siemRuleActionGroups } from './siem_rule_action_groups'; import { findMlSignals } from './find_ml_signals'; import { findThresholdSignals } from './find_threshold_signals'; -import { findPreviousThresholdSignals } from './find_previous_threshold_signals'; import { bulkCreateMlSignals } from './bulk_create_ml_signals'; import { bulkCreateThresholdSignals } from './bulk_create_threshold_signals'; +import { getThresholdBucketFilters } from './threshold_get_bucket_filters'; import { scheduleNotificationActions, NotificationRuleTypeParams, @@ -307,21 +305,11 @@ export const signalRulesAlertType = ({ ]); } else if (isThresholdRule(type) && threshold) { const inputIndex = await getInputIndex(services, version, index); - const esFilter = await getFilter({ - type, - filters, - language, - query, - savedId, - services, - index: inputIndex, - lists: exceptionItems ?? [], - }); const { - searchResult: previousSignals, + filters: bucketFilters, searchErrors: previousSearchErrors, - } = await findPreviousThresholdSignals({ + } = await getThresholdBucketFilters({ indexPattern: [outputIndex], from, to, @@ -333,29 +321,15 @@ export const signalRulesAlertType = ({ buildRuleMessage, }); - previousSignals.aggregations.threshold.buckets.forEach((bucket: ThresholdQueryBucket) => { - esFilter.bool.filter.push(({ - bool: { - must_not: { - bool: { - must: [ - { - term: { - [threshold.field || 'signal.rule.rule_id']: bucket.key, - }, - }, - { - range: { - [timestampOverride ?? '@timestamp']: { - lte: bucket.lastSignalTimestamp.value_as_string, - }, - }, - }, - ], - }, - }, - }, - } as unknown) as Filter); + const esFilter = await getFilter({ + type, + filters: filters ? filters.concat(bucketFilters) : bucketFilters, + language, + query, + savedId, + services, + index: inputIndex, + lists: exceptionItems ?? [], }); const { searchResult: thresholdResults, searchErrors } = await findThresholdSignals({ @@ -400,6 +374,7 @@ export const signalRulesAlertType = ({ tags, buildRuleMessage, }); + result = mergeReturns([ result, createSearchAfterReturnTypeFromResponse({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_find_previous_signals.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_find_previous_signals.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts new file mode 100644 index 00000000000000..bf060da1e76b8f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { isEmpty } from 'lodash'; + +import { Filter } from 'src/plugins/data/common'; +import { ESFilter } from '../../../../../../typings/elasticsearch'; + +import { TimestampOverrideOrUndefined } from '../../../../common/detection_engine/schemas/common/schemas'; +import { AlertServices } from '../../../../../alerts/server'; +import { Logger } from '../../../../../../../src/core/server'; +import { ThresholdQueryBucket } from './types'; +import { BuildRuleMessage } from './rule_messages'; +import { findPreviousThresholdSignals } from './threshold_find_previous_signals'; + +interface GetThresholdBucketFiltersParams { + from: string; + to: string; + indexPattern: string[]; + services: AlertServices; + logger: Logger; + ruleId: string; + bucketByField: string; + timestampOverride: TimestampOverrideOrUndefined; + buildRuleMessage: BuildRuleMessage; +} + +export const getThresholdBucketFilters = async ({ + from, + to, + indexPattern, + services, + logger, + ruleId, + bucketByField, + timestampOverride, + buildRuleMessage, +}: GetThresholdBucketFiltersParams): Promise<{ + filters: Filter[]; + searchErrors: string[]; +}> => { + const { searchResult, searchErrors } = await findPreviousThresholdSignals({ + indexPattern, + from, + to, + services, + logger, + ruleId, + bucketByField, + timestampOverride, + buildRuleMessage, + }); + + const filters = searchResult.aggregations.threshold.buckets.reduce( + (acc: ESFilter[], bucket: ThresholdQueryBucket): ESFilter[] => { + const filter = { + bool: { + filter: [ + { + range: { + [timestampOverride ?? '@timestamp']: { + lte: bucket.lastSignalTimestamp.value_as_string, + }, + }, + }, + ], + }, + } as ESFilter; + + if (!isEmpty(bucketByField)) { + (filter.bool.filter as ESFilter[]).push({ + term: { + [bucketByField]: bucket.key, + }, + }); + } + + return [...acc, filter]; + }, + [] as ESFilter[] + ); + + return { + filters: [ + ({ + bool: { + must_not: filters, + }, + } as unknown) as Filter, + ], + searchErrors, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index dd936776f691aa..073e30bbc6e26b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -36,6 +36,7 @@ import { mergeReturns, createTotalHitsFromSearchResult, lastValidDate, + calculateThresholdSignalUuid, } from './utils'; import { BulkResponseErrorAggregation, SearchAfterAndBulkCreateReturnType } from './types'; import { @@ -1303,4 +1304,18 @@ describe('utils', () => { expect(result).toEqual(4); }); }); + + describe('calculateThresholdSignalUuid', () => { + it('should generate a uuid without key', () => { + const startedAt = new Date('2020-12-17T16:27:00Z'); + const signalUuid = calculateThresholdSignalUuid('abcd', startedAt, 'agent.name'); + expect(signalUuid).toEqual('c0cbe4b7-48de-5734-ae81-d8de3e79839d'); + }); + + it('should generate a uuid with key', () => { + const startedAt = new Date('2019-11-18T13:32:00Z'); + const signalUuid = calculateThresholdSignalUuid('abcd', startedAt, 'host.ip', '1.2.3.4'); + expect(signalUuid).toEqual('f568509e-b570-5d3c-a7ed-7c73fd29ddaf'); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 2114f21d9cead6..18f6e8d127b1b3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -5,6 +5,7 @@ */ import { createHash } from 'crypto'; import moment from 'moment'; +import uuidv5 from 'uuid/v5'; import dateMath from '@elastic/datemath'; import { TimestampOverrideOrUndefined } from '../../../../common/detection_engine/schemas/common/schemas'; @@ -661,3 +662,20 @@ export const createTotalHitsFromSearchResult = ({ : searchResult.hits.total.value; return totalHits; }; + +export const calculateThresholdSignalUuid = ( + ruleId: string, + startedAt: Date, + thresholdField: string, + key?: string +): string => { + // used to generate constant Threshold Signals ID when run with the same params + const NAMESPACE_ID = '0684ec03-7201-4ee0-8ee0-3a3f6b2479b2'; + + let baseString = `${ruleId}${startedAt}${thresholdField}`; + if (key != null) { + baseString = `${baseString}${key}`; + } + + return uuidv5(baseString, NAMESPACE_ID); +};