From eb06b8cc5e0891384d26f1105d31e84e02cdc351 Mon Sep 17 00:00:00 2001 From: Marshall Main <55718608+marshallmain@users.noreply.github.com> Date: Fri, 25 Sep 2020 18:04:48 -0400 Subject: [PATCH] [Detections][EQL] EQL rule execution in detection engine (#77419) (#78550) * First draft of EQL rules in detection engine * Reorganize functions to separate files * Start adding eventCategoryOverride option for EQL rules * Add building block alerts for each event within sequence * Use eql instead of eql_query for rule type * Remove unused imports * Fix tests * Add basic tests for buildEqlSearchRequest * Add rulesSchema tests for eql * Add buildSignalFromSequence test * Add threat rule fields to buildRuleWithoutOverrides * Fix buildSignalFromSequence typecheck error * Add more tests * Add tests for wrapBuildingBlock and generateSignalId * Use isEqlRule function and fix import error * delete frank * Move sequence interface to types.ts * Fix import * Remove EQL execution placeholder, add back language to eql rule type * allow no indices for eql search * Fix unit tests for language update * Fix buildEqlSearchRequest tests * Replace signal.child with signal.group * remove unused import * Move sequence signal group building to separate testable function * Unbork the merge conflict resolution Co-authored-by: Elastic Machine Co-authored-by: Elastic Machine --- .../detection_engine/get_query_filter.test.ts | 136 +++++++++++- .../detection_engine/get_query_filter.ts | 79 ++++++- .../schemas/common/schemas.ts | 6 + .../request/add_prepackaged_rules_schema.ts | 2 + .../schemas/request/create_rules_schema.ts | 2 + .../schemas/request/import_rules_schema.ts | 2 + .../schemas/request/patch_rules_schema.ts | 2 + .../schemas/request/update_rules_schema.ts | 2 + .../schemas/response/rules_schema.mocks.ts | 13 +- .../schemas/response/rules_schema.test.ts | 33 ++- .../schemas/response/rules_schema.ts | 23 +- .../routes/__mocks__/request_responses.ts | 1 + .../routes/index/signals_mapping.json | 10 + .../routes/rules/create_rules_bulk_route.ts | 2 + .../routes/rules/create_rules_route.ts | 7 +- .../routes/rules/import_rules_route.ts | 3 + .../routes/rules/patch_rules_bulk_route.ts | 2 + .../routes/rules/patch_rules_route.ts | 2 + .../routes/rules/update_rules_bulk_route.ts | 2 + .../routes/rules/update_rules_route.ts | 2 + .../detection_engine/routes/rules/utils.ts | 1 + .../rules/create_rules.mock.ts | 2 + .../detection_engine/rules/create_rules.ts | 2 + .../rules/install_prepacked_rules.ts | 2 + .../rules/patch_rules.mock.ts | 2 + .../lib/detection_engine/rules/patch_rules.ts | 2 + .../lib/detection_engine/rules/types.ts | 4 + .../rules/update_prepacked_rules.ts | 2 + .../rules/update_rules.mock.ts | 2 + .../detection_engine/rules/update_rules.ts | 2 + .../lib/detection_engine/rules/utils.test.ts | 3 + .../lib/detection_engine/rules/utils.ts | 2 + .../scripts/rules/queries/query_eql.json | 68 ++++++ .../signals/__mocks__/es_results.ts | 108 +++++++++- .../signals/build_bulk_body.test.ts | 201 +++++++++++++++++- .../signals/build_bulk_body.ts | 97 ++++++++- .../signals/build_event_type_signal.ts | 4 +- .../signals/build_rule.test.ts | 124 +++++++++-- .../detection_engine/signals/build_rule.ts | 72 ++++++- .../signals/build_signal.test.ts | 90 +++++--- .../detection_engine/signals/build_signal.ts | 10 +- .../detection_engine/signals/get_filter.ts | 4 +- .../signals/signal_params_schema.mock.ts | 1 + .../signals/signal_params_schema.ts | 1 + .../signals/signal_rule_alert_type.ts | 60 +++++- .../signals/single_bulk_create.ts | 62 +++++- .../lib/detection_engine/signals/types.ts | 19 +- .../detection_engine/signals/utils.test.ts | 53 +++++ .../lib/detection_engine/signals/utils.ts | 57 +++++ .../server/lib/detection_engine/types.ts | 2 + .../security_solution/server/lib/types.ts | 55 +++-- 51 files changed, 1337 insertions(+), 108 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_eql.json diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts index 72ef230a42342c..0224caafb41a84 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getQueryFilter, buildExceptionFilter } from './get_query_filter'; +import { getQueryFilter, buildExceptionFilter, buildEqlSearchRequest } from './get_query_filter'; import { Filter, EsQueryConfig } from 'src/plugins/data/public'; import { getExceptionListItemSchemaMock } from '../../../lists/common/schemas/response/exception_list_item_schema.mock'; @@ -1085,4 +1085,138 @@ describe('get_filter', () => { }); }); }); + + describe('buildEqlSearchRequest', () => { + test('should build a basic request with time range', () => { + const request = buildEqlSearchRequest( + 'process where true', + ['testindex1', 'testindex2'], + 'now-5m', + 'now', + 100, + undefined, + [], + undefined + ); + expect(request).toEqual({ + method: 'POST', + path: `/testindex1,testindex2/_eql/search?allow_no_indices=true`, + body: { + size: 100, + query: 'process where true', + filter: { + range: { + '@timestamp': { + gte: 'now-5m', + lte: 'now', + }, + }, + }, + }, + }); + }); + + test('should build a request with timestamp and event category overrides', () => { + const request = buildEqlSearchRequest( + 'process where true', + ['testindex1', 'testindex2'], + 'now-5m', + 'now', + 100, + 'event.ingested', + [], + 'event.other_category' + ); + expect(request).toEqual({ + method: 'POST', + path: `/testindex1,testindex2/_eql/search?allow_no_indices=true`, + event_category_field: 'event.other_category', + body: { + size: 100, + query: 'process where true', + filter: { + range: { + 'event.ingested': { + gte: 'now-5m', + lte: 'now', + }, + }, + }, + }, + }); + }); + + test('should build a request with exceptions', () => { + const request = buildEqlSearchRequest( + 'process where true', + ['testindex1', 'testindex2'], + 'now-5m', + 'now', + 100, + undefined, + [getExceptionListItemSchemaMock()], + undefined + ); + expect(request).toEqual({ + method: 'POST', + path: `/testindex1,testindex2/_eql/search?allow_no_indices=true`, + body: { + size: 100, + query: 'process where true', + filter: { + range: { + '@timestamp': { + gte: 'now-5m', + lte: 'now', + }, + }, + bool: { + must_not: { + bool: { + should: [ + { + bool: { + filter: [ + { + nested: { + path: 'some.parentField', + query: { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.parentField.nested.field': 'some value', + }, + }, + ], + }, + }, + score_mode: 'none', + }, + }, + { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.not.nested.field': 'some value', + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + }, + }, + }, + }); + }); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts index 466a004c14c660..05c706164ab44c 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts @@ -17,7 +17,12 @@ import { CreateExceptionListItemSchema, } from '../../../lists/common/schemas'; import { buildExceptionListQueries } from './build_exceptions_query'; -import { Query as QueryString, Language, Index } from './schemas/common/schemas'; +import { + Query as QueryString, + Language, + Index, + TimestampOverrideOrUndefined, +} from './schemas/common/schemas'; export const getQueryFilter = ( query: QueryString, @@ -67,6 +72,78 @@ export const getQueryFilter = ( return buildEsQuery(indexPattern, initialQuery, enabledFilters, config); }; +interface EqlSearchRequest { + method: string; + path: string; + body: object; + event_category_field?: string; +} + +export const buildEqlSearchRequest = ( + query: string, + index: string[], + from: string, + to: string, + size: number, + timestampOverride: TimestampOverrideOrUndefined, + exceptionLists: ExceptionListItemSchema[], + eventCategoryOverride: string | undefined +): EqlSearchRequest => { + const timestamp = timestampOverride ?? '@timestamp'; + const indexPattern: IIndexPattern = { + fields: [], + title: index.join(), + }; + const config: EsQueryConfig = { + allowLeadingWildcards: true, + queryStringOptions: { analyze_wildcard: true }, + ignoreFilterIfFieldNotInIndex: false, + dateFormatTZ: 'Zulu', + }; + const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists: exceptionLists }); + let exceptionFilter: Filter | undefined; + if (exceptionQueries.length > 0) { + // Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value), + // allowing us to make 1024-item chunks of exception list items. + // Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a + // very conservative value. + exceptionFilter = buildExceptionFilter(exceptionQueries, indexPattern, config, true, 1024); + } + const indexString = index.join(); + const baseRequest = { + method: 'POST', + path: `/${indexString}/_eql/search?allow_no_indices=true`, + body: { + size, + query, + filter: { + range: { + [timestamp]: { + gte: from, + lte: to, + }, + }, + bool: + exceptionFilter !== undefined + ? { + must_not: { + bool: exceptionFilter?.query.bool, + }, + } + : undefined, + }, + }, + }; + if (eventCategoryOverride) { + return { + ...baseRequest, + event_category_field: eventCategoryOverride, + }; + } else { + return baseRequest; + } +}; + export const buildExceptionFilter = ( exceptionQueries: Query[], indexPattern: IIndexPattern, diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts index 5fbba84467ecf4..e8d7f409de20a2 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts @@ -40,6 +40,12 @@ export type Enabled = t.TypeOf; export const enabledOrUndefined = t.union([enabled, t.undefined]); export type EnabledOrUndefined = t.TypeOf; +export const event_category_override = t.string; +export type EventCategoryOverride = t.TypeOf; + +export const eventCategoryOverrideOrUndefined = t.union([event_category_override, t.undefined]); +export type EventCategoryOverrideOrUndefined = t.TypeOf; + export const false_positives = t.array(t.string); export type FalsePositives = t.TypeOf; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts index 69538f025d95dd..3f338c57dd930c 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts @@ -44,6 +44,7 @@ import { Author, RiskScoreMapping, SeverityMapping, + event_category_override, } from '../common/schemas'; import { threat_index, @@ -96,6 +97,7 @@ export const addPrepackagedRulesSchema = t.intersection([ author: DefaultStringArray, // defaults to empty array of strings if not set during decode building_block_type, // defaults to undefined if not set during decode enabled: DefaultBooleanFalse, // defaults to false if not set during decode + event_category_override, // defaults to "undefined" if not set during decode false_positives: DefaultStringArray, // defaults to empty string array if not set during decode filters, // defaults to undefined if not set during decode from: DefaultFromString, // defaults to "now-6m" if not set during decode diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts index c024ba1c48f8d8..2489210a26c8f3 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts @@ -45,6 +45,7 @@ import { Author, RiskScoreMapping, SeverityMapping, + event_category_override, } from '../common/schemas'; import { threat_index, @@ -88,6 +89,7 @@ export const createRulesSchema = t.intersection([ author: DefaultStringArray, // defaults to empty array of strings if not set during decode building_block_type, // defaults to undefined if not set during decode enabled: DefaultBooleanTrue, // defaults to true if not set during decode + event_category_override, // defaults to "undefined" if not set during decode false_positives: DefaultStringArray, // defaults to empty string array if not set during decode filters, // defaults to undefined if not set during decode from: DefaultFromString, // defaults to "now-6m" if not set during decode diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts index b63d70783b7b52..a411b3d439a1f6 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts @@ -51,6 +51,7 @@ import { Author, RiskScoreMapping, SeverityMapping, + event_category_override, } from '../common/schemas'; import { threat_index, @@ -107,6 +108,7 @@ export const importRulesSchema = t.intersection([ author: DefaultStringArray, // defaults to empty array of strings if not set during decode building_block_type, // defaults to undefined if not set during decode enabled: DefaultBooleanTrue, // defaults to true if not set during decode + event_category_override, // defaults to "undefined" if not set during decode false_positives: DefaultStringArray, // defaults to empty string array if not set during decode filters, // defaults to undefined if not set during decode from: DefaultFromString, // defaults to "now-6m" if not set during decode diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts index a674ac86af87bd..40e79d96a9e6b9 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts @@ -46,6 +46,7 @@ import { timestamp_override, risk_score_mapping, severity_mapping, + event_category_override, } from '../common/schemas'; import { listArrayOrUndefined } from '../types/lists'; @@ -65,6 +66,7 @@ export const patchRulesSchema = t.exact( actions, anomaly_threshold, enabled, + event_category_override, false_positives, filters, from, diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts index 1299dada065e15..8a13dd2f4e9087 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts @@ -47,6 +47,7 @@ import { Author, RiskScoreMapping, SeverityMapping, + event_category_override, } from '../common/schemas'; import { @@ -90,6 +91,7 @@ export const updateRulesSchema = t.intersection([ author: DefaultStringArray, // defaults to empty array of strings if not set during decode building_block_type, // defaults to undefined if not set during decode enabled: DefaultBooleanTrue, // defaults to true if not set during decode + event_category_override, false_positives: DefaultStringArray, // defaults to empty string array if not set during decode filters, // defaults to undefined if not set during decode from: DefaultFromString, // defaults to "now-6m" if not set during decode diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts index a462b297d37f84..aaa246c82d9d77 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts @@ -52,7 +52,7 @@ export const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchem severity: 'high', severity_mapping: [], updated_by: 'elastic_kibana', - tags: [], + tags: ['some fake tag 1', 'some fake tag 2'], to: 'now', type: 'query', threat: [], @@ -61,7 +61,7 @@ export const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchem status_date: '2020-02-22T16:47:50.047Z', last_success_at: '2020-02-22T16:47:50.047Z', last_success_message: 'succeeded', - output_index: '.siem-signals-hassanabad-frank-default', + output_index: '.siem-signals-default', max_signals: 100, risk_score: 55, risk_score_mapping: [], @@ -110,3 +110,12 @@ export const getThreatMatchingSchemaMock = (anchorDate: string = ANCHOR_DATE): R ], }; }; + +export const getRulesEqlSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchema => { + return { + ...getRulesSchemaMock(anchorDate), + language: 'eql', + type: 'eql', + query: 'process where true', + }; +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts index 3a47d4af6ac145..c5bad3c55066b5 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts @@ -18,6 +18,7 @@ import { addTimelineTitle, addMlFields, addThreatMatchFields, + addEqlFields, } from './rules_schema'; import { exactCheck } from '../../../exact_check'; import { foldLeftRight, getPaths } from '../../../test_utils'; @@ -26,6 +27,7 @@ import { getRulesSchemaMock, getRulesMlSchemaMock, getThreatMatchingSchemaMock, + getRulesEqlSchemaMock, } from './rules_schema.mocks'; import { ListArray } from '../types/lists'; @@ -628,6 +630,19 @@ describe('rules_schema', () => { ]); expect(message.schema).toEqual({}); }); + + test('it validates an eql rule response', () => { + const payload = getRulesEqlSchemaMock(); + + const dependents = getDependents(payload); + const decoded = dependents.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + const expected = getRulesEqlSchemaMock(); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(expected); + }); }); describe('addSavedId', () => { @@ -668,11 +683,6 @@ describe('rules_schema', () => { expect(fields.length).toEqual(2); }); - test('should return two fields for a rule of type "eql"', () => { - const fields = addQueryFields({ type: 'eql' }); - expect(fields.length).toEqual(2); - }); - test('should return two fields for a rule of type "threshold"', () => { const fields = addQueryFields({ type: 'threshold' }); expect(fields.length).toEqual(2); @@ -757,4 +767,17 @@ describe('rules_schema', () => { expect(fields.length).toEqual(5); }); }); + + describe('addEqlFields', () => { + test('should return empty array if type is not "eql"', () => { + const fields = addEqlFields({ type: 'query' }); + const expected: t.Mixed[] = []; + expect(fields).toEqual(expected); + }); + + test('should return 3 fields for a rule of type "eql"', () => { + const fields = addEqlFields({ type: 'eql' }); + expect(fields.length).toEqual(3); + }); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts index 1c2254f9f8f099..908425a7496d07 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts @@ -16,6 +16,7 @@ import { anomaly_threshold, description, enabled, + event_category_override, false_positives, from, id, @@ -121,6 +122,9 @@ export const dependentRulesSchema = t.partial({ language, query, + // eql only fields + event_category_override, + // when type = saved_query, saved_id is required saved_id, @@ -219,9 +223,7 @@ export const addTimelineTitle = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mi }; export const addQueryFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => { - if ( - ['eql', 'query', 'saved_query', 'threshold', 'threat_match'].includes(typeAndTimelineOnly.type) - ) { + if (['query', 'saved_query', 'threshold', 'threat_match'].includes(typeAndTimelineOnly.type)) { return [ t.exact(t.type({ query: dependentRulesSchema.props.query })), t.exact(t.type({ language: dependentRulesSchema.props.language })), @@ -255,6 +257,20 @@ export const addThresholdFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t. } }; +export const addEqlFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => { + if (typeAndTimelineOnly.type === 'eql') { + return [ + t.exact( + t.partial({ event_category_override: dependentRulesSchema.props.event_category_override }) + ), + t.exact(t.type({ query: dependentRulesSchema.props.query })), + t.exact(t.type({ language: dependentRulesSchema.props.language })), + ]; + } else { + return []; + } +}; + export const addThreatMatchFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => { if (typeAndTimelineOnly.type === 'threat_match') { return [ @@ -278,6 +294,7 @@ export const getDependents = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed ...addQueryFields(typeAndTimelineOnly), ...addMlFields(typeAndTimelineOnly), ...addThresholdFields(typeAndTimelineOnly), + ...addEqlFields(typeAndTimelineOnly), ...addThreatMatchFields(typeAndTimelineOnly), ]; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts index fb01f922555168..5d9cfb4bb44928 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts @@ -348,6 +348,7 @@ export const getResult = (): RuleAlertType => ({ description: 'Detecting root and admin users', ruleId: 'rule-1', index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + eventCategoryOverride: undefined, falsePositives: [], from: 'now-6m', immutable: false, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json index cfce0199100714..7255325358baf6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json @@ -60,6 +60,16 @@ } } }, + "group": { + "properties": { + "id": { + "type": "keyword" + }, + "index": { + "type": "integer" + } + } + }, "rule": { "properties": { "id": { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts index dd887233c36a31..067a4352e10809 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts @@ -69,6 +69,7 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query: queryOrUndefined, @@ -153,6 +154,7 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, immutable: false, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts index 26ab89ad8ea7cd..54df87ca17787d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { isEqlRule } from '../../../../../common/detection_engine/utils'; import { createRuleValidateTypeDependents } from '../../../../../common/detection_engine/schemas/request/create_rules_type_dependents'; import { RuleAlertAction } from '../../../../../common/detection_engine/types'; import { buildRouteValidation } from '../../../../utils/build_validation/route_validation'; @@ -53,6 +54,7 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query: queryOrUndefined, @@ -94,7 +96,9 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void const query = !isMlRule(type) && queryOrUndefined == null ? '' : queryOrUndefined; const language = - !isMlRule(type) && languageOrUndefined == null ? 'kuery' : languageOrUndefined; + !isMlRule(type) && !isEqlRule(type) && languageOrUndefined == null + ? 'kuery' + : languageOrUndefined; // TODO: Fix these either with an is conversion or by better typing them within io-ts const actions: RuleAlertAction[] = actionsRest as RuleAlertAction[]; @@ -138,6 +142,7 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, immutable: false, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts index 0f5d0304f5ca02..4dbca5df0041cf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts @@ -135,6 +135,7 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, immutable, @@ -194,6 +195,7 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, immutable, @@ -242,6 +244,7 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP savedObjectsClient, description, enabled, + eventCategoryOverride, falsePositives, from, query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts index 5099cf5de958fe..39bbe9ee686a49 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts @@ -59,6 +59,7 @@ export const patchRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query, @@ -119,6 +120,7 @@ export const patchRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts index 3b3efd2ed166dc..879bd8d5b8a1d1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts @@ -50,6 +50,7 @@ export const patchRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query, @@ -117,6 +118,7 @@ export const patchRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts index 0e414e130849a5..4df0773f86317a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts @@ -62,6 +62,7 @@ export const updateRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query: queryOrUndefined, @@ -127,6 +128,7 @@ export const updateRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts index 553d084b626337..ef698db008d804 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts @@ -52,6 +52,7 @@ export const updateRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query: queryOrUndefined, @@ -117,6 +118,7 @@ export const updateRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts index 556ea209152e6c..c75b32b614e078 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts @@ -114,6 +114,7 @@ export const transformAlertToRule = ( description: alert.params.description, enabled: alert.enabled, anomaly_threshold: alert.params.anomalyThreshold, + event_category_override: alert.params.eventCategoryOverride, false_positives: alert.params.falsePositives, filters: alert.params.filters, from: alert.params.from, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts index 95067e57868d14..a6034f3d7b7b37 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts @@ -14,6 +14,7 @@ export const getCreateRulesOptionsMock = (): CreateRulesOptions => ({ anomalyThreshold: undefined, description: 'some description', enabled: true, + eventCategoryOverride: undefined, falsePositives: ['false positive 1', 'false positive 2'], from: 'now-6m', query: 'user.name: root or user.name: admin', @@ -61,6 +62,7 @@ export const getCreateMlRulesOptionsMock = (): CreateRulesOptions => ({ anomalyThreshold: 55, description: 'some description', enabled: true, + eventCategoryOverride: undefined, falsePositives: ['false positive 1', 'false positive 2'], from: 'now-6m', query: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts index 9ed94cd7bff2e6..3a311d03e3c89f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts @@ -17,6 +17,7 @@ export const createRules = async ({ buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, query, @@ -69,6 +70,7 @@ export const createRules = async ({ description, ruleId, index, + eventCategoryOverride, falsePositives, from, immutable, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts index 59e14dcffc3c01..38adc03c00d502 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts @@ -22,6 +22,7 @@ export const installPrepackagedRules = ( building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query, @@ -70,6 +71,7 @@ export const installPrepackagedRules = ( buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, immutable: true, // At the moment we force all prepackaged rules to be immutable diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts index cfb40056eb85d1..aeb136a969aa18 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts @@ -120,6 +120,7 @@ export const getPatchRulesOptionsMock = (): PatchRulesOptions => ({ anomalyThreshold: undefined, description: 'some description', enabled: true, + eventCategoryOverride: undefined, falsePositives: ['false positive 1', 'false positive 2'], from: 'now-6m', query: 'user.name: root or user.name: admin', @@ -163,6 +164,7 @@ export const getPatchMlRulesOptionsMock = (): PatchRulesOptions => ({ anomalyThreshold: 55, description: 'some description', enabled: true, + eventCategoryOverride: undefined, falsePositives: ['false positive 1', 'false positive 2'], from: 'now-6m', query: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts index e0814647b4c39a..852ff06bdc736c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts @@ -18,6 +18,7 @@ export const patchRules = async ({ buildingBlockType, savedObjectsClient, description, + eventCategoryOverride, falsePositives, enabled, query, @@ -62,6 +63,7 @@ export const patchRules = async ({ author, buildingBlockType, description, + eventCategoryOverride, falsePositives, query, language, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts index 6b851351f27f25..d688e1b338e21b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts @@ -84,6 +84,7 @@ import { TimestampOverrideOrUndefined, BuildingBlockTypeOrUndefined, RuleNameOverrideOrUndefined, + EventCategoryOverrideOrUndefined, } from '../../../../common/detection_engine/schemas/common/schemas'; import { ThreatIndexOrUndefined, @@ -187,6 +188,7 @@ export interface CreateRulesOptions { buildingBlockType: BuildingBlockTypeOrUndefined; description: Description; enabled: Enabled; + eventCategoryOverride: EventCategoryOverrideOrUndefined; falsePositives: FalsePositives; from: From; query: QueryOrUndefined; @@ -236,6 +238,7 @@ export interface UpdateRulesOptions { buildingBlockType: BuildingBlockTypeOrUndefined; description: Description; enabled: Enabled; + eventCategoryOverride: EventCategoryOverrideOrUndefined; falsePositives: FalsePositives; from: From; query: QueryOrUndefined; @@ -279,6 +282,7 @@ export interface PatchRulesOptions { buildingBlockType: BuildingBlockTypeOrUndefined; description: DescriptionOrUndefined; enabled: EnabledOrUndefined; + eventCategoryOverride: EventCategoryOverrideOrUndefined; falsePositives: FalsePositivesOrUndefined; from: FromOrUndefined; query: QueryOrUndefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts index bf97784e8d917e..01a481ed7b2d9b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts @@ -23,6 +23,7 @@ export const updatePrepackagedRules = async ( author, building_block_type: buildingBlockType, description, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query, @@ -69,6 +70,7 @@ export const updatePrepackagedRules = async ( author, buildingBlockType, description, + eventCategoryOverride, falsePositives, from, query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts index 650b59fb85bc03..8cdc904a861c7a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts @@ -17,6 +17,7 @@ export const getUpdateRulesOptionsMock = (): UpdateRulesOptions => ({ anomalyThreshold: undefined, description: 'some description', enabled: true, + eventCategoryOverride: undefined, falsePositives: ['false positive 1', 'false positive 2'], from: 'now-6m', query: 'user.name: root or user.name: admin', @@ -61,6 +62,7 @@ export const getUpdateMlRulesOptionsMock = (): UpdateRulesOptions => ({ anomalyThreshold: 55, description: 'some description', enabled: true, + eventCategoryOverride: undefined, falsePositives: ['false positive 1', 'false positive 2'], from: 'now-6m', query: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts index 494a4e221d8629..08df785884b76b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts @@ -18,6 +18,7 @@ export const updateRules = async ({ buildingBlockType, savedObjectsClient, description, + eventCategoryOverride, falsePositives, enabled, query, @@ -64,6 +65,7 @@ export const updateRules = async ({ author, buildingBlockType, description, + eventCategoryOverride, falsePositives, query, language, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts index 17505a44782618..227f574bc4e4b4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts @@ -31,6 +31,7 @@ describe('utils', () => { author: [], buildingBlockType: undefined, description: 'some description change', + eventCategoryOverride: undefined, falsePositives: undefined, query: undefined, language: undefined, @@ -73,6 +74,7 @@ describe('utils', () => { author: [], buildingBlockType: undefined, description: 'some description change', + eventCategoryOverride: undefined, falsePositives: undefined, query: undefined, language: undefined, @@ -115,6 +117,7 @@ describe('utils', () => { author: [], buildingBlockType: undefined, description: 'some description change', + eventCategoryOverride: undefined, falsePositives: undefined, query: undefined, language: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts index 49c02f92ff3361..d9f953f2803a61 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts @@ -39,6 +39,7 @@ import { RuleNameOverrideOrUndefined, SeverityMappingOrUndefined, TimestampOverrideOrUndefined, + EventCategoryOverrideOrUndefined, } from '../../../../common/detection_engine/schemas/common/schemas'; import { PartialFilter } from '../types'; import { ListArrayOrUndefined } from '../../../../common/detection_engine/schemas/types'; @@ -60,6 +61,7 @@ export interface UpdateProperties { author: AuthorOrUndefined; buildingBlockType: BuildingBlockTypeOrUndefined; description: DescriptionOrUndefined; + eventCategoryOverride: EventCategoryOverrideOrUndefined; falsePositives: FalsePositivesOrUndefined; from: FromOrUndefined; query: QueryOrUndefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_eql.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_eql.json new file mode 100644 index 00000000000000..598f2182002c1d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_eql.json @@ -0,0 +1,68 @@ +{ + "name": "EQL query rule", + "description": "Rule with an eql query", + "false_positives": [ + "https://www.example.com/some-article-about-a-false-positive", + "some text string about why another condition could be a false positive" + ], + "rule_id": "rule-id-eql", + "enabled": false, + "index": [".ds-logs-endpoint.events.process-default-000001"], + "interval": "30s", + "query": "sequence [process where process.name = \"mimikatz.exe\"] [process where process.name = \"explorer.exe\"]", + "output_index": ".siem-signals-default", + "meta": { + "anything_you_want_ui_related_or_otherwise": { + "as_deep_structured_as_you_need": { + "any_data_type": {} + } + } + }, + "risk_score": 1, + "max_signals": 100, + "tags": ["tag 1", "tag 2", "any tag you want"], + "to": "now", + "from": "now-300m", + "severity": "high", + "type": "eql", + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0040", + "name": "impact", + "reference": "https://attack.mitre.org/tactics/TA0040/" + }, + "technique": [ + { + "id": "T1499", + "name": "endpoint denial of service", + "reference": "https://attack.mitre.org/techniques/T1499/" + } + ] + }, + { + "framework": "Some other Framework you want", + "tactic": { + "id": "some-other-id", + "name": "Some other name", + "reference": "https://example.com" + }, + "technique": [ + { + "id": "some-other-id", + "name": "some other technique name", + "reference": "https://example.com" + } + ] + } + ], + "references": [ + "http://www.example.com/some-article-about-attack", + "Some plain text string here explaining why this is a valid thing to look out for" + ], + "timeline_id": "timeline_id", + "timeline_title": "timeline_title", + "note": "# note markdown", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 9ee8c5cf298a16..b37bc7d0fab69c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -4,7 +4,14 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SignalSourceHit, SignalSearchResponse, BulkResponse, BulkItem } from '../types'; +import { + SignalSourceHit, + SignalSearchResponse, + BulkResponse, + BulkItem, + RuleAlertAttributes, + SignalHit, +} from '../types'; import { Logger, SavedObject, @@ -24,6 +31,7 @@ export const sampleRuleAlertParams = ( buildingBlockType: 'default', ruleId: 'rule-1', description: 'Detecting root and admin users', + eventCategoryOverride: undefined, falsePositives: [], immutable: false, index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], @@ -60,6 +68,30 @@ export const sampleRuleAlertParams = ( exceptionsList: getListArrayMock(), }); +export const sampleRuleSO = (): SavedObject => { + return { + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'alert', + version: '1', + updated_at: '2020-03-27T22:55:59.577Z', + attributes: { + actions: [], + enabled: true, + name: 'rule-name', + tags: ['some fake tag 1', 'some fake tag 2'], + createdBy: 'sample user', + createdAt: '2020-03-27T22:55:59.577Z', + updatedBy: 'sample user', + schedule: { + interval: '5m', + }, + throttle: 'no_actions', + params: sampleRuleAlertParams(), + }, + references: [], + }; +}; + export const sampleDocNoSortIdNoVersion = (someUuid: string = sampleIdGuid): SignalSourceHit => ({ _index: 'myFakeSignalIndex', _type: 'doc', @@ -189,6 +221,80 @@ export const sampleDocWithAncestors = (): SignalSearchResponse => { }; }; +export const sampleSignalHit = (): SignalHit => ({ + '@timestamp': '2020-04-20T21:27:45+0000', + event: { + kind: 'signal', + }, + signal: { + parents: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + { + id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + ancestors: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + { + id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + status: 'open', + rule: { + author: [], + id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + created_at: '2020-04-20T21:27:45+0000', + updated_at: '2020-04-20T21:27:45+0000', + created_by: 'elastic', + description: 'some description', + enabled: true, + false_positives: ['false positive 1', 'false positive 2'], + from: 'now-6m', + immutable: false, + name: 'Query with a rule id', + query: 'user.name: root or user.name: admin', + references: ['test 1', 'test 2'], + severity: 'high', + severity_mapping: [], + updated_by: 'elastic_kibana', + tags: ['some fake tag 1', 'some fake tag 2'], + to: 'now', + type: 'query', + threat: [], + version: 1, + status: 'succeeded', + status_date: '2020-02-22T16:47:50.047Z', + last_success_at: '2020-02-22T16:47:50.047Z', + last_success_message: 'succeeded', + output_index: '.siem-signals-default', + max_signals: 100, + risk_score: 55, + risk_score_mapping: [], + language: 'kuery', + rule_id: 'query-rule-id', + interval: '5m', + exceptions_list: getListArrayMock(), + }, + depth: 1, + }, +}); + export const sampleBulkCreateDuplicateResult = { took: 60, errors: true, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index 967dc5331e46b1..f45a408cd32b8e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -9,8 +9,10 @@ import { sampleDocNoSortId, sampleRuleGuid, sampleIdGuid, + sampleDocWithAncestors, + sampleRuleSO, } from './__mocks__/es_results'; -import { buildBulkBody } from './build_bulk_body'; +import { buildBulkBody, buildSignalFromSequence, buildSignalFromEvent } from './build_bulk_body'; import { SignalHit } from './types'; import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; @@ -433,3 +435,200 @@ describe('buildBulkBody', () => { expect(fakeSignalSourceHit).toEqual(expected); }); }); + +describe('buildSignalFromSequence', () => { + test('builds a basic signal from a sequence of building blocks', () => { + const blocks = [sampleDocWithAncestors().hits.hits[0], sampleDocWithAncestors().hits.hits[0]]; + const ruleSO = sampleRuleSO(); + const signal = buildSignalFromSequence(blocks, ruleSO); + // Timestamp will potentially always be different so remove it for the test + // @ts-expect-error + delete signal['@timestamp']; + const expected: Omit = { + event: { + kind: 'signal', + }, + signal: { + parents: [ + { + id: sampleIdGuid, + rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + { + id: sampleIdGuid, + rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + ], + ancestors: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + { + id: sampleIdGuid, + rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + { + id: sampleIdGuid, + rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + ], + status: 'open', + rule: { + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'sample user', + updated_by: 'sample user', + version: 1, + updated_at: ruleSO.updated_at ?? '', + created_at: ruleSO.attributes.createdAt, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + }, + depth: 2, + group: { + id: '269c1f5754bff92fb8040283b687258e99b03e8b2ab1262cc20c82442e5de5ea', + }, + }, + }; + expect(signal).toEqual(expected); + }); +}); + +describe('buildSignalFromEvent', () => { + test('builds a basic signal from a single event', () => { + const ancestor = sampleDocWithAncestors().hits.hits[0]; + delete ancestor._source.source; + const ruleSO = sampleRuleSO(); + const signal = buildSignalFromEvent(ancestor, ruleSO); + // Timestamp will potentially always be different so remove it for the test + // @ts-expect-error + delete signal['@timestamp']; + const expected: Omit & { someKey: 'someValue' } = { + someKey: 'someValue', + event: { + kind: 'signal', + }, + signal: { + original_time: '2020-04-20T21:27:45+0000', + parent: { + id: sampleIdGuid, + rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + parents: [ + { + id: sampleIdGuid, + rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + ], + ancestors: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + { + id: sampleIdGuid, + rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + ], + status: 'open', + rule: { + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'sample user', + updated_by: 'sample user', + version: 1, + updated_at: ruleSO.updated_at ?? '', + created_at: ruleSO.attributes.createdAt, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + }, + depth: 2, + }, + }; + expect(signal).toEqual(expected); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index 7be97e46f91f24..01a6b0e7aefadb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -4,12 +4,22 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SignalSourceHit, SignalHit, Signal } from './types'; -import { buildRule } from './build_rule'; +import { SavedObject } from 'src/core/types'; +import { + SignalSourceHit, + SignalHit, + Signal, + RuleAlertAttributes, + BaseSignalHit, + SignalSource, +} from './types'; +import { buildRule, buildRuleWithoutOverrides } from './build_rule'; import { additionalSignalFields, buildSignal } from './build_signal'; import { buildEventTypeSignal } from './build_event_type_signal'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams } from '../types'; +import { generateSignalId, wrapBuildingBlocks, wrapSignal } from './utils'; +import { EqlSequence } from '../../types'; interface BuildBulkBodyParams { doc: SignalSourceHit; @@ -71,3 +81,86 @@ export const buildBulkBody = ({ }; return signalHit; }; + +/** + * Takes N raw documents from ES that form a sequence and builds them into N+1 signals ready to be indexed - + * one signal for each event in the sequence, and a "shell" signal that ties them all together. All N+1 signals + * share the same signal.group.id to make it easy to query them. + * @param sequence The raw ES documents that make up the sequence + * @param ruleSO SavedObject representing the rule that found the sequence + * @param outputIndex Index to write the resulting signals to + */ +export const buildSignalGroupFromSequence = ( + sequence: EqlSequence, + ruleSO: SavedObject, + outputIndex: string +): BaseSignalHit[] => { + const wrappedBuildingBlocks = wrapBuildingBlocks( + sequence.events.map((event) => { + const signal = buildSignalFromEvent(event, ruleSO); + signal.signal.rule.building_block_type = 'default'; + return signal; + }), + outputIndex + ); + + // Now that we have an array of building blocks for the events in the sequence, + // we can build the signal that links the building blocks together + // and also insert the group id (which is also the "shell" signal _id) in each building block + const sequenceSignal = wrapSignal( + buildSignalFromSequence(wrappedBuildingBlocks, ruleSO), + outputIndex + ); + wrappedBuildingBlocks.forEach((block, idx) => { + // TODO: fix type of blocks so we don't have to check existence of _source.signal + if (block._source.signal) { + block._source.signal.group = { + id: sequenceSignal._id, + index: idx, + }; + } + }); + return [...wrappedBuildingBlocks, sequenceSignal]; +}; + +export const buildSignalFromSequence = ( + events: BaseSignalHit[], + ruleSO: SavedObject +): SignalHit => { + const rule = buildRuleWithoutOverrides(ruleSO); + const signal: Signal = buildSignal(events, rule); + return { + '@timestamp': new Date().toISOString(), + event: { + kind: 'signal', + }, + signal: { + ...signal, + group: { + // This is the same function that is used later to generate the _id for the sequence signal document, + // so _id should equal signal.group.id for the "shell" document + id: generateSignalId(signal), + }, + }, + }; +}; + +export const buildSignalFromEvent = ( + event: BaseSignalHit, + ruleSO: SavedObject +): SignalHit => { + const rule = buildRuleWithoutOverrides(ruleSO); + const signal = { + ...buildSignal([event], rule), + ...additionalSignalFields(event), + }; + const eventFields = buildEventTypeSignal(event); + // TODO: better naming for SignalHit - it's really a new signal to be inserted + const signalHit: SignalHit = { + ...event._source, + '@timestamp': new Date().toISOString(), + event: eventFields, + signal, + }; + return signalHit; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts index 59cdc020c611d4..81c9d1dedcc56c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts @@ -4,9 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SignalSourceHit } from './types'; +import { BaseSignalHit } from './types'; -export const buildEventTypeSignal = (doc: SignalSourceHit): object => { +export const buildEventTypeSignal = (doc: BaseSignalHit): object => { if (doc._source.event != null && doc._source.event instanceof Object) { return { ...doc._source.event, kind: 'signal' }; } else { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts index ba815a0b62f0d3..62e5854037d9e1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts @@ -4,12 +4,17 @@ * you may not use this file except in compliance with the Elastic License. */ -import { buildRule, removeInternalTagsFromRule } from './build_rule'; -import { sampleDocNoSortId, sampleRuleAlertParams, sampleRuleGuid } from './__mocks__/es_results'; +import { buildRule, removeInternalTagsFromRule, buildRuleWithoutOverrides } from './build_rule'; +import { + sampleDocNoSortId, + sampleRuleAlertParams, + sampleRuleGuid, + sampleRuleSO, +} from './__mocks__/es_results'; import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; import { INTERNAL_RULE_ID_KEY, INTERNAL_IMMUTABLE_KEY } from '../../../../common/constants'; -import { getPartialRulesSchemaMock } from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; +import { getRulesSchemaMock } from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; describe('buildRule', () => { beforeEach(() => { @@ -272,9 +277,11 @@ describe('buildRule', () => { }; expect(rule).toEqual(expected); }); +}); +describe('removeInternalTagsFromRule', () => { test('it removes internal tags from a typical rule', () => { - const rule = getPartialRulesSchemaMock(); + const rule = getRulesSchemaMock(); rule.tags = [ 'some fake tag 1', 'some fake tag 2', @@ -282,30 +289,113 @@ describe('buildRule', () => { `${INTERNAL_IMMUTABLE_KEY}:true`, ]; const noInternals = removeInternalTagsFromRule(rule); - expect(noInternals).toEqual(getPartialRulesSchemaMock()); + expect(noInternals).toEqual(getRulesSchemaMock()); }); test('it works with an empty array', () => { - const rule = getPartialRulesSchemaMock(); + const rule = getRulesSchemaMock(); rule.tags = []; const noInternals = removeInternalTagsFromRule(rule); - const expected = getPartialRulesSchemaMock(); + const expected = getRulesSchemaMock(); expected.tags = []; expect(noInternals).toEqual(expected); }); - test('it works if tags does not exist', () => { - const rule = getPartialRulesSchemaMock(); - delete rule.tags; - const noInternals = removeInternalTagsFromRule(rule); - const expected = getPartialRulesSchemaMock(); - delete expected.tags; - expect(noInternals).toEqual(expected); - }); - test('it works if tags contains normal values and no internal values', () => { - const rule = getPartialRulesSchemaMock(); + const rule = getRulesSchemaMock(); const noInternals = removeInternalTagsFromRule(rule); expect(noInternals).toEqual(rule); }); }); + +describe('buildRuleWithoutOverrides', () => { + test('builds a rule using rule SO', () => { + const ruleSO = sampleRuleSO(); + const rule = buildRuleWithoutOverrides(ruleSO); + expect(rule).toEqual({ + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'sample user', + updated_by: 'sample user', + version: 1, + updated_at: ruleSO.updated_at ?? '', + created_at: ruleSO.attributes.createdAt, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + }); + }); + + test('builds a rule using rule SO and removes internal tags', () => { + const ruleSO = sampleRuleSO(); + ruleSO.attributes.tags = [ + 'some fake tag 1', + 'some fake tag 2', + `${INTERNAL_RULE_ID_KEY}:rule-1`, + `${INTERNAL_IMMUTABLE_KEY}:true`, + ]; + const rule = buildRuleWithoutOverrides(ruleSO); + expect(rule).toEqual({ + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'sample user', + updated_by: 'sample user', + version: 1, + updated_at: ruleSO.updated_at ?? '', + created_at: ruleSO.attributes.createdAt, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts index aacf9b8be31b41..e5370735333bc7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts @@ -4,12 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { pickBy } from 'lodash/fp'; +import { SavedObject } from 'src/core/types'; import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams } from '../types'; import { buildRiskScoreFromMapping } from './mappings/build_risk_score_from_mapping'; -import { SignalSourceHit } from './types'; +import { SignalSourceHit, RuleAlertAttributes } from './types'; import { buildSeverityFromMapping } from './mappings/build_severity_from_mapping'; import { buildRuleNameFromMapping } from './mappings/build_rule_name_from_mapping'; import { INTERNAL_IDENTIFIER } from '../../../../common/constants'; @@ -44,7 +44,7 @@ export const buildRule = ({ interval, tags, throttle, -}: BuildRuleParams): Partial => { +}: BuildRuleParams): RulesSchema => { const { riskScore, riskScoreMeta } = buildRiskScoreFromMapping({ doc, riskScore: ruleParams.riskScore, @@ -65,7 +65,7 @@ export const buildRule = ({ const meta = { ...ruleParams.meta, ...riskScoreMeta, ...severityMeta, ...ruleNameMeta }; - const rule = pickBy((value: unknown) => value != null, { + const rule = { id, rule_id: ruleParams.ruleId ?? '(unknown rule_id)', actions, @@ -111,15 +111,73 @@ export const buildRule = ({ machine_learning_job_id: ruleParams.machineLearningJobId, anomaly_threshold: ruleParams.anomalyThreshold, threshold: ruleParams.threshold, - }); + }; + return removeInternalTagsFromRule(rule); +}; + +export const buildRuleWithoutOverrides = ( + ruleSO: SavedObject +): RulesSchema => { + const ruleParams = ruleSO.attributes.params; + const rule: RulesSchema = { + id: ruleSO.id, + rule_id: ruleParams.ruleId, + actions: ruleSO.attributes.actions, + author: ruleParams.author ?? [], + building_block_type: ruleParams.buildingBlockType, + false_positives: ruleParams.falsePositives, + saved_id: ruleParams.savedId, + timeline_id: ruleParams.timelineId, + timeline_title: ruleParams.timelineTitle, + meta: ruleParams.meta, + max_signals: ruleParams.maxSignals, + risk_score: ruleParams.riskScore, + risk_score_mapping: ruleParams.riskScoreMapping ?? [], + output_index: ruleParams.outputIndex, + description: ruleParams.description, + note: ruleParams.note, + from: ruleParams.from, + immutable: ruleParams.immutable, + index: ruleParams.index, + interval: ruleSO.attributes.schedule.interval, + language: ruleParams.language, + license: ruleParams.license, + name: ruleSO.attributes.name, + query: ruleParams.query, + references: ruleParams.references, + rule_name_override: ruleParams.ruleNameOverride, + severity: ruleParams.severity, + severity_mapping: ruleParams.severityMapping ?? [], + tags: ruleSO.attributes.tags, + type: ruleParams.type, + to: ruleParams.to, + enabled: ruleSO.attributes.enabled, + filters: ruleParams.filters, + created_by: ruleSO.attributes.createdBy, + updated_by: ruleSO.attributes.updatedBy, + threat: ruleParams.threat ?? [], + timestamp_override: ruleParams.timestampOverride, // TODO: Timestamp Override via timestamp_override + throttle: ruleSO.attributes.throttle, + version: ruleParams.version, + created_at: ruleSO.attributes.createdAt, + updated_at: ruleSO.updated_at ?? '', + exceptions_list: ruleParams.exceptionsList ?? [], + machine_learning_job_id: ruleParams.machineLearningJobId, + anomaly_threshold: ruleParams.anomalyThreshold, + threshold: ruleParams.threshold, + threat_filters: ruleParams.threatFilters, + threat_index: ruleParams.threatIndex, + threat_query: ruleParams.threatQuery, + threat_mapping: ruleParams.threatMapping, + }; return removeInternalTagsFromRule(rule); }; -export const removeInternalTagsFromRule = (rule: Partial): Partial => { +export const removeInternalTagsFromRule = (rule: RulesSchema): RulesSchema => { if (rule.tags == null) { return rule; } else { - const ruleWithoutInternalTags: Partial = { + const ruleWithoutInternalTags: RulesSchema = { ...rule, tags: rule.tags.filter((tag) => !tag.startsWith(INTERNAL_IDENTIFIER)), }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts index d684807a09126f..d0c451bbdf2e2b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts @@ -7,7 +7,11 @@ import { sampleDocNoSortId } from './__mocks__/es_results'; import { buildSignal, buildParent, buildAncestors, additionalSignalFields } from './build_signal'; import { Signal, Ancestor } from './types'; -import { getPartialRulesSchemaMock } from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; +import { + getRulesSchemaMock, + ANCHOR_DATE, +} from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; +import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; describe('buildSignal', () => { beforeEach(() => { @@ -17,7 +21,7 @@ describe('buildSignal', () => { test('it builds a signal as expected without original_event if event does not exist', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); delete doc._source.event; - const rule = getPartialRulesSchemaMock(); + const rule = getRulesSchemaMock(); const signal = { ...buildSignal([doc], rule), ...additionalSignalFields(doc), @@ -48,31 +52,39 @@ describe('buildSignal', () => { original_time: '2020-04-20T21:27:45+0000', status: 'open', rule: { + author: [], + id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + created_at: new Date(ANCHOR_DATE).toISOString(), + updated_at: new Date(ANCHOR_DATE).toISOString(), created_by: 'elastic', - description: 'Detecting root and admin users', + description: 'some description', enabled: true, - false_positives: [], + false_positives: ['false positive 1', 'false positive 2'], from: 'now-6m', - id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', immutable: false, - index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], - interval: '5m', - risk_score: 50, - rule_id: 'rule-1', - language: 'kuery', - max_signals: 100, - name: 'Detect Root/Admin Users', - output_index: '.siem-signals', + name: 'Query with a rule id', query: 'user.name: root or user.name: admin', - references: ['http://www.example.com', 'https://ww.example.com'], + references: ['test 1', 'test 2'], severity: 'high', - updated_by: 'elastic', + severity_mapping: [], + updated_by: 'elastic_kibana', tags: ['some fake tag 1', 'some fake tag 2'], to: 'now', type: 'query', - note: '', - updated_at: signal.rule.updated_at, - created_at: signal.rule.created_at, + threat: [], + version: 1, + status: 'succeeded', + status_date: '2020-02-22T16:47:50.047Z', + last_success_at: '2020-02-22T16:47:50.047Z', + last_success_message: 'succeeded', + output_index: '.siem-signals-default', + max_signals: 100, + risk_score: 55, + risk_score_mapping: [], + language: 'kuery', + rule_id: 'query-rule-id', + interval: '5m', + exceptions_list: getListArrayMock(), }, depth: 1, }; @@ -87,7 +99,7 @@ describe('buildSignal', () => { kind: 'event', module: 'system', }; - const rule = getPartialRulesSchemaMock(); + const rule = getRulesSchemaMock(); const signal = { ...buildSignal([doc], rule), ...additionalSignalFields(doc), @@ -124,31 +136,39 @@ describe('buildSignal', () => { }, status: 'open', rule: { + author: [], + id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + created_at: new Date(ANCHOR_DATE).toISOString(), + updated_at: new Date(ANCHOR_DATE).toISOString(), created_by: 'elastic', - description: 'Detecting root and admin users', + description: 'some description', enabled: true, - false_positives: [], + false_positives: ['false positive 1', 'false positive 2'], from: 'now-6m', - id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', immutable: false, - index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], - interval: '5m', - risk_score: 50, - rule_id: 'rule-1', - language: 'kuery', - max_signals: 100, - name: 'Detect Root/Admin Users', - output_index: '.siem-signals', + name: 'Query with a rule id', query: 'user.name: root or user.name: admin', - references: ['http://www.example.com', 'https://ww.example.com'], + references: ['test 1', 'test 2'], severity: 'high', - updated_by: 'elastic', + severity_mapping: [], + updated_by: 'elastic_kibana', tags: ['some fake tag 1', 'some fake tag 2'], to: 'now', type: 'query', - note: '', - updated_at: signal.rule.updated_at, - created_at: signal.rule.created_at, + threat: [], + version: 1, + status: 'succeeded', + status_date: '2020-02-22T16:47:50.047Z', + last_success_at: '2020-02-22T16:47:50.047Z', + last_success_message: 'succeeded', + output_index: '.siem-signals-default', + max_signals: 100, + risk_score: 55, + risk_score_mapping: [], + language: 'kuery', + rule_id: 'query-rule-id', + interval: '5m', + exceptions_list: getListArrayMock(), }, depth: 1, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index 78818779dd661c..947938de6caca6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -5,14 +5,14 @@ */ import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; -import { SignalSourceHit, Signal, Ancestor } from './types'; +import { Signal, Ancestor, BaseSignalHit } from './types'; /** * Takes a parent signal or event document and extracts the information needed for the corresponding entry in the child * signal's `signal.parents` array. * @param doc The parent signal or event */ -export const buildParent = (doc: SignalSourceHit): Ancestor => { +export const buildParent = (doc: BaseSignalHit): Ancestor => { if (doc._source.signal != null) { return { rule: doc._source.signal.rule.id, @@ -38,7 +38,7 @@ export const buildParent = (doc: SignalSourceHit): Ancestor => { * creating an array of N+1 ancestors. * @param doc The parent signal/event for which to extend the ancestry. */ -export const buildAncestors = (doc: SignalSourceHit): Ancestor[] => { +export const buildAncestors = (doc: BaseSignalHit): Ancestor[] => { const newAncestor = buildParent(doc); const existingAncestors = doc._source.signal?.ancestors; if (existingAncestors != null) { @@ -53,7 +53,7 @@ export const buildAncestors = (doc: SignalSourceHit): Ancestor[] => { * @param docs The parent signals/events of the new signal to be built. * @param rule The rule that is generating the new signal. */ -export const buildSignal = (docs: SignalSourceHit[], rule: Partial): Signal => { +export const buildSignal = (docs: BaseSignalHit[], rule: RulesSchema): Signal => { const parents = docs.map(buildParent); const depth = parents.reduce((acc, parent) => Math.max(parent.depth, acc), 0) + 1; const ancestors = docs.reduce((acc: Ancestor[], doc) => acc.concat(buildAncestors(doc)), []); @@ -70,7 +70,7 @@ export const buildSignal = (docs: SignalSourceHit[], rule: Partial) * Creates signal fields that are only available in the special case where a signal has only 1 parent signal/event. * @param doc The parent signal/event of the new signal to be built. */ -export const additionalSignalFields = (doc: SignalSourceHit) => { +export const additionalSignalFields = (doc: BaseSignalHit) => { return { parent: buildParent(doc), original_time: doc._source['@timestamp'], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts index 6ce0be54a9e7b6..522f4bfa5ef982 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts @@ -90,7 +90,6 @@ export const getFilter = async ({ }; switch (type) { - case 'eql': case 'threat_match': case 'threshold': { return savedId != null ? savedQueryFilter() : queryFilter(); @@ -106,6 +105,9 @@ export const getFilter = async ({ 'Unsupported Rule of type "machine_learning" supplied to getFilter' ); } + case 'eql': { + throw new BadRequestError('Unsupported Rule of type "eql" supplied to getFilter'); + } default: { return assertUnreachable(type); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts index c8f8341392553e..922fadb13a298c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts @@ -22,6 +22,7 @@ export const getSignalParamsSchemaDecodedMock = (): SignalParamsSchema => ({ author: [], buildingBlockType: null, description: 'Detecting root and admin users', + eventCategoryOverride: undefined, falsePositives: [], filters: null, from: 'now-6m', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts index dbb48d59d3a3f9..4006345b243856 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts @@ -14,6 +14,7 @@ const signalSchema = schema.object({ buildingBlockType: schema.nullable(schema.string()), description: schema.string(), note: schema.nullable(schema.string()), + eventCategoryOverride: schema.maybe(schema.string()), falsePositives: schema.arrayOf(schema.string(), { defaultValue: [] }), from: schema.string(), ruleId: schema.string(), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 97ab12f9053585..f7b56f42755ab4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -24,13 +24,19 @@ import { SetupPlugins } from '../../../plugin'; import { getInputIndex } from './get_input_output_index'; import { searchAfterAndBulkCreate } from './search_after_bulk_create'; import { getFilter } from './get_filter'; -import { SignalRuleAlertTypeDefinition, RuleAlertAttributes } from './types'; +import { + SignalRuleAlertTypeDefinition, + RuleAlertAttributes, + EqlSignalSearchResponse, + BaseSignalHit, +} from './types'; import { getGapBetweenRuns, getListsClient, getExceptions, getGapMaxCatchupRatio, MAX_RULE_GAP_RATIO, + wrapSignal, createErrorsFromShard, createSearchAfterReturnType, mergeReturns, @@ -50,6 +56,9 @@ import { ruleStatusServiceFactory } from './rule_status_service'; import { buildRuleMessageFactory } from './rule_messages'; import { ruleStatusSavedObjectsClientFactory } from './rule_status_saved_objects_client'; import { getNotificationResultsLink } from '../notifications/utils'; +import { buildEqlSearchRequest } from '../../../../common/detection_engine/get_query_filter'; +import { bulkInsertSignals } from './single_bulk_create'; +import { buildSignalFromEvent, buildSignalGroupFromSequence } from './build_bulk_body'; import { createThreatSignals } from './threat_mapping/create_threat_signals'; export const signalRulesAlertType = ({ @@ -265,8 +274,6 @@ export const signalRulesAlertType = ({ bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], }), ]); - } else if (isEqlRule(type)) { - throw new Error('EQL Rules are under development, execution is not yet implemented'); } else if (isThresholdRule(type) && threshold) { const inputIndex = await getInputIndex(services, version, index); const esFilter = await getFilter({ @@ -378,7 +385,7 @@ export const signalRulesAlertType = ({ buildRuleMessage, threatIndex, }); - } else { + } else if (type === 'query' || type === 'saved_query') { const inputIndex = await getInputIndex(services, version, index); const esFilter = await getFilter({ type, @@ -417,6 +424,51 @@ export const signalRulesAlertType = ({ throttle, buildRuleMessage, }); + } else if (isEqlRule(type)) { + if (query === undefined) { + throw new Error('eql query rule must have a query defined'); + } + const inputIndex = await getInputIndex(services, version, index); + const request = buildEqlSearchRequest( + query, + inputIndex, + params.from, + params.to, + searchAfterSize, + params.timestampOverride, + exceptionItems ?? [], + params.eventCategoryOverride + ); + const response: EqlSignalSearchResponse = await services.callCluster( + 'transport.request', + request + ); + let newSignals: BaseSignalHit[] | undefined; + if (response.hits.sequences !== undefined) { + newSignals = response.hits.sequences.reduce( + (acc: BaseSignalHit[], sequence) => + acc.concat(buildSignalGroupFromSequence(sequence, savedObject, outputIndex)), + [] + ); + } else if (response.hits.events !== undefined) { + newSignals = response.hits.events.map((event) => + wrapSignal(buildSignalFromEvent(event, savedObject), outputIndex) + ); + } else { + throw new Error( + 'eql query response should have either `sequences` or `events` but had neither' + ); + } + // TODO: replace with code that filters out recursive rule signals while allowing sequences and their building blocks + // const filteredSignals = filterDuplicateSignals(alertId, newSignals); + if (newSignals.length > 0) { + const insertResult = await bulkInsertSignals(newSignals, logger, services, refresh); + result.bulkCreateTimes.push(insertResult.bulkCreateDuration); + result.createdSignalsCount += insertResult.createdItemsCount; + } + result.success = true; + } else { + throw new Error(`unknown rule type ${type}`); } if (result.success) { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts index e8f254e6a8966b..e3c3c940b3225b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts @@ -7,7 +7,7 @@ import { countBy, isEmpty } from 'lodash'; import { performance } from 'perf_hooks'; import { AlertServices } from '../../../../../alerts/server'; -import { SignalSearchResponse, BulkResponse } from './types'; +import { SignalSearchResponse, BulkResponse, SignalHit, BaseSignalHit } from './types'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams, RefreshTypes } from '../types'; import { generateId, makeFloatString, errorAggregator } from './utils'; @@ -59,6 +59,18 @@ export const filterDuplicateRules = ( }); }; +/** + * Similar to filterDuplicateRules, but operates on candidate signal documents rather than events that matched + * the detection query. This means we only have to compare the ruleId against the ancestors array. + * @param ruleId The rule id + * @param signals The candidate new signals + */ +export const filterDuplicateSignals = (ruleId: string, signals: SignalHit[]) => { + return signals.filter( + (doc) => !doc.signal.ancestors.some((ancestor) => ancestor.rule === ruleId) + ); +}; + export interface SingleBulkCreateResponse { success: boolean; bulkCreateDuration?: string; @@ -66,6 +78,11 @@ export interface SingleBulkCreateResponse { errors: string[]; } +export interface BulkInsertSignalsResponse { + bulkCreateDuration: string; + createdItemsCount: number; +} + // Bulk Index documents. export const singleBulkCreate = async ({ filteredEvents, @@ -167,3 +184,46 @@ export const singleBulkCreate = async ({ }; } }; + +// Bulk Index new signals. +export const bulkInsertSignals = async ( + signals: BaseSignalHit[], + logger: Logger, + services: AlertServices, + refresh: RefreshTypes +): Promise => { + // index documents after creating an ID based on the + // id and index of each parent and the rule ID + const bulkBody = signals.flatMap((doc) => [ + { + create: { + _index: doc._index, + _id: doc._id, + }, + }, + doc._source, + ]); + const start = performance.now(); + const response: BulkResponse = await services.callCluster('bulk', { + refresh, + body: bulkBody, + }); + const end = performance.now(); + logger.debug(`individual bulk process time took: ${makeFloatString(end - start)} milliseconds`); + logger.debug(`took property says bulk took: ${response.took} milliseconds`); + + if (response.errors) { + const duplicateSignalsCount = countBy(response.items, 'create.status')['409']; + logger.debug(`ignored ${duplicateSignalsCount} duplicate signals`); + const errorCountByMessage = errorAggregator(response, [409]); + if (!isEmpty(errorCountByMessage)) { + logger.error( + `[-] bulkResponse had errors with responses of: ${JSON.stringify(errorCountByMessage)}` + ); + } + } + + const createdItemsCount = countBy(response.items, 'create.status')['201'] ?? 0; + logger.debug(`bulk created ${createdItemsCount} signals`); + return { bulkCreateDuration: makeFloatString(end - start), createdItemsCount }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 6ebdca0764e9d5..2f6ed0c1e3a8e9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -16,7 +16,7 @@ import { } from '../../../../../alerts/server'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams, RefreshTypes } from '../types'; -import { SearchResponse } from '../../types'; +import { SearchResponse, EqlSearchResponse, BaseHit } from '../../types'; import { ListClient } from '../../../../../lists/server'; import { Logger } from '../../../../../../../src/core/server'; import { ExceptionListItemSchema } from '../../../../../lists/common/schemas'; @@ -53,6 +53,8 @@ export type SearchTypes = export interface SignalSource { [key: string]: SearchTypes; + // TODO: SignalSource is being used as the type for documents matching detection engine queries, but they may not + // actually have @timestamp if a timestamp override is used '@timestamp': string; signal?: { // parent is deprecated: new signals should populate parents instead @@ -60,6 +62,10 @@ export interface SignalSource { parent?: Ancestor; parents?: Ancestor[]; ancestors: Ancestor[]; + group?: { + id: string; + index?: number; + }; rule: { id: string; }; @@ -116,6 +122,9 @@ export interface GetResponse { export type EventSearchResponse = SearchResponse; export type SignalSearchResponse = SearchResponse; export type SignalSourceHit = SignalSearchResponse['hits']['hits'][number]; +export type BaseSignalHit = BaseHit; + +export type EqlSignalSearchResponse = EqlSearchResponse; export type RuleExecutorOptions = Omit & { params: RuleTypeParams; @@ -140,11 +149,15 @@ export interface Ancestor { } export interface Signal { - rule: Partial; + rule: RulesSchema; // DEPRECATED: use parents instead of parent parent?: Ancestor; parents: Ancestor[]; ancestors: Ancestor[]; + group?: { + id: string; + index?: number; + }; original_time?: string; original_event?: SearchTypes; status: Status; @@ -155,7 +168,7 @@ export interface Signal { export interface SignalHit { '@timestamp': string; event: object; - signal: Partial; + signal: Signal; } export interface AlertAttributes { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index 97f3dbeaf44890..14e12b2ea46328 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -25,6 +25,8 @@ import { getListsClient, getSignalTimeTuples, getExceptions, + wrapBuildingBlocks, + generateSignalId, createErrorsFromShard, createSearchAfterReturnTypeFromResponse, createSearchAfterReturnType, @@ -38,6 +40,7 @@ import { sampleBulkError, sampleBulkErrorItem, mockLogger, + sampleSignalHit, sampleDocSearchResultsWithSortId, sampleEmptyDocSearchResults, sampleDocSearchResultsNoSortIdNoHits, @@ -794,6 +797,56 @@ describe('utils', () => { }); }); + describe('wrapBuildingBlocks', () => { + it('should generate a unique id for each building block', () => { + const wrappedBlocks = wrapBuildingBlocks( + [sampleSignalHit(), sampleSignalHit()], + 'test-index' + ); + const blockIds: string[] = []; + wrappedBlocks.forEach((block) => { + expect(blockIds.includes(block._id)).toEqual(false); + blockIds.push(block._id); + }); + }); + + it('should generate different ids for identical documents in different sequences', () => { + const wrappedBlockSequence1 = wrapBuildingBlocks([sampleSignalHit()], 'test-index'); + const wrappedBlockSequence2 = wrapBuildingBlocks( + [sampleSignalHit(), sampleSignalHit()], + 'test-index' + ); + const blockId = wrappedBlockSequence1[0]._id; + wrappedBlockSequence2.forEach((block) => { + expect(block._id).not.toEqual(blockId); + }); + }); + + it('should generate the same ids when given the same sequence twice', () => { + const wrappedBlockSequence1 = wrapBuildingBlocks( + [sampleSignalHit(), sampleSignalHit()], + 'test-index' + ); + const wrappedBlockSequence2 = wrapBuildingBlocks( + [sampleSignalHit(), sampleSignalHit()], + 'test-index' + ); + wrappedBlockSequence1.forEach((block, idx) => { + expect(block._id).toEqual(wrappedBlockSequence2[idx]._id); + }); + }); + }); + + describe('generateSignalId', () => { + it('generates a unique signal id for same signal with different rule id', () => { + const signalId1 = generateSignalId(sampleSignalHit().signal); + const modifiedSignal = sampleSignalHit(); + modifiedSignal.signal.rule.id = 'some other rule id'; + const signalIdModified = generateSignalId(modifiedSignal.signal); + expect(signalId1).not.toEqual(signalIdModified); + }); + }); + describe('createErrorsFromShard', () => { test('empty errors will return an empty array', () => { const createdErrors = createErrorsFromShard({ errors: [] }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 2eabc03dccad71..53089b7f1ca2b8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -16,8 +16,11 @@ import { BulkResponse, BulkResponseErrorAggregation, isValidUnit, + SignalHit, + BaseSignalHit, SearchAfterAndBulkCreateReturnType, SignalSearchResponse, + Signal, } from './types'; import { BuildRuleMessage } from './rule_messages'; import { parseScheduleDates } from '../../../../common/detection_engine/parse_schedule_dates'; @@ -212,6 +215,60 @@ export const generateId = ( ruleId: string ): string => createHash('sha256').update(docIndex.concat(docId, version, ruleId)).digest('hex'); +// TODO: do we need to include version in the id? If it does matter then we should include it in signal.parents as well +export const generateSignalId = (signal: Signal) => + createHash('sha256') + .update( + signal.parents + .reduce((acc, parent) => acc.concat(parent.id, parent.index), '') + .concat(signal.rule.id) + ) + .digest('hex'); + +/** + * Generates unique doc ids for each building block signal within a sequence. The id of each building block + * depends on the parents of every building block, so that a signal which appears in multiple different sequences + * (e.g. if multiple rules build sequences that share a common event/signal) will get a unique id per sequence. + * @param buildingBlocks The full list of building blocks in the sequence. + */ +export const generateBuildingBlockIds = (buildingBlocks: SignalHit[]): string[] => { + const baseHashString = buildingBlocks.reduce( + (baseString, block) => + baseString + .concat( + block.signal.parents.reduce((acc, parent) => acc.concat(parent.id, parent.index), '') + ) + .concat(block.signal.rule.id), + '' + ); + return buildingBlocks.map((block, idx) => + createHash('sha256').update(baseHashString).update(String(idx)).digest('hex') + ); +}; + +export const wrapBuildingBlocks = (buildingBlocks: SignalHit[], index: string): BaseSignalHit[] => { + const blockIds = generateBuildingBlockIds(buildingBlocks); + return buildingBlocks.map((block, idx) => { + return { + _id: blockIds[idx], + _index: index, + _source: { + ...block, + }, + }; + }); +}; + +export const wrapSignal = (signal: SignalHit, index: string): BaseSignalHit => { + return { + _id: generateSignalId(signal.signal), + _index: index, + _source: { + ...signal, + }, + }; +}; + export const parseInterval = (intervalString: string): moment.Duration | null => { try { return moment.duration(parseDuration(intervalString)); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts index b0554adcc46b0f..728f5b1dd867fd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts @@ -37,6 +37,7 @@ import { SeverityMappingOrUndefined, TimestampOverrideOrUndefined, Type, + EventCategoryOverrideOrUndefined, } from '../../../common/detection_engine/schemas/common/schemas'; import { ThreatIndexOrUndefined, @@ -56,6 +57,7 @@ export interface RuleTypeParams { buildingBlockType: BuildingBlockTypeOrUndefined; description: Description; note: NoteOrUndefined; + eventCategoryOverride: EventCategoryOverrideOrUndefined; falsePositives: FalsePositives; from: From; ruleId: RuleId; diff --git a/x-pack/plugins/security_solution/server/lib/types.ts b/x-pack/plugins/security_solution/server/lib/types.ts index 7e59280cd1358d..117cffd844cfb1 100644 --- a/x-pack/plugins/security_solution/server/lib/types.ts +++ b/x-pack/plugins/security_solution/server/lib/types.ts @@ -16,6 +16,7 @@ import { Sources } from './sources'; import { Note } from './note/saved_object'; import { PinnedEvent } from './pinned_event/saved_object'; import { Timeline } from './timeline/saved_object'; +import { SearchTypes } from './detection_engine/signals/types'; export * from './hosts'; @@ -44,6 +45,12 @@ export interface TotalValue { relation: string; } +export interface BaseHit { + _index: string; + _id: string; + _source: T; +} + export interface SearchResponse { took: number; timed_out: boolean; @@ -52,27 +59,43 @@ export interface SearchResponse { hits: { total: TotalValue | number; max_score: number; - hits: Array<{ - _index: string; - _type: string; - _id: string; - _score: number; - _source: T; - _version?: number; - _explanation?: Explanation; - fields?: string[]; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - highlight?: any; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - inner_hits?: any; - matched_queries?: string[]; - sort?: string[]; - }>; + hits: Array< + BaseHit & { + _type: string; + _score: number; + _version?: number; + _explanation?: Explanation; + fields?: string[]; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + highlight?: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + inner_hits?: any; + matched_queries?: string[]; + sort?: string[]; + } + >; }; // eslint-disable-next-line @typescript-eslint/no-explicit-any aggregations?: any; } +export interface EqlSequence { + join_keys: SearchTypes[]; + events: Array>; +} + +export interface EqlSearchResponse { + is_partial: boolean; + is_running: boolean; + took: number; + timed_out: boolean; + hits: { + total: TotalValue; + sequences?: Array>; + events?: Array>; + }; +} + export interface ShardsResponse { total: number; successful: number;