[RAC][Security Solution] Adds Threshold rule type and removes relianc…

…e on outputIndex (elastic#111437) * Initial commit * Properly handle signal history * Fix elastic#95258 - cardinality sort bug * Init threshold rule * Create working threshold rule * Fix threshold signal generation * Fix tests * Update mappings * ALERT_TYPE_ID => RULE_TYPE_ID * Add tests * Fix types
madirey · Sep 14, 2021 · 5a555d1 · 5a555d1
1 parent 0e8cb0c
commit 5a555d1
Show file tree

Hide file tree

Showing 31 changed files with 625 additions and 197 deletions.
diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts
@@ -197,7 +197,6 @@ export const EQL_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.eqlRule` as const;
 export const INDICATOR_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.indicatorRule` as const;
 export const ML_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.mlRule` as const;
 export const QUERY_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.queryRule` as const;
-export const SAVED_QUERY_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.savedQueryRule` as const;
 export const THRESHOLD_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.thresholdRule` as const;
 
 /**

diff --git a/...k/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/threshold.ts b/...k/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/threshold.ts
@@ -5,7 +5,35 @@
  * 2.0.
  */
 
+import {
+  SPACE_IDS,
+  ALERT_RULE_CONSUMER,
+  ALERT_REASON,
+  ALERT_STATUS,
+  ALERT_STATUS_ACTIVE,
+  ALERT_WORKFLOW_STATUS,
+  ALERT_RULE_NAMESPACE,
+  ALERT_INSTANCE_ID,
+  ALERT_UUID,
+  ALERT_RULE_TYPE_ID,
+  ALERT_RULE_PRODUCER,
+  ALERT_RULE_CATEGORY,
+  ALERT_RULE_UUID,
+  ALERT_RULE_NAME,
+} from '@kbn/rule-data-utils';
+import { TypeOfFieldMap } from '../../../../../../rule_registry/common/field_map';
+import { SERVER_APP_ID } from '../../../../../common/constants';
+import { ANCHOR_DATE } from '../../../../../common/detection_engine/schemas/response/rules_schema.mocks';
+import { getListArrayMock } from '../../../../../common/detection_engine/schemas/types/lists.mock';
 import { sampleDocNoSortId } from '../../signals/__mocks__/es_results';
+import { flattenWithPrefix } from '../factories/utils/flatten_with_prefix';
+import { RulesFieldMap } from '../field_maps';
+import {
+  ALERT_ANCESTORS,
+  ALERT_ORIGINAL_TIME,
+  ALERT_ORIGINAL_EVENT,
+} from '../field_maps/field_names';
+import { WrappedRACAlert } from '../types';
 
 export const mockThresholdResults = {
   rawResponse: {
@@ -59,3 +87,79 @@ export const mockThresholdResults = {
     },
   },
 };
+
+export const sampleThresholdAlert: WrappedRACAlert = {
+  _id: 'b3ad77a4-65bd-4c4e-89cf-13c46f54bc4d',
+  _index: 'some-index',
+  _source: {
+    '@timestamp': '2020-04-20T21:26:30.000Z',
+    [SPACE_IDS]: ['default'],
+    [ALERT_INSTANCE_ID]: 'b3ad77a4-65bd-4c4e-89cf-13c46f54bc4d',
+    [ALERT_UUID]: '310158f7-994d-4a38-8cdc-152139ac4d29',
+    [ALERT_RULE_CONSUMER]: SERVER_APP_ID,
+    [ALERT_ANCESTORS]: [
+      {
+        id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
+        type: 'event',
+        index: 'myFakeSignalIndex',
+        depth: 0,
+      },
+    ],
+    [ALERT_ORIGINAL_TIME]: '2020-04-20T21:27:45.000Z',
+    [ALERT_ORIGINAL_EVENT]: {
+      action: 'socket_opened',
+      dataset: 'socket',
+      kind: 'event',
+      module: 'system',
+    },
+    [ALERT_REASON]: 'alert reasonable reason',
+    [ALERT_STATUS]: ALERT_STATUS_ACTIVE,
+    [ALERT_WORKFLOW_STATUS]: 'open',
+    'source.ip': '127.0.0.1',
+    'host.name': 'garden-gnomes',
+    [ALERT_RULE_CATEGORY]: 'security',
+    [ALERT_RULE_NAME]: 'a threshold rule',
+    [ALERT_RULE_PRODUCER]: 'siem',
+    [ALERT_RULE_TYPE_ID]: 'query-rule-id',
+    [ALERT_RULE_UUID]: '151af49f-2e82-4b6f-831b-7f8cb341a5ff',
+    ...(flattenWithPrefix(ALERT_RULE_NAMESPACE, {
+      author: [],
+      uuid: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9',
+      created_at: new Date(ANCHOR_DATE).toISOString(),
+      updated_at: new Date(ANCHOR_DATE).toISOString(),
+      created_by: 'elastic',
+      description: 'some description',
+      enabled: true,
+      false_positives: ['false positive 1', 'false positive 2'],
+      from: 'now-6m',
+      immutable: false,
+      name: 'Query with a rule id',
+      query: 'user.name: root or user.name: admin',
+      references: ['test 1', 'test 2'],
+      severity: 'high',
+      severity_mapping: [],
+      updated_by: 'elastic_kibana',
+      tags: ['some fake tag 1', 'some fake tag 2'],
+      to: 'now',
+      type: 'query',
+      threat: [],
+      threshold: {
+        field: ['source.ip', 'host.name'],
+        value: 1,
+      },
+      version: 1,
+      status: 'succeeded',
+      status_date: '2020-02-22T16:47:50.047Z',
+      last_success_at: '2020-02-22T16:47:50.047Z',
+      last_success_message: 'succeeded',
+      max_signals: 100,
+      risk_score: 55,
+      risk_score_mapping: [],
+      language: 'kuery',
+      rule_id: 'f88a544c-1d4e-4652-ae2a-c953b38da5d0',
+      interval: '5m',
+      exceptions_list: getListArrayMock(),
+    }) as TypeOfFieldMap<RulesFieldMap>),
+    'kibana.alert.depth': 1,
+  },
+};
diff --git a/...security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts b/...security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts
@@ -5,6 +5,8 @@
  * 2.0.
  */
 
+import { ALERT_INSTANCE_ID } from '@kbn/rule-data-utils';
+
 import { performance } from 'perf_hooks';
 import { countBy, isEmpty } from 'lodash';
 
@@ -62,11 +64,15 @@ export const bulkCreateFactory = <TContext extends AlertInstanceContext>(
   );
 
   const createdItems = wrappedDocs
-    .map((doc, index) => ({
-      _id: response.body.items[index].index?._id ?? '',
-      _index: response.body.items[index].index?._index ?? '',
-      ...doc._source,
-    }))
+    .map((doc, index) => {
+      const responseIndex = response.body.items[index].index;
+      return {
+        _id: responseIndex?._id ?? '',
+        _index: responseIndex?._index ?? '',
+        [ALERT_INSTANCE_ID]: responseIndex?._id ?? '',
+        ...doc._source,
+      };
+    })
     .filter((_, index) => response.body.items[index].index?.status === 201);
   const createdItemsCount = createdItems.length;
 

diff --git a/...urity_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts b/...urity_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts
@@ -102,7 +102,6 @@ describe('buildAlert', () => {
         status_date: '2020-02-22T16:47:50.047Z',
         last_success_at: '2020-02-22T16:47:50.047Z',
         last_success_message: 'succeeded',
-        output_index: '.siem-signals-default',
         max_signals: 100,
         risk_score: 55,
         risk_score_mapping: [],
@@ -179,7 +178,6 @@ describe('buildAlert', () => {
         status_date: '2020-02-22T16:47:50.047Z',
         last_success_at: '2020-02-22T16:47:50.047Z',
         last_success_message: 'succeeded',
-        output_index: '.siem-signals-default',
         max_signals: 100,
         risk_score: 55,
         risk_score_mapping: [],

diff --git a/...s/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts b/...s/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts
@@ -120,7 +120,7 @@ export const buildAlert = (
     []
   );
 
-  const { id, ...mappedRule } = rule;
+  const { id, output_index: outputIndex, ...mappedRule } = rule;
   mappedRule.uuid = id;
 
   return ({

diff --git a/...ty_solution/server/lib/detection_engine/rule_types/factories/utils/flatten_with_prefix.ts b/...ty_solution/server/lib/detection_engine/rule_types/factories/utils/flatten_with_prefix.ts
@@ -5,16 +5,26 @@
  * 2.0.
  */
 
+import { isPlainObject } from 'lodash';
 import { SearchTypes } from '../../../../../../common/detection_engine/types';
 
 export const flattenWithPrefix = (
   prefix: string,
-  obj: Record<string, SearchTypes>
+  maybeObj: unknown
 ): Record<string, SearchTypes> => {
-  return Object.keys(obj).reduce((acc: Record<string, SearchTypes>, key) => {
+  if (maybeObj != null && isPlainObject(maybeObj)) {
+    return Object.keys(maybeObj as Record<string, SearchTypes>).reduce(
+      (acc: Record<string, SearchTypes>, key) => {
+        return {
+          ...acc,
+          ...flattenWithPrefix(`${prefix}.${key}`, (maybeObj as Record<string, SearchTypes>)[key]),
+        };
+      },
+      {}
+    );
+  } else {
     return {
-      ...acc,
-      [`${prefix}.${key}`]: obj[key],
+      [prefix]: maybeObj as SearchTypes,
     };
-  }, {});
+  }
 };
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts
@@ -43,11 +43,6 @@ export const alertsFieldMap: FieldMap = {
     array: false,
     required: true,
   },
-  'kibana.alert.group': {
-    type: 'object',
-    array: false,
-    required: false,
-  },
   'kibana.alert.group.id': {
     type: 'keyword',
     array: false,
@@ -58,11 +53,6 @@ export const alertsFieldMap: FieldMap = {
     array: false,
     required: false,
   },
-  'kibana.alert.original_event': {
-    type: 'object',
-    array: false,
-    required: false,
-  },
   'kibana.alert.original_event.action': {
     type: 'keyword',
     array: false,
@@ -198,81 +188,6 @@ export const alertsFieldMap: FieldMap = {
     array: false,
     required: false,
   },
-  'kibana.alert.threat': {
-    type: 'object',
-    array: false,
-    required: false,
-  },
-  'kibana.alert.threat.framework': {
-    type: 'keyword',
-    array: false,
-    required: true,
-  },
-  'kibana.alert.threat.tactic': {
-    type: 'object',
-    array: false,
-    required: true,
-  },
-  'kibana.alert.threat.tactic.id': {
-    type: 'keyword',
-    array: false,
-    required: true,
-  },
-  'kibana.alert.threat.tactic.name': {
-    type: 'keyword',
-    array: false,
-    required: true,
-  },
-  'kibana.alert.threat.tactic.reference': {
-    type: 'keyword',
-    array: false,
-    required: true,
-  },
-  'kibana.alert.threat.technique': {
-    type: 'object',
-    array: false,
-    required: true,
-  },
-  'kibana.alert.threat.technique.id': {
-    type: 'keyword',
-    array: false,
-    required: true,
-  },
-  'kibana.alert.threat.technique.name': {
-    type: 'keyword',
-    array: false,
-    required: true,
-  },
-  'kibana.alert.threat.technique.reference': {
-    type: 'keyword',
-    array: false,
-    required: true,
-  },
-  'kibana.alert.threat.technique.subtechnique': {
-    type: 'object',
-    array: false,
-    required: true,
-  },
-  'kibana.alert.threat.technique.subtechnique.id': {
-    type: 'keyword',
-    array: false,
-    required: true,
-  },
-  'kibana.alert.threat.technique.subtechnique.name': {
-    type: 'keyword',
-    array: false,
-    required: true,
-  },
-  'kibana.alert.threat.technique.subtechnique.reference': {
-    type: 'keyword',
-    array: false,
-    required: true,
-  },
-  'kibana.alert.threshold_result': {
-    type: 'object',
-    array: false,
-    required: false,
-  },
   'kibana.alert.threshold_result.cardinality': {
     type: 'object',
     array: false,
@@ -300,7 +215,7 @@ export const alertsFieldMap: FieldMap = {
   },
   'kibana.alert.threshold_result.terms': {
     type: 'object',
-    array: false,
+    array: true,
     required: false,
   },
   'kibana.alert.threshold_result.terms.field': {

diff --git a/...lugins/security_solution/server/lib/detection_engine/rule_types/field_maps/field_names.ts b/...lugins/security_solution/server/lib/detection_engine/rule_types/field_maps/field_names.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { ALERT_NAMESPACE } from '@kbn/rule-data-utils';
+import { ALERT_NAMESPACE, ALERT_RULE_NAMESPACE } from '@kbn/rule-data-utils';
 
 export const ALERT_ANCESTORS = `${ALERT_NAMESPACE}.ancestors` as const;
 export const ALERT_BUILDING_BLOCK_TYPE = `${ALERT_NAMESPACE}.building_block_type` as const;
@@ -14,3 +14,6 @@ export const ALERT_GROUP_ID = `${ALERT_NAMESPACE}.group.id` as const;
 export const ALERT_GROUP_INDEX = `${ALERT_NAMESPACE}.group.index` as const;
 export const ALERT_ORIGINAL_EVENT = `${ALERT_NAMESPACE}.original_event` as const;
 export const ALERT_ORIGINAL_TIME = `${ALERT_NAMESPACE}.original_time` as const;
+
+const ALERT_RULE_THRESHOLD = `${ALERT_RULE_NAMESPACE}.threshold` as const;
+export const ALERT_RULE_THRESHOLD_FIELD = `${ALERT_RULE_THRESHOLD}.field` as const;