From f4cb144cd3775505c9cc0c979c804715b8eef848 Mon Sep 17 00:00:00 2001 From: Marshall Main <55718608+marshallmain@users.noreply.github.com> Date: Mon, 14 Sep 2020 11:28:48 -0400 Subject: [PATCH] [Security_solution][Detections] Refactor signal ancestry to allow multiple parents (#76531) (#77346) * Refactors signal ancestry to allow multiple parents * Fix depth calculation for 7.10+ signals on pre-7.10 signals * Comment build_signal functions * Rename buildAncestorsSignal to buildAncestors * Update detection engine depth test scripts and docs * Update halting test readme * Match up rule ids in readme * Continue populating signal.parent along with signal.parents * pr comments Co-authored-by: Elastic Machine Co-authored-by: Elastic Machine --- .../routes/index/signals_mapping.json | 25 ++ .../signals_on_signals/depth_test/README.md | 244 +++++++++-------- .../depth_test/query_single_id.json | 2 +- .../depth_test/signal_on_signal_depth_1.json | 4 +- .../depth_test/signal_on_signal_depth_2.json | 4 +- .../signals_on_signals/halting_test/README.md | 182 +++++++------ .../signals/__mocks__/es_results.ts | 10 +- .../signals/build_bulk_body.test.ts | 60 +++-- .../signals/build_bulk_body.ts | 9 +- .../signals/build_rule.test.ts | 102 ++++++- .../detection_engine/signals/build_rule.ts | 16 +- .../signals/build_signal.test.ts | 250 ++++-------------- .../detection_engine/signals/build_signal.ts | 82 +++--- .../signals/signal_rule_alert_type.ts | 3 +- .../signals/single_bulk_create.test.ts | 32 +-- .../signals/single_bulk_create.ts | 5 +- .../lib/detection_engine/signals/types.ts | 19 +- 17 files changed, 568 insertions(+), 481 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json index 7d80a319e9e520..cfce0199100714 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json @@ -22,11 +22,33 @@ } } }, + "parents": { + "properties": { + "rule": { + "type": "keyword" + }, + "index": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "depth": { + "type": "long" + } + } + }, "ancestors": { "properties": { "rule": { "type": "keyword" }, + "index": { + "type": "keyword" + }, "id": { "type": "keyword" }, @@ -299,6 +321,9 @@ }, "threshold_count": { "type": "float" + }, + "depth": { + "type": "integer" } } } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/README.md b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/README.md index 2310ba979da202..7cf7d11e4c1f87 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/README.md +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/README.md @@ -22,7 +22,7 @@ which will write a single signal document into the signals index by searching fo signal_on_signal_depth_1.json ``` -which has this key part of its query: `"query": "signal.parent.depth: 1 and _id: *"` which will only create signals +which has this key part of its query: `"query": "signal.depth: 1 and _id: *"` which will only create signals from all signals that point directly to an event (signal -> event). Then a second rule called @@ -34,7 +34,7 @@ signal_on_signal_depth_2.json which will only create signals from all signals that point directly to another signal (signal -> signal) with this query ```json -"query": "signal.parent.depth: 2 and _id: *" +"query": "signal.depth: 2 and _id: *" ``` ## Setup @@ -90,38 +90,43 @@ And then you can query against that: GET .siem-signals-default/_search ``` -Check your parent section of the signal and you will see something like this: +Check your `signal` section of the signal and you will see something like this: ```json -"parent" : { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", - "id" : "o8G7vm8BvLT8jmu5B1-M", - "type" : "event", - "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 -}, -"ancestors" : [ +"parents" : [ { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 } -] +], +"ancestors" : [ + { + "id" : "o8G7vm8BvLT8jmu5B1-M", + "type" : "event", + "index" : "filebeat-8.0.0-2019.12.18-000001", + "depth" : 0 + }, +], +"depth": 1, +"rule": { + "id": "74e0dd0c-4609-416f-b65e-90f8b2564612" +} ``` -The parent and ancestors structure is defined as: +The parents structure is defined as: ``` -rule -> The id of the rule. You can view the rule by ./get_rule_by_rule_id.sh ded57b36-9c4e-4ee4-805d-be4e92033e41 +rule -> The id of the rule, if the parent was generated by a rule. You can view the rule by ./get_rule_by_rule_id.sh ded57b36-9c4e-4ee4-805d-be4e92033e41 id -> The original _id of the document type -> The type of the document, it will be either event or signal index -> The original location of the index -depth -> The depth of this signal. It will be at least 1 to indicate it is a signal generated from a event. Otherwise 2 or more to indicate a signal on signal and what depth we are at -ancestors -> An array tracking all of the parents of this particular signal. As depth increases this will too. +depth -> The depth of the parent event/signal. It will be 0 if the parent is an event, or 1+ if the parent is another signal. ``` +The ancestors structure has the same fields as parents, but is an array of all ancestors (parents, grandparents, etc) of the signal. + This is indicating that you have a single parent of an event from the signal (signal -> event) and this document has a single ancestor of that event. Each 30 seconds that goes it will use de-duplication technique to ensure that this signal is not re-inserted. If after each 30 seconds you DO SEE multiple signals then the bug is a de-duplication bug and a critical bug. If you ever see a duplicate rule in the @@ -138,55 +143,64 @@ running in the system which are generating signals on top of signals. After 30 s documents in the signals index. The first signal is our original (signal -> event) document with a rule id: ```json -"parent" : { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", - "id" : "o8G7vm8BvLT8jmu5B1-M", - "type" : "event", - "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 -}, +"parents" : [ + { + "id" : "o8G7vm8BvLT8jmu5B1-M", + "type" : "event", + "index" : "filebeat-8.0.0-2019.12.18-000001", + "depth" : 0 + } +], "ancestors" : [ { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 } -] +], +"depth": 1, +"rule": { + "id": "74e0dd0c-4609-416f-b65e-90f8b2564612" +} ``` and the second document is a signal on top of a signal like so: ```json -"parent" : { - "rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c", - "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", - "type" : "signal", - "index" : ".siem-signals-default-000001", - "depth" : 2 -}, -"ancestors" : [ +"parents" : [ { "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", + "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", + "type" : "signal", + "index" : ".siem-signals-default-000001", + "depth" : 1 + } +] +"ancestors" : [ + { "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c", + "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 } -] +], +"depth": 2, +"rule": { + "id": "1d3b3735-66ef-4e53-b7f5-4340026cc40c" +} ``` Notice that the depth indicates it is at level 2 and its parent is that of a signal. Also notice that the ancestors is an array of size 2 indicating that this signal terminates at an event. Each and every signal ancestors array should terminate at an event and should ONLY contain 1 -event and NEVER 2 or more events. After 30+ seconds you should NOT see any new documents being created and you should be stable +event and NEVER 2 or more events for KQL query based rules. EQL query based rules that use sequences may have multiple parents at the same level. After 30+ seconds you should NOT see any new documents being created and you should be stable at 2. Otherwise we have AND/OR a de-duplication issue, signal on signal issue. Now, post this same rule a second time as a second instance which is going to run against these two documents. @@ -212,79 +226,93 @@ The expected behavior is that eventually you will get 3 total documents but not The original event rule 74e0dd0c-4609-416f-b65e-90f8b2564612 (event -> signal) ```json -"parent" : { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", - "id" : "o8G7vm8BvLT8jmu5B1-M", - "type" : "event", - "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 -}, +"parents" : [ + { + "id" : "o8G7vm8BvLT8jmu5B1-M", + "type" : "event", + "index" : "filebeat-8.0.0-2019.12.18-000001", + "depth" : 0 + } +], "ancestors" : [ { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 } -] +], +"depth": 1, +"rule": { + "id": "74e0dd0c-4609-416f-b65e-90f8b2564612" +} ``` The first signal to signal rule 1d3b3735-66ef-4e53-b7f5-4340026cc40c (signal -> event) ```json -"parent" : { - "rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c", - "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", - "type" : "signal", - "index" : ".siem-signals-default-000001", - "depth" : 2 -}, -"ancestors" : [ +"parents" : [ { "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", + "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", + "type" : "signal", + "index" : ".siem-signals-default-000001", + "depth" : 1 + } +] +"ancestors" : [ + { "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c", + "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 } -] +], +"depth": 2, +"rule": { + "id": "1d3b3735-66ef-4e53-b7f5-4340026cc40c" +} ``` Then our second signal to signal rule c93ddb57-e7e9-4973-9886-72ddefb4d22e (signal -> event) which finds the same thing as the first signal to signal ```json -"parent" : { - "rule" : "c93ddb57-e7e9-4973-9886-72ddefb4d22e", - "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", - "type" : "signal", - "index" : ".siem-signals-default-000001", - "depth" : 2 -}, -"ancestors" : [ +"parents" : [ { "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", + "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", + "type" : "signal", + "index" : ".siem-signals-default-000001", + "depth" : 1 + } +], +"ancestors" : [ + { "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "c93ddb57-e7e9-4973-9886-72ddefb4d22e", + "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 } -] +], +"depth": 2, +"rule": { + "id": "c93ddb57-e7e9-4973-9886-72ddefb4d22e" +} ``` We should be able to post this depth level as many times as we want and get only 1 new document each time. If we decide though to @@ -298,69 +326,79 @@ The expectation is that a document for each of the previous depth 1 documents wo depth 1 rules running then the signals at depth 2 will produce two new ones and those two will look like so: ```json -"parent" : { - "rule" : "a1f7b520-5bfd-451d-af59-428f60753fee", - "id" : "365236ce5e77770508152403b4e16613f407ae4b1a135a450dcfec427f2a3231", - "type" : "signal", - "index" : ".siem-signals-default-000001", - "depth" : 3 -}, +"parents" : [ + { + "rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c", + "id" : "365236ce5e77770508152403b4e16613f407ae4b1a135a450dcfec427f2a3231", + "type" : "signal", + "index" : ".siem-signals-default-000001", + "depth" : 2 + } +], "ancestors" : [ { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c", + "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 }, { - "rule" : "a1f7b520-5bfd-451d-af59-428f60753fee", + "rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c", "id" : "365236ce5e77770508152403b4e16613f407ae4b1a135a450dcfec427f2a3231", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 3 + "depth" : 2 } -] +], +"depth": 3, +"rule": { + "id": "a1f7b520-5bfd-451d-af59-428f60753fee" +} ``` ```json -"parent" : { - "rule" : "a1f7b520-5bfd-451d-af59-428f60753fee", - "id" : "e8b1f1adb40fd642fa524dea89ef94232e67b05e99fb0b2683f1e47e90b759fb", - "type" : "signal", - "index" : ".siem-signals-default-000001", - "depth" : 3 -}, +"parents" : [ + { + "rule" : "c93ddb57-e7e9-4973-9886-72ddefb4d22e", + "id" : "e8b1f1adb40fd642fa524dea89ef94232e67b05e99fb0b2683f1e47e90b759fb", + "type" : "signal", + "index" : ".siem-signals-default-000001", + "depth" : 2 + } +], "ancestors" : [ { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "c93ddb57-e7e9-4973-9886-72ddefb4d22e", + "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 }, { - "rule" : "a1f7b520-5bfd-451d-af59-428f60753fee", + "rule" : "c93ddb57-e7e9-4973-9886-72ddefb4d22e", "id" : "e8b1f1adb40fd642fa524dea89ef94232e67b05e99fb0b2683f1e47e90b759fb", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 3 + "depth" : 2 } -] +], +"depth": 3, +"rule": { + "id": "a1f7b520-5bfd-451d-af59-428f60753fee" +} ``` The total number of documents should be 5 at this point. If you were to post this same rule a second time to get a second instance diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/query_single_id.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/query_single_id.json index dc05c656d7cf1f..305aa349926236 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/query_single_id.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/query_single_id.json @@ -7,6 +7,6 @@ "from": "now-1d", "interval": "30s", "to": "now", - "query": "_id: o8G7vm8BvLT8jmu5B1-M", + "query": "event.id: 08cde4aa-d249-4e6b-8300-06f3d56c7fe7", "enabled": true } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_1.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_1.json index fb13413a02791b..c9132ddb0a590b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_1.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_1.json @@ -7,7 +7,7 @@ "from": "now-1d", "interval": "30s", "to": "now", - "query": "signal.parent.depth: 1 and _id: *", + "query": "signal.depth: 1 and _id: *", "enabled": true, - "index": ".siem-signals-default" + "index": [".siem-signals-default"] } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_2.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_2.json index c1b7594653ec73..d1a27497926861 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_2.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_2.json @@ -7,7 +7,7 @@ "from": "now-1d", "interval": "30s", "to": "now", - "query": "signal.parent.depth: 2 and _id: *", + "query": "signal.depth: 2 and _id: *", "enabled": true, - "index": ".siem-signals-default" + "index": [".siem-signals-default"] } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/halting_test/README.md b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/halting_test/README.md index b1a83f5317776f..01b21bf762e44a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/halting_test/README.md +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/halting_test/README.md @@ -69,38 +69,43 @@ And then you can query against that: GET .siem-signals-default/_search ``` -Check your parent section of the signal and you will see something like this: +Check your `signal` section of the signal and you will see something like this: ```json -"parent" : { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", - "id" : "o8G7vm8BvLT8jmu5B1-M", - "type" : "event", - "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 -}, -"ancestors" : [ +"parents" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 } -] +], +"ancestors" : [ + { + "id" : "o8G7vm8BvLT8jmu5B1-M", + "type" : "event", + "index" : "filebeat-8.0.0-2019.12.18-000001", + "depth" : 0 + }, +], +"depth": 1, +"rule": { + "id": "ded57b36-9c4e-4ee4-805d-be4e92033e41" +} ``` -The parent and ancestors structure is defined as: +The parents structure is defined as: ``` -rule -> The id of the rule. You can view the rule by ./get_rule_by_rule_id.sh ded57b36-9c4e-4ee4-805d-be4e92033e41 +rule -> The id of the rule, if the parent was generated by a rule. You can view the rule by ./get_rule_by_rule_id.sh ded57b36-9c4e-4ee4-805d-be4e92033e41 id -> The original _id of the document type -> The type of the document, it will be either event or signal index -> The original location of the index -depth -> The depth of this signal. It will be at least 1 to indicate it is a signal generated from a event. Otherwise 2 or more to indicate a signal on signal and what depth we are at -ancestors -> An array tracking all of the parents of this particular signal. As depth increases this will too. +depth -> The depth of the parent event/signal. It will be 0 if the parent is an event, or 1+ if the parent is another signal. ``` +The ancestors structure has the same fields as parents, but is an array of all ancestors (parents, grandparents, etc) of the signal. + This is indicating that you have a single parent of an event from the signal (signal -> event) and this document has a single ancestor of that event. Each 30 seconds that goes it will use de-duplication technique to ensure that this signal is not re-inserted. If after each 30 seconds you DO SEE multiple signals then the bug is a de-duplication bug and a critical bug. If you ever see a duplicate rule in the @@ -119,22 +124,26 @@ documents in the signals index. The first signal is our original (signal -> even (signal -> event) ```json -"parent" : { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", - "id" : "o8G7vm8BvLT8jmu5B1-M", - "type" : "event", - "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 -}, -"ancestors" : [ +"parents" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 } -] +], +"ancestors" : [ + { + "id" : "o8G7vm8BvLT8jmu5B1-M", + "type" : "event", + "index" : "filebeat-8.0.0-2019.12.18-000001", + "depth" : 0 + }, +], +"depth": 1, +"rule": { + "id": "ded57b36-9c4e-4ee4-805d-be4e92033e41" +} ``` and the second document is a signal on top of a signal like so: @@ -143,28 +152,31 @@ and the second document is a signal on top of a signal like so: ```json "parent" : { - "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 }, "ancestors" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 } -] +], +"depth": 2, +"rule": { + "id": "161fa5b8-0b96-4985-b066-0d99b2bcb904" +} ``` Notice that the depth indicates it is at level 2 and its parent is that of a signal. Also notice that the ancestors is an array of size 2 @@ -195,50 +207,57 @@ The expected behavior is that eventually you will get 5 total documents but not The original event rule ded57b36-9c4e-4ee4-805d-be4e92033e41 (event -> signal) ```json -"parent" : { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", - "id" : "o8G7vm8BvLT8jmu5B1-M", - "type" : "event", - "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 -}, -"ancestors" : [ +"parents" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 } -] +], +"ancestors" : [ + { + "id" : "o8G7vm8BvLT8jmu5B1-M", + "type" : "event", + "index" : "filebeat-8.0.0-2019.12.18-000001", + "depth" : 0 + }, +], +"depth": 1, +"rule": { + "id": "ded57b36-9c4e-4ee4-805d-be4e92033e41" +} ``` The first signal to signal rule 161fa5b8-0b96-4985-b066-0d99b2bcb904 (signal -> event) ```json "parent" : { - "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 }, "ancestors" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 } -] +], +"depth": 2, +"rule": { + "id": "161fa5b8-0b96-4985-b066-0d99b2bcb904" +} ``` Then our second signal to signal rule f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406 (signal -> event) which finds the same thing as the first @@ -246,28 +265,31 @@ signal to signal ```json "parent" : { - "rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 }, "ancestors" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 } -] +], +"depth": 2, +"rule": { + "id": "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406" +} ``` But then f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406 also finds the first signal to signal rule from 161fa5b8-0b96-4985-b066-0d99b2bcb904 @@ -275,35 +297,38 @@ and writes that document out with a depth of 3. (signal -> signal -> event) ```json "parent" : { - "rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406", + "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", "id" : "c627e5e2576f1b10952c6c57249947e89b6153b763a59fb9e391d0b56be8e7fe", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 3 + "depth" : 2 }, "ancestors" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 }, { - "rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406", + "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", "id" : "c627e5e2576f1b10952c6c57249947e89b6153b763a59fb9e391d0b56be8e7fe", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 3 + "depth" : 2 } -] +], +"depth": 3, +"rule": { + "id": "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406" +} ``` Since it wrote that document, the first signal to signal 161fa5b8-0b96-4985-b066-0d99b2bcb904 writes out it found this newly created signal @@ -311,35 +336,38 @@ Since it wrote that document, the first signal to signal 161fa5b8-0b96-4985-b066 ```json "parent" : { - "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", + "rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406", "id" : "efbe514e8d806a5ef3da7658cfa73961e25befefc84f622e963b45dcac798868", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 3 + "depth" : 2 }, "ancestors" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 }, { - "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", + "rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406", "id" : "efbe514e8d806a5ef3da7658cfa73961e25befefc84f622e963b45dcac798868", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 3 + "depth" : 2 } -] +], +"depth": 3, +"rule": { + "id": "161fa5b8-0b96-4985-b066-0d99b2bcb904" +} ``` You will be "halted" at this point as the signal ancestry and de-duplication ensures that we do not report twice on signals and that we do not diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 95ec753c21fd84..9d3eb29be08dde 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -149,21 +149,23 @@ export const sampleDocWithAncestors = (): SignalSearchResponse => { delete sampleDoc._source.source; sampleDoc._source.signal = { parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ancestors: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], + rule: { + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + }, + depth: 1, }; return { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index ee83c826371bc1..967dc5331e46b1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -48,19 +48,25 @@ describe('buildBulkBody', () => { }, signal: { parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], ancestors: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], original_time: '2020-04-20T21:27:45+0000', @@ -102,6 +108,7 @@ describe('buildBulkBody', () => { updated_at: fakeSignalSourceHit.signal.rule?.updated_at, exceptions_list: getListArrayMock(), }, + depth: 1, }, }; expect(fakeSignalSourceHit).toEqual(expected); @@ -151,19 +158,25 @@ describe('buildBulkBody', () => { module: 'system', }, parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], ancestors: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], original_time: '2020-04-20T21:27:45+0000', @@ -205,6 +218,7 @@ describe('buildBulkBody', () => { threat: [], exceptions_list: getListArrayMock(), }, + depth: 1, }, }; expect(fakeSignalSourceHit).toEqual(expected); @@ -252,19 +266,25 @@ describe('buildBulkBody', () => { module: 'system', }, parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], ancestors: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], original_time: '2020-04-20T21:27:45+0000', @@ -306,6 +326,7 @@ describe('buildBulkBody', () => { throttle: 'no_actions', exceptions_list: getListArrayMock(), }, + depth: 1, }, }; expect(fakeSignalSourceHit).toEqual(expected); @@ -346,19 +367,25 @@ describe('buildBulkBody', () => { kind: 'event', }, parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], ancestors: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], original_time: '2020-04-20T21:27:45+0000', @@ -400,6 +427,7 @@ describe('buildBulkBody', () => { throttle: 'no_actions', exceptions_list: getListArrayMock(), }, + depth: 1, }, }; expect(fakeSignalSourceHit).toEqual(expected); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index 218750ac30a2aa..7be97e46f91f24 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -4,9 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SignalSourceHit, SignalHit } from './types'; +import { SignalSourceHit, SignalHit, Signal } from './types'; import { buildRule } from './build_rule'; -import { buildSignal } from './build_signal'; +import { additionalSignalFields, buildSignal } from './build_signal'; import { buildEventTypeSignal } from './build_event_type_signal'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams } from '../types'; @@ -58,7 +58,10 @@ export const buildBulkBody = ({ tags, throttle, }); - const signal = buildSignal(doc, rule); + const signal: Signal = { + ...buildSignal([doc], rule), + ...additionalSignalFields(doc), + }; const event = buildEventTypeSignal(doc); const signalHit: SignalHit = { ...doc._source, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts index 7257e5952ff055..ba815a0b62f0d3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts @@ -4,10 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { buildRule } from './build_rule'; +import { buildRule, removeInternalTagsFromRule } from './build_rule'; import { sampleDocNoSortId, sampleRuleAlertParams, sampleRuleGuid } from './__mocks__/es_results'; import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; +import { INTERNAL_RULE_ID_KEY, INTERNAL_IMMUTABLE_KEY } from '../../../../common/constants'; +import { getPartialRulesSchemaMock } from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; describe('buildRule', () => { beforeEach(() => { @@ -208,4 +210,102 @@ describe('buildRule', () => { }; expect(rule).toEqual(expected); }); + + test('it builds a rule and removes internal tags', () => { + const ruleParams = sampleRuleAlertParams(); + const rule = buildRule({ + actions: [], + doc: sampleDocNoSortId(), + ruleParams, + name: 'some-name', + id: sampleRuleGuid, + enabled: false, + createdAt: '2020-01-28T15:58:34.810Z', + updatedAt: '2020-01-28T15:59:14.004Z', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: 'some interval', + tags: [ + 'some fake tag 1', + 'some fake tag 2', + `${INTERNAL_RULE_ID_KEY}:rule-1`, + `${INTERNAL_IMMUTABLE_KEY}:true`, + ], + throttle: 'no_actions', + }); + const expected: Partial = { + actions: [], + author: ['Elastic'], + building_block_type: 'default', + created_by: 'elastic', + description: 'Detecting root and admin users', + enabled: false, + false_positives: [], + from: 'now-6m', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: 'some interval', + language: 'kuery', + license: 'Elastic License', + max_signals: 10000, + name: 'some-name', + output_index: '.siem-signals', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + risk_score: 50, + risk_score_mapping: [], + rule_id: 'rule-1', + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + to: 'now', + type: 'query', + note: '', + updated_by: 'elastic', + updated_at: rule.updated_at, + created_at: rule.created_at, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + version: 1, + }; + expect(rule).toEqual(expected); + }); + + test('it removes internal tags from a typical rule', () => { + const rule = getPartialRulesSchemaMock(); + rule.tags = [ + 'some fake tag 1', + 'some fake tag 2', + `${INTERNAL_RULE_ID_KEY}:rule-1`, + `${INTERNAL_IMMUTABLE_KEY}:true`, + ]; + const noInternals = removeInternalTagsFromRule(rule); + expect(noInternals).toEqual(getPartialRulesSchemaMock()); + }); + + test('it works with an empty array', () => { + const rule = getPartialRulesSchemaMock(); + rule.tags = []; + const noInternals = removeInternalTagsFromRule(rule); + const expected = getPartialRulesSchemaMock(); + expected.tags = []; + expect(noInternals).toEqual(expected); + }); + + test('it works if tags does not exist', () => { + const rule = getPartialRulesSchemaMock(); + delete rule.tags; + const noInternals = removeInternalTagsFromRule(rule); + const expected = getPartialRulesSchemaMock(); + delete expected.tags; + expect(noInternals).toEqual(expected); + }); + + test('it works if tags contains normal values and no internal values', () => { + const rule = getPartialRulesSchemaMock(); + const noInternals = removeInternalTagsFromRule(rule); + expect(noInternals).toEqual(rule); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts index e02a0154d63c9b..aacf9b8be31b41 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts @@ -12,6 +12,7 @@ import { buildRiskScoreFromMapping } from './mappings/build_risk_score_from_mapp import { SignalSourceHit } from './types'; import { buildSeverityFromMapping } from './mappings/build_severity_from_mapping'; import { buildRuleNameFromMapping } from './mappings/build_rule_name_from_mapping'; +import { INTERNAL_IDENTIFIER } from '../../../../common/constants'; interface BuildRuleParams { ruleParams: RuleTypeParams; @@ -64,7 +65,7 @@ export const buildRule = ({ const meta = { ...ruleParams.meta, ...riskScoreMeta, ...severityMeta, ...ruleNameMeta }; - return pickBy((value: unknown) => value != null, { + const rule = pickBy((value: unknown) => value != null, { id, rule_id: ruleParams.ruleId ?? '(unknown rule_id)', actions, @@ -111,4 +112,17 @@ export const buildRule = ({ anomaly_threshold: ruleParams.anomalyThreshold, threshold: ruleParams.threshold, }); + return removeInternalTagsFromRule(rule); +}; + +export const removeInternalTagsFromRule = (rule: Partial): Partial => { + if (rule.tags == null) { + return rule; + } else { + const ruleWithoutInternalTags: Partial = { + ...rule, + tags: rule.tags.filter((tag) => !tag.startsWith(INTERNAL_IDENTIFIER)), + }; + return ruleWithoutInternalTags; + } }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts index 6aebf8815659a6..d684807a09126f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts @@ -5,14 +5,8 @@ */ import { sampleDocNoSortId } from './__mocks__/es_results'; -import { - buildSignal, - buildAncestor, - buildAncestorsSignal, - removeInternalTagsFromRule, -} from './build_signal'; +import { buildSignal, buildParent, buildAncestors, additionalSignalFields } from './build_signal'; import { Signal, Ancestor } from './types'; -import { INTERNAL_RULE_ID_KEY, INTERNAL_IMMUTABLE_KEY } from '../../../../common/constants'; import { getPartialRulesSchemaMock } from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; describe('buildSignal', () => { @@ -24,22 +18,31 @@ describe('buildSignal', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); delete doc._source.event; const rule = getPartialRulesSchemaMock(); - const signal = buildSignal(doc, rule); + const signal = { + ...buildSignal([doc], rule), + ...additionalSignalFields(doc), + }; const expected: Signal = { parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, + parents: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], ancestors: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], original_time: '2020-04-20T21:27:45+0000', @@ -71,6 +74,7 @@ describe('buildSignal', () => { updated_at: signal.rule.updated_at, created_at: signal.rule.created_at, }, + depth: 1, }; expect(signal).toEqual(expected); }); @@ -84,94 +88,31 @@ describe('buildSignal', () => { module: 'system', }; const rule = getPartialRulesSchemaMock(); - const signal = buildSignal(doc, rule); + const signal = { + ...buildSignal([doc], rule), + ...additionalSignalFields(doc), + }; const expected: Signal = { parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, - ancestors: [ + parents: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', - original_event: { - action: 'socket_opened', - dataset: 'socket', - kind: 'event', - module: 'system', - }, - status: 'open', - rule: { - created_by: 'elastic', - description: 'Detecting root and admin users', - enabled: true, - false_positives: [], - from: 'now-6m', - id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - immutable: false, - index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], - interval: '5m', - risk_score: 50, - rule_id: 'rule-1', - language: 'kuery', - max_signals: 100, - name: 'Detect Root/Admin Users', - output_index: '.siem-signals', - query: 'user.name: root or user.name: admin', - references: ['http://www.example.com', 'https://ww.example.com'], - severity: 'high', - updated_by: 'elastic', - tags: ['some fake tag 1', 'some fake tag 2'], - to: 'now', - type: 'query', - note: '', - updated_at: signal.rule.updated_at, - created_at: signal.rule.created_at, - }, - }; - expect(signal).toEqual(expected); - }); - - test('it builds a signal as expected with original_event if is present and without internal tags in them', () => { - const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - doc._source.event = { - action: 'socket_opened', - dataset: 'socket', - kind: 'event', - module: 'system', - }; - const rule = getPartialRulesSchemaMock(); - rule.tags = [ - 'some fake tag 1', - 'some fake tag 2', - `${INTERNAL_RULE_ID_KEY}:rule-1`, - `${INTERNAL_IMMUTABLE_KEY}:true`, - ]; - const signal = buildSignal(doc, rule); - const expected: Signal = { - parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, ancestors: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], original_time: '2020-04-20T21:27:45+0000', @@ -209,6 +150,7 @@ describe('buildSignal', () => { updated_at: signal.rule.updated_at, created_at: signal.rule.created_at, }, + depth: 1, }; expect(signal).toEqual(expected); }); @@ -221,14 +163,12 @@ describe('buildSignal', () => { kind: 'event', module: 'system', }; - const rule = getPartialRulesSchemaMock(); - const signal = buildAncestor(doc, rule); + const signal = buildParent(doc); const expected: Ancestor = { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }; expect(signal).toEqual(expected); }); @@ -242,76 +182,34 @@ describe('buildSignal', () => { module: 'system', }; doc._source.signal = { - parent: { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', - id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, - ancestors: [ + parents: [ { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], - }; - const rule = getPartialRulesSchemaMock(); - const signal = buildAncestor(doc, rule); - const expected: Ancestor = { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'signal', - index: 'myFakeSignalIndex', - depth: 2, - }; - expect(signal).toEqual(expected); - }); - - test('it builds a ancestor correctly if the parent does exist without internal tags in them', () => { - const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - doc._source.event = { - action: 'socket_opened', - dataset: 'socket', - kind: 'event', - module: 'system', - }; - doc._source.signal = { - parent: { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', - id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, ancestors: [ { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], + depth: 1, + rule: { + id: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', + }, }; - const rule = getPartialRulesSchemaMock(); - rule.tags = [ - 'some fake tag 1', - 'some fake tag 2', - `${INTERNAL_RULE_ID_KEY}:rule-1`, - `${INTERNAL_IMMUTABLE_KEY}:true`, - ]; - - const signal = buildAncestor(doc, rule); + const signal = buildParent(doc); const expected: Ancestor = { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'signal', index: 'myFakeSignalIndex', - depth: 2, + depth: 1, }; expect(signal).toEqual(expected); }); @@ -324,15 +222,13 @@ describe('buildSignal', () => { kind: 'event', module: 'system', }; - const rule = getPartialRulesSchemaMock(); - const signal = buildAncestorsSignal(doc, rule); + const signal = buildAncestors(doc); const expected: Ancestor[] = [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ]; expect(signal).toEqual(expected); @@ -347,77 +243,43 @@ describe('buildSignal', () => { module: 'system', }; doc._source.signal = { - parent: { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', - id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, + parents: [ + { + id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], ancestors: [ { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], + rule: { + id: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', + }, + depth: 1, }; - const rule = getPartialRulesSchemaMock(); - const signal = buildAncestorsSignal(doc, rule); + const signal = buildAncestors(doc); const expected: Ancestor[] = [ { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'signal', index: 'myFakeSignalIndex', - depth: 2, + depth: 1, }, ]; expect(signal).toEqual(expected); }); - - test('it removes internal tags from a typical rule', () => { - const rule = getPartialRulesSchemaMock(); - rule.tags = [ - 'some fake tag 1', - 'some fake tag 2', - `${INTERNAL_RULE_ID_KEY}:rule-1`, - `${INTERNAL_IMMUTABLE_KEY}:true`, - ]; - const noInternals = removeInternalTagsFromRule(rule); - expect(noInternals).toEqual(getPartialRulesSchemaMock()); - }); - - test('it works with an empty array', () => { - const rule = getPartialRulesSchemaMock(); - rule.tags = []; - const noInternals = removeInternalTagsFromRule(rule); - const expected = getPartialRulesSchemaMock(); - expected.tags = []; - expect(noInternals).toEqual(expected); - }); - - test('it works if tags does not exist', () => { - const rule = getPartialRulesSchemaMock(); - delete rule.tags; - const noInternals = removeInternalTagsFromRule(rule); - const expected = getPartialRulesSchemaMock(); - delete expected.tags; - expect(noInternals).toEqual(expected); - }); - - test('it works if tags contains normal values and no internal values', () => { - const rule = getPartialRulesSchemaMock(); - const noInternals = removeInternalTagsFromRule(rule); - expect(noInternals).toEqual(rule); - }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index e7098c015c1654..78818779dd661c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -5,35 +5,41 @@ */ import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; -import { INTERNAL_IDENTIFIER } from '../../../../common/constants'; import { SignalSourceHit, Signal, Ancestor } from './types'; -export const buildAncestor = (doc: SignalSourceHit, rule: Partial): Ancestor => { - const existingSignal = doc._source.signal?.parent; - if (existingSignal != null) { +/** + * Takes a parent signal or event document and extracts the information needed for the corresponding entry in the child + * signal's `signal.parents` array. + * @param doc The parent signal or event + */ +export const buildParent = (doc: SignalSourceHit): Ancestor => { + if (doc._source.signal != null) { return { - rule: rule.id != null ? rule.id : '', + rule: doc._source.signal.rule.id, id: doc._id, type: 'signal', index: doc._index, - depth: existingSignal.depth + 1, + // We first look for signal.depth and use that if it exists. If it doesn't exist, this should be a pre-7.10 signal + // and should have signal.parent.depth instead. signal.parent.depth in this case is treated as equivalent to signal.depth. + depth: doc._source.signal.depth ?? doc._source.signal.parent?.depth ?? 1, }; } else { return { - rule: rule.id != null ? rule.id : '', id: doc._id, type: 'event', index: doc._index, - depth: 1, + depth: 0, }; } }; -export const buildAncestorsSignal = ( - doc: SignalSourceHit, - rule: Partial -): Signal['ancestors'] => { - const newAncestor = buildAncestor(doc, rule); +/** + * Takes a parent signal or event document with N ancestors and adds the parent document to the ancestry array, + * creating an array of N+1 ancestors. + * @param doc The parent signal/event for which to extend the ancestry. + */ +export const buildAncestors = (doc: SignalSourceHit): Ancestor[] => { + const newAncestor = buildParent(doc); const existingAncestors = doc._source.signal?.ancestors; if (existingAncestors != null) { return [...existingAncestors, newAncestor]; @@ -42,35 +48,33 @@ export const buildAncestorsSignal = ( } }; -export const buildSignal = (doc: SignalSourceHit, rule: Partial): Signal => { - const ruleWithoutInternalTags = removeInternalTagsFromRule(rule); - const parent = buildAncestor(doc, rule); - const ancestors = buildAncestorsSignal(doc, rule); - let signal: Signal = { - parent, +/** + * Builds the `signal.*` fields that are common across all signals. + * @param docs The parent signals/events of the new signal to be built. + * @param rule The rule that is generating the new signal. + */ +export const buildSignal = (docs: SignalSourceHit[], rule: Partial): Signal => { + const parents = docs.map(buildParent); + const depth = parents.reduce((acc, parent) => Math.max(parent.depth, acc), 0) + 1; + const ancestors = docs.reduce((acc: Ancestor[], doc) => acc.concat(buildAncestors(doc)), []); + return { + parents, ancestors, - original_time: doc._source['@timestamp'], status: 'open', - rule: ruleWithoutInternalTags, + rule, + depth, }; - if (doc._source.event != null) { - signal = { ...signal, original_event: doc._source.event }; - } - if (doc._source.threshold_count != null) { - signal = { ...signal, threshold_count: doc._source.threshold_count }; - delete doc._source.threshold_count; - } - return signal; }; -export const removeInternalTagsFromRule = (rule: Partial): Partial => { - if (rule.tags == null) { - return rule; - } else { - const ruleWithoutInternalTags: Partial = { - ...rule, - tags: rule.tags.filter((tag) => !tag.startsWith(INTERNAL_IDENTIFIER)), - }; - return ruleWithoutInternalTags; - } +/** + * Creates signal fields that are only available in the special case where a signal has only 1 parent signal/event. + * @param doc The parent signal/event of the new signal to be built. + */ +export const additionalSignalFields = (doc: SignalSourceHit) => { + return { + parent: buildParent(doc), + original_time: doc._source['@timestamp'], + original_event: doc._source.event ?? undefined, + threshold_count: doc._source.threshold_count ?? undefined, + }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index da17d4a1f123a7..7ee157beec789c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -120,7 +120,6 @@ export const signalRulesAlertType = ({ enabled, schedule: { interval }, throttle, - params: ruleParams, } = savedObject.attributes; const updatedAt = savedObject.updated_at ?? ''; const refresh = actions.length ? 'wait_for' : false; @@ -343,7 +342,7 @@ export const signalRulesAlertType = ({ if (result.success) { if (actions.length) { const notificationRuleParams: NotificationRuleTypeParams = { - ...ruleParams, + ...params, name, id: savedObject.id, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts index 8b9fb0574efe97..41c825ea4d9787 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts @@ -291,37 +291,7 @@ describe('singleBulkCreate', () => { test('filter duplicate rules will return nothing filtered when the two rule ids do not match with each other', () => { const filtered = filterDuplicateRules('some id', sampleDocWithAncestors()); - expect(filtered).toEqual([ - { - _index: 'myFakeSignalIndex', - _type: 'doc', - _score: 100, - _version: 1, - _id: 'e1e08ddc-5e37-49ff-a258-5393aa44435a', - _source: { - someKey: 'someValue', - '@timestamp': '2020-04-20T21:27:45+0000', - signal: { - parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, - ancestors: [ - { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, - ], - }, - }, - }, - ]); + expect(filtered).toEqual(sampleDocWithAncestors().hits.hits); }); test('filters duplicate rules will return empty array when the two rule ids match each other', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts index 74709f31563eed..be71c67615a4c6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts @@ -51,7 +51,10 @@ export const filterDuplicateRules = ( if (doc._source.signal == null) { return true; } else { - return !doc._source.signal.ancestors.some((ancestor) => ancestor.rule === ruleId); + return !( + doc._source.signal.ancestors.some((ancestor) => ancestor.rule === ruleId) || + doc._source.signal.rule.id === ruleId + ); } }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index aecdbe10695d26..700a8fb5022d72 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -44,8 +44,16 @@ export interface SignalSource { [key: string]: SearchTypes; '@timestamp': string; signal?: { - parent: Ancestor; + // parent is deprecated: new signals should populate parents instead + // both are optional until all signals with parent are gone and we can safely remove it + parent?: Ancestor; + parents?: Ancestor[]; ancestors: Ancestor[]; + rule: { + id: string; + }; + // signal.depth doesn't exist on pre-7.10 signals + depth?: number; }; } @@ -113,7 +121,7 @@ export type SignalRuleAlertTypeDefinition = Omit & { }; export interface Ancestor { - rule: string; + rule?: string; id: string; type: string; index: string; @@ -122,12 +130,15 @@ export interface Ancestor { export interface Signal { rule: Partial; - parent: Ancestor; + // DEPRECATED: use parents instead of parent + parent?: Ancestor; + parents: Ancestor[]; ancestors: Ancestor[]; - original_time: string; + original_time?: string; original_event?: SearchTypes; status: Status; threshold_count?: SearchTypes; + depth: number; } export interface SignalHit {