Skip to content

Commit

Permalink
[DOCS] EQL: Use data streams in docs (#70822) (#70873)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jrodewig
jrodewig authored Mar 25, 2021
1 parent 9948115 commit 8f78784
Show file tree
Hide file tree
Showing 5 changed files with 276 additions and 225 deletions.
44 changes: 24 additions & 20 deletions docs/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,28 +201,30 @@ buildRestTests.setups['messages'] = '''
// Used for EQL
buildRestTests.setups['sec_logs'] = '''
- do:
indices.create:
index: my-index-000001
body:
settings:
number_of_shards: 1
number_of_replicas: 1
indices.put_index_template:
name: my-data-stream-template
body: |
{
"index_patterns": [ "my-data-stream*" ],
"data_stream": { },
"priority": 500
}
- do:
bulk:
index: my-index-000001
index: my-data-stream
refresh: true
body: |
{"index":{}}
{"create":{}}
{"@timestamp": "2099-12-06T11:04:05.000Z", "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }}
{"index":{}}
{"create":{}}
{"@timestamp": "2099-12-06T11:04:07.000Z", "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2099-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\\\Windows\\\\System32\\\\cmd.exe", "type": "file", "size": 16384 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }}
{"index":{}}
{"create":{}}
{"@timestamp": "2099-12-07T11:06:07.000Z", "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" } }
{"index":{}}
{"create":{}}
{"@timestamp": "2099-12-07T11:07:09.000Z", "event": { "category": "process", "id": "aR3NWVOs", "sequence": 4 }, "process": { "pid": 2012, "name": "regsvr32.exe", "command_line": "regsvr32.exe /s /u /i:https://...RegSvr32.sct scrobj.dll", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }}
{"index":{}}
{"create":{}}
{"@timestamp": "2099-12-07T11:07:10.000Z", "event": { "category": "file", "id": "tZ1NWVOs", "sequence": 5 }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }, "file": { "path": "C:\\\\Windows\\\\System32\\\\scrobj.dll", "name": "scrobj.dll" }}
{"index":{}}
{"create":{}}
{"@timestamp": "2099-12-07T11:07:10.000Z", "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }}'''

buildRestTests.setups['host'] = '''
Expand Down Expand Up @@ -1502,15 +1504,17 @@ setups['setup-repository'] = '''
// Fake sec logs data used by EQL search
setups['atomic_red_regsvr32'] = '''
- do:
indices.create:
index: my-index-000001
body:
settings:
number_of_shards: 5
number_of_routing_shards: 5
indices.put_index_template:
name: my-data-stream-template
body: |
{
"index_patterns": [ "my-data-stream*" ],
"data_stream": { },
"priority": 500
}
- do:
bulk:
index: my-index-000001
index: my-data-stream
refresh: true
body: |
#atomic_red_data#
Expand Down
55 changes: 40 additions & 15 deletions docs/reference/eql/detect-threats-with-eql.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,32 +36,54 @@ events imitating a Squiblydoo attack. The data has been mapped to

To get started:

. Create an <<index-templates,index template>> with
<<create-a-data-stream-template,data stream enabled>>:
+
////
[source,console]
----
DELETE /_data_stream/*
DELETE /_index_template/*
----
// TEARDOWN
////
+
[source,console]
----
PUT /_index_template/my-data-stream-template
{
"index_patterns": [ "my-data-stream*" ],
"data_stream": { },
"priority": 500
}
----

. Download https://raw.githubusercontent.com/elastic/elasticsearch/{branch}/docs/src/test/resources/normalized-T1117-AtomicRed-regsvr32.json[`normalized-T1117-AtomicRed-regsvr32.json`].

. Use the <<docs-bulk,bulk API>> to index the data:
. Use the <<docs-bulk,bulk API>> to index the data to a matching stream:
+
[source,sh]
----
curl -H "Content-Type: application/json" -XPOST "localhost:9200/my-index-000001/_bulk?pretty&refresh" --data-binary "@normalized-T1117-AtomicRed-regsvr32.json"
curl -H "Content-Type: application/json" -XPOST "localhost:9200/my-data-stream/_bulk?pretty&refresh" --data-binary "@normalized-T1117-AtomicRed-regsvr32.json"
----
// NOTCONSOLE

. Use the <<cat-indices,cat indices API>> to verify the data was indexed:
+
[source,console]
----
GET /_cat/indices/my-index-000001?v=true&h=health,status,index,docs.count
GET /_cat/indices/my-data-stream?v=true&h=health,status,index,docs.count
----
// TEST[setup:atomic_red_regsvr32]
+
The response should show a `docs.count` of `150`.
+
[source,txt]
----
health status index docs.count
yellow open my-index-000001 150
health status index docs.count
yellow open .ds-my-data-stream-2099.12.07-000001 150
----
// TESTRESPONSE[non_json]
// TESTRESPONSE[s/.ds-my-data-stream-2099.12.07-000001/.+/ non_json]

[discrete]
[[eql-ex-get-a-count-of-regsvr32-events]]
Expand All @@ -71,7 +93,7 @@ First, get a count of events associated with a `regsvr32.exe` process:

[source,console]
----
GET /my-index-000001/_eql/search?filter_path=-hits.events <1>
GET /my-data-stream/_eql/search?filter_path=-hits.events <1>
{
"query": """
any where process.name == "regsvr32.exe" <2>
Expand Down Expand Up @@ -116,7 +138,7 @@ utility. Narrow your results to processes where the command line was used:

[source,console]
----
GET /my-index-000001/_eql/search
GET /my-data-stream/_eql/search
{
"query": """
process where process.name == "regsvr32.exe" and process.command_line.keyword != null
Expand Down Expand Up @@ -144,7 +166,7 @@ This fits the behavior of a Squiblydoo attack.
},
"events": [
{
"_index": "my-index-000001",
"_index": ".ds-my-data-stream-2099.12.07-000001",
"_id": "gl5MJXMBMk1dGnErnBW8",
"_source": {
"process": {
Expand Down Expand Up @@ -178,6 +200,7 @@ This fits the behavior of a Squiblydoo attack.
}
----
// TESTRESPONSE[s/"took": 21/"took": $body.took/]
// TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.events.0._index/]
// TESTRESPONSE[s/"_id": "gl5MJXMBMk1dGnErnBW8"/"_id": $body.hits.events.0._id/]

[discrete]
Expand All @@ -188,7 +211,7 @@ Check if `regsvr32.exe` later loads the `scrobj.dll` library:

[source,console]
----
GET /my-index-000001/_eql/search
GET /my-data-stream/_eql/search
{
"query": """
library where process.name == "regsvr32.exe" and dll.name == "scrobj.dll"
Expand All @@ -213,7 +236,7 @@ The query matches an event, confirming `scrobj.dll` was loaded.
},
"events": [
{
"_index": "my-index-000001",
"_index": ".ds-my-data-stream-2099.12.07-000001",
"_id": "ol5MJXMBMk1dGnErnBW8",
"_source": {
"process": {
Expand All @@ -237,6 +260,7 @@ The query matches an event, confirming `scrobj.dll` was loaded.
}
----
// TESTRESPONSE[s/"took": 5/"took": $body.took/]
// TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.events.0._index/]
// TESTRESPONSE[s/"_id": "ol5MJXMBMk1dGnErnBW8"/"_id": $body.hits.events.0._id/]

[discrete]
Expand All @@ -258,7 +282,7 @@ detect similar threats.

[source,console]
----
GET /my-index-000001/_eql/search
GET /my-data-stream/_eql/search
{
"query": """
sequence by process.pid
Expand Down Expand Up @@ -291,7 +315,7 @@ The query matches a sequence, indicating the attack likely succeeded.
],
"events": [
{
"_index": "my-index-000001",
"_index": ".ds-my-data-stream-2099.12.07-000001",
"_id": "gl5MJXMBMk1dGnErnBW8",
"_source": {
"process": {
Expand Down Expand Up @@ -321,7 +345,7 @@ The query matches a sequence, indicating the attack likely succeeded.
}
},
{
"_index": "my-index-000001",
"_index": ".ds-my-data-stream-2099.12.07-000001",
"_id": "ol5MJXMBMk1dGnErnBW8",
"_source": {
"process": {
Expand All @@ -341,7 +365,7 @@ The query matches a sequence, indicating the attack likely succeeded.
}
},
{
"_index": "my-index-000001",
"_index": ".ds-my-data-stream-2099.12.07-000001",
"_id": "EF5MJXMBMk1dGnErnBa9",
"_source": {
"process": {
Expand Down Expand Up @@ -380,6 +404,7 @@ The query matches a sequence, indicating the attack likely succeeded.
}
----
// TESTRESPONSE[s/"took": 25/"took": $body.took/]
// TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.sequences.0.events.0._index/]
// TESTRESPONSE[s/"_id": "gl5MJXMBMk1dGnErnBW8"/"_id": $body.hits.sequences.0.events.0._id/]
// TESTRESPONSE[s/"_id": "ol5MJXMBMk1dGnErnBW8"/"_id": $body.hits.sequences.0.events.1._id/]
// TESTRESPONSE[s/"_id": "EF5MJXMBMk1dGnErnBa9"/"_id": $body.hits.sequences.0.events.2._id/]
25 changes: 18 additions & 7 deletions docs/reference/eql/eql-search-api.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,18 @@ Returns search results for an <<eql,Event Query Language (EQL)>> query.
EQL assumes each document in a data stream or index corresponds to an
event.

////
[source,console]
----
GET /my-index-000001/_eql/search
DELETE /_data_stream/*
DELETE /_index_template/*
----
// TEARDOWN
////

[source,console]
----
GET /my-data-stream/_eql/search
{
"query": """
process where process.name == "regsvr32.exe"
Expand Down Expand Up @@ -533,7 +542,7 @@ The following EQL search request searches for events with an `event.category` of

[source,console]
----
GET /my-index-000001/_eql/search
GET /my-data-stream/_eql/search
{
"query": """
process where (process.name == "cmd.exe" and process.pid != 2013)
Expand Down Expand Up @@ -565,7 +574,7 @@ the events in ascending order.
},
"events": [
{
"_index": "my-index-000001",
"_index": ".ds-my-data-stream-2099.12.07-000001",
"_id": "babI3XMBI9IjHuIqU0S_",
"_source": {
"@timestamp": "2099-12-06T11:04:05.000Z",
Expand All @@ -582,7 +591,7 @@ the events in ascending order.
}
},
{
"_index": "my-index-000001",
"_index": ".ds-my-data-stream-2099.12.07-000001",
"_id": "b6bI3XMBI9IjHuIqU0S_",
"_source": {
"@timestamp": "2099-12-07T11:06:07.000Z",
Expand All @@ -603,6 +612,7 @@ the events in ascending order.
}
----
// TESTRESPONSE[s/"took": 6/"took": $body.took/]
// TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.events.0._index/]
// TESTRESPONSE[s/"_id": "babI3XMBI9IjHuIqU0S_"/"_id": $body.hits.events.0._id/]
// TESTRESPONSE[s/"_id": "b6bI3XMBI9IjHuIqU0S_"/"_id": $body.hits.events.1._id/]

Expand Down Expand Up @@ -630,7 +640,7 @@ These events must also share the same `process.pid` value.

[source,console]
----
GET /my-index-000001/_eql/search
GET /my-data-stream/_eql/search
{
"query": """
sequence by process.pid
Expand Down Expand Up @@ -664,7 +674,7 @@ shared `process.pid` value for each matching event.
],
"events": [
{
"_index": "my-index-000001",
"_index": ".ds-my-data-stream-2099.12.07-000001",
"_id": "AtOJ4UjUBAAx3XR5kcCM",
"_source": {
"@timestamp": "2099-12-06T11:04:07.000Z",
Expand All @@ -688,7 +698,7 @@ shared `process.pid` value for each matching event.
}
},
{
"_index": "my-index-000001",
"_index": ".ds-my-data-stream-2099.12.07-000001",
"_id": "OQmfCaduce8zoHT93o4H",
"_source": {
"@timestamp": "2099-12-07T11:07:09.000Z",
Expand All @@ -712,5 +722,6 @@ shared `process.pid` value for each matching event.
}
----
// TESTRESPONSE[s/"took": 6/"took": $body.took/]
// TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.sequences.0.events.0._index/]
// TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.sequences.0.events.0._id/]
// TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.sequences.0.events.1._id/]
Loading

0 comments on commit 8f78784

Please sign in to comment.