diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 4ed0610a0b8..7cbc3127d9d 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -4904,6 +4904,17 @@ Logger name -- +*`elasticsearch.slowlog.took`*:: ++ +-- +type: keyword + +example: 300ms + +Time it took to execute the query + +-- + *`elasticsearch.slowlog.types`*:: + -- diff --git a/filebeat/module/elasticsearch/slowlog/_meta/fields.yml b/filebeat/module/elasticsearch/slowlog/_meta/fields.yml index 5e4517d04f1..85eaa48dbed 100644 --- a/filebeat/module/elasticsearch/slowlog/_meta/fields.yml +++ b/filebeat/module/elasticsearch/slowlog/_meta/fields.yml @@ -7,6 +7,10 @@ description: "Logger name" example: "index.search.slowlog.fetch" type: keyword + - name: took + description: "Time it took to execute the query" + example: "300ms" + type: keyword - name: types description: "Types" example: "" diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json index 3e67a971800..1981c3711f7 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json @@ -41,7 +41,6 @@ { "remove": { "field": [ - "elasticsearch.slowlog.took", "elasticsearch.slowlog.timestamp" ] } diff --git a/filebeat/module/elasticsearch/slowlog/test/auditlog_index_indexing_slowlog.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/auditlog_index_indexing_slowlog.log-expected.json index 1d5a5b89ffe..32dda026a23 100644 --- a/filebeat/module/elasticsearch/slowlog/test/auditlog_index_indexing_slowlog.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/auditlog_index_indexing_slowlog.log-expected.json @@ -9,6 +9,7 @@ "elasticsearch.slowlog.logger": "index.indexing.slowlog.index", "elasticsearch.slowlog.routing": "", "elasticsearch.slowlog.source_query": "{\"@timestamp\":\"2018-07-04T21:50:40.799Z\",\"metricset\":{\"module\":\"system\",\"rtt\":9610,\"name\":\"network\"},\"system\":{\"network\":{\"name\":\"bridg\",\"in\":{\"packets\":0,\"errors\":0,\"dropped\":0,\"bytes\":0},\"out\":{\"errors\":0,\"dropped\":0,\"packets\":1,\"bytes\":342}}},\"beat\":{\"name\":\"Rados-MacBook-Pro.local\",\"hostname\":\"Rados-MacBook-Pro.local\",\"version\":\"6.3.0\"},\"host\":{\"name\":\"Rados-MacBook-Pro.local\"}}", + "elasticsearch.slowlog.took": "221micros", "elasticsearch.slowlog.type": "doc", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, @@ -30,6 +31,7 @@ "elasticsearch.slowlog.logger": "index.indexing.slowlog.index", "elasticsearch.slowlog.routing": "", "elasticsearch.slowlog.source_query": "{\"@timestamp\":\"2018-07-04T21:50:40.799Z\",\"metricset\":{\"rtt\":9616,\"name\":\"network\",\"module\":\"system\"},\"system\":{\"network\":{\"name\":\"utun0\",\"in\":{\"dropped\":0,\"bytes\":0,\"packets\":0,\"errors\":0},\"out\":{\"packets\":2,\"bytes\":200,\"errors\":0,\"dropped\":0}}},\"beat\":{\"version\":\"6.3.0\",\"name\":\"Rados-MacBook-Pro.local\",\"hostname\":\"Rados-MacBook-Pro.local\"},\"host\":{\"name\":\"Rados-MacBook-Pro.local\"}}", + "elasticsearch.slowlog.took": "388.6micros", "elasticsearch.slowlog.type": "doc", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, @@ -51,6 +53,7 @@ "elasticsearch.slowlog.logger": "index.indexing.slowlog.index", "elasticsearch.slowlog.routing": "", "elasticsearch.slowlog.source_query": "{\"@timestamp\":\"2018-07-04T21:50:40.799Z\",\"metricset\":{\"rtt\":9640,\"name\":\"network\",\"module\":\"system\"},\"system\":{\"network\":{\"name\":\"utun1\",\"in\":{\"packets\":200,\"errors\":0,\"dropped\":0,\"bytes\":44296},\"out\":{\"errors\":0,\"dropped\":0,\"packets\":208,\"bytes\":59626}}},\"beat\":{\"name\":\"Rados-MacBook-Pro.local\",\"hostname\":\"Rados-MacBook-Pro.local\",\"version\":\"6.3.0\"},\"host\":{\"name\":\"Rados-MacBook-Pro.local\"}}", + "elasticsearch.slowlog.took": "287.1micros", "elasticsearch.slowlog.type": "doc", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, @@ -72,6 +75,7 @@ "elasticsearch.slowlog.logger": "index.indexing.slowlog.index", "elasticsearch.slowlog.routing": "", "elasticsearch.slowlog.source_query": "\n{\n \"@timestamp\":\"2018-07-04T21:27:30.730Z\",\n \"metricset\":{\n \"name\":\"network\",\n \"module\":\"system\",\n \"rtt\":7264},\n \"system\":{\n \"network\":{\n \"name\":\"lo0\",\n \"in\":{\n \"errors\":0,\n \"dropped\":0,\n \"bytes\":77666873,\n \"packets\":244595},\n \"out\":{\n \"packets\":244595,\n \"bytes\":77666873,\n \"errors\":0,\n \"dropped\":0\n }\n }\n },\n \"beat\":{\n \"name\":\"Rados-MacBook-Pro.local\",\n \"hostname\":\"Rados-MacBook-Pro.local\",\n \"version\":\"6.3.0\"\n },\n \"host\":{\n \"name\":\"Rados-MacBook-Pro.local\"\n }\n }", + "elasticsearch.slowlog.took": "1.7ms", "elasticsearch.slowlog.type": "doc", "event.dataset": "elasticsearch.slowlog", "event.duration": 1000000, @@ -96,6 +100,7 @@ "elasticsearch.slowlog.logger": "index.indexing.slowlog.index", "elasticsearch.slowlog.routing": "", "elasticsearch.slowlog.source_query": "{\"@timestamp\":\"2018-07-04T21:50:42.117Z\",\"beat\":{\"name\":\"Rados-MacBook-Pro.local\",\"hostname\":\"Rados-MacBook-Pro.local\",\"version\":\"6.3.0\"},\"host\":{\"name\":\"Rados-MacBook-Pro.local\"},\"metricset\":{\"module\":\"system\",\"rtt\":39463,\"name\":\"process\"},\"system\":{\"process\":{\"state\":\"running\",\"pid\":6274,\"name\":\"iTerm2\",\"cmdline\":\"/Applications/iTerm.app/Contents/MacOS/iTerm2\",\"ppid\":1,\"pgid\":6274,\"username\":\"rado\",\"memory\":{\"size\":6263349248,\"rss\":{\"bytes\":226975744,\"pct\":0.0132},\"share\":0},\"cpu\":{\"total\":{\"value\":921790,\"pct\":0.1368,\"norm\":{\"pct\":0.0342}},\"start_time\":\"2018-07-02T10:40:29.756Z\"}}}}", + "elasticsearch.slowlog.took": "560.6micros", "elasticsearch.slowlog.type": "doc", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, @@ -117,6 +122,7 @@ "elasticsearch.slowlog.logger": "index.indexing.slowlog.index", "elasticsearch.slowlog.routing": "", "elasticsearch.slowlog.source_query": "{\"@timestamp\":\"2018-07-04T21:50:42.117Z\",\"beat\":{\"name\":\"Rados-MacBook-Pro.local\",\"hostname\":\"Rados-MacBook-Pro.local\",\"version\":\"6.3.0\"},\"host\":{\"name\":\"Rados-MacBook-Pro.local\"},\"metricset\":{\"name\":\"process\",\"module\":\"system\",\"rtt\":39476},\"system\":{\"process\":{\"username\":\"rado\",\"state\":\"running\",\"cmdline\":\"com.docker.hyperkit -A -u -F vms/0/hyperkit.pid -c 2 -m 6144M -s 0:0,hostbridge -s 31,lpc -s 1:0,virtio-vpnkit,path=s50,uuid=18fcb277-636a-4fd7-99d2-9bd2dd50a58c -U b1496a26-aed9-4ee1-818d-a3683593b754 -s 2:0,ahci-hd,file:///Users/rado/Library/Containers/com.docker.docker/Data/vms/0/Docker.qcow2?sync=os\\u0026buffered=1,format=qcow,qcow-config=discard=true;compact_after_unmaps=262144;keep_erased=262144;runtime_asserts=false -s 3,virtio-sock,guest_cid=3,path=vms/0,guest_forwards=2376;1525 -s 4,ahci-cd,/Applications/Docker.app/Contents/Resources/linuxkit/docker-for-mac.iso -s 5,ahci-cd,vms/0/config.iso -s 6,virtio-rnd -s 7,virtio-9p,path=s51,tag=port -l com1,autopty=vms/0/tty,log=vms/0/console-ring -f bootrom,/Applications/Docker.app/Contents/Resources/uefi/UEFI.fd,,\",\"ppid\":559,\"pgid\":555,\"name\":\"com.docker.hype\",\"cpu\":{\"total\":{\"pct\":0.1181,\"norm\":{\"pct\":0.0295},\"value\":8.7575e+06},\"start_time\":\"2018-07-01T22:13:07.748Z\"},\"pid\":567,\"memory\":{\"share\":0,\"size\":11128897536,\"rss\":{\"pct\":0.0205,\"bytes\":352854016}}}}}", + "elasticsearch.slowlog.took": "469.9micros", "elasticsearch.slowlog.type": "doc", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, diff --git a/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json index 6052e602647..92559d982bf 100644 --- a/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json @@ -12,6 +12,7 @@ "group1", "group2" ], + "elasticsearch.slowlog.took": "4.5ms", "elasticsearch.slowlog.total_hits": 19435, "elasticsearch.slowlog.total_shards": 1, "elasticsearch.slowlog.types": "", @@ -35,6 +36,7 @@ "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH", "elasticsearch.slowlog.source_query": "{\"query\":{\"match_all\":{\"boost\":1.0}}}", "elasticsearch.slowlog.stats": "", + "elasticsearch.slowlog.took": "10.8ms", "elasticsearch.slowlog.total_hits": 19435, "elasticsearch.slowlog.total_shards": 1, "elasticsearch.slowlog.types": "", @@ -58,6 +60,7 @@ "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH", "elasticsearch.slowlog.source_query": "{\"size\":500,\"query\":{\"match_none\":{\"boost\":1.0}},\"version\":true,\"_source\":{\"includes\":[],\"excludes\":[]},\"stored_fields\":\"*\",\"docvalue_fields\":[\"@timestamp\",\"ceph.monitor_health.last_updated\",\"docker.container.created\",\"docker.healthcheck.event.end_date\",\"docker.healthcheck.event.start_date\",\"docker.image.created\",\"kubernetes.container.start_time\",\"kubernetes.event.metadata.timestamp.created\",\"kubernetes.node.start_time\",\"kubernetes.pod.start_time\",\"kubernetes.system.start_time\",\"mongodb.status.background_flushing.last_finished\",\"mongodb.status.local_time\",\"php_fpm.pool.start_time\",\"postgresql.activity.backend_start\",\"postgresql.activity.query_start\",\"postgresql.activity.state_change\",\"postgresql.activity.transaction_start\",\"postgresql.bgwriter.stats_reset\",\"postgresql.database.stats_reset\",\"system.process.cpu.start_time\"],\"script_fields\":{},\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"unmapped_type\":\"boolean\"}}],\"aggregations\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"time_zone\":\"Europe/Berlin\",\"interval\":\"30s\",\"offset\":0,\"order\":{\"_key\":\"asc\"},\"keyed\":false,\"min_doc_count\":1}}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fragment_size\":2147483647,\"fields\":{\"*\":{}}}}", "elasticsearch.slowlog.stats": "", + "elasticsearch.slowlog.took": "124.3ms", "elasticsearch.slowlog.total_hits": 0, "elasticsearch.slowlog.total_shards": 1, "elasticsearch.slowlog.types": "", @@ -81,6 +84,7 @@ "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH", "elasticsearch.slowlog.source_query": "{\"size\":500,\"query\":{\"match_none\":{\"boost\":1.0}},\"version\":true,\"_source\":{\"includes\":[],\"excludes\":[]},\"stored_fields\":\"*\",\"docvalue_fields\":[\"@timestamp\",\"ceph.monitor_health.last_updated\",\"docker.container.created\",\"docker.healthcheck.event.end_date\",\"docker.healthcheck.event.start_date\",\"docker.image.created\",\"kubernetes.container.start_time\",\"kubernetes.event.metadata.timestamp.created\",\"kubernetes.node.start_time\",\"kubernetes.pod.start_time\",\"kubernetes.system.start_time\",\"mongodb.status.background_flushing.last_finished\",\"mongodb.status.local_time\",\"php_fpm.pool.start_time\",\"postgresql.activity.backend_start\",\"postgresql.activity.query_start\",\"postgresql.activity.state_change\",\"postgresql.activity.transaction_start\",\"postgresql.bgwriter.stats_reset\",\"postgresql.database.stats_reset\",\"system.process.cpu.start_time\"],\"script_fields\":{},\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"unmapped_type\":\"boolean\"}}],\"aggregations\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"time_zone\":\"Europe/Berlin\",\"interval\":\"30s\",\"offset\":0,\"order\":{\"_key\":\"asc\"},\"keyed\":false,\"min_doc_count\":1}}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fragment_size\":2147483647,\"fields\":{\"*\":{}}}}", "elasticsearch.slowlog.stats": "", + "elasticsearch.slowlog.took": "7.2ms", "elasticsearch.slowlog.total_hits": 0, "elasticsearch.slowlog.total_shards": 1, "elasticsearch.slowlog.types": "", @@ -104,6 +108,7 @@ "elasticsearch.slowlog.logger": "index.indexing.slowlog.index", "elasticsearch.slowlog.routing": "", "elasticsearch.slowlog.source_query": "{\"@timestamp\":\"2018-07-04T13:47:50.747Z\",\"system\":{\"process\":{\"ppid\":34526,\"state\":\"running\",\"cpu\":{\"total\":{\"value\":734879,\"pct\":0.0173,\"norm\":{\"pct\":0.0043}},\"start_time\":\"2018-07-04T06:56:34.863Z\"},\"pgid\":34526,\"cmdline\":\"/Applications/Firefox.app/Contents/MacOS/plugin-container.app/Contents/MacOS/plugin-container -childID 1 -isForBrowser -prefsLen 22119 -schedulerPrefs 0001,2 -greomni /Applications/Firefox.app/Contents/Resources/omni.ja -appomni /Applications/Firefox.app/Contents/Resources/browser/omni.ja -appdir /Applications/Firefox.app/Contents/Resources/browser -profile /Users/rado/Library/Application Support/Firefox/Profiles/pt6eoq1j.default-1484133908360 34526 gecko-crash-server-pipe.34526 org.mozilla.machname.231926932 tab\",\"name\":\"plugin-containe\",\"memory\":{\"size\":7489249280,\"rss\":{\"bytes\":567619584,\"pct\":0.033},\"share\":0},\"pid\":34528,\"username\":\"rado\"}},\"metricset\":{\"name\":\"process\",\"module\":\"system\",\"rtt\":43856},\"beat\":{\"hostname\":\"Rados-MacBook-Pro.local\",\"version\":\"6.3.0\",\"name\":\"Rados-MacBook-Pro.local\"},\"host\":{\"name\":\"Rados-MacBook-Pro.local\"}}", + "elasticsearch.slowlog.took": "1.4ms", "elasticsearch.slowlog.type": "doc", "event.dataset": "elasticsearch.slowlog", "event.duration": 1000000, @@ -125,6 +130,7 @@ "elasticsearch.slowlog.logger": "index.indexing.slowlog.index", "elasticsearch.slowlog.routing": "", "elasticsearch.slowlog.source_query": "\n{\n \"@timestamp\":\"2018-07-04T21:27:30.730Z\",\n \"metricset\":{\n \"name\":\"network\",\n \"module\":\"system\",\n \"rtt\":7264},\n \"system\":{\n \"network\":{\n \"name\":\"lo0\",\n \"in\":{\n \"errors\":0,\n \"dropped\":0,\n \"bytes\":77666873,\n \"packets\":244595},\n \"out\":{\n \"packets\":244595,\n \"bytes\":77666873,\n \"errors\":0,\n \"dropped\":0\n }\n }\n },\n \"beat\":{\n \"name\":\"Rados-MacBook-Pro.local\",\n \"hostname\":\"Rados-MacBook-Pro.local\",\n \"version\":\"6.3.0\"\n },\n \"host\":{\n \"name\":\"Rados-MacBook-Pro.local\"\n }\n }", + "elasticsearch.slowlog.took": "1.7ms", "elasticsearch.slowlog.type": "doc", "event.dataset": "elasticsearch.slowlog", "event.duration": 1000000,