Support mariadb slowlog

CHANGELOG.next.asciidoc

@@ -120,6 +120,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for ssl_request_log in apache2 module. {issue}8088[8088] {pull}9833[9833]
 - Add support for iis 7.5 log format. {issue}9753[9753] {pull}9967[9967]
 - Add service.type field to all Modules. By default the field is set with the module name. It can be overwritten with `service.type` config. {pull}10042[10042]
+- Add support for MariaDB in the `slowlog` fileset of `mysql` module. {pull}9731[9731]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -6418,6 +6418,16 @@ type: long
 The number of rows scanned by the query.
 
 
+--
+
+*`mysql.slowlog.rows_affected`*::
++
+--
+type: long
+
+The number of rows modified by the query.
+
+
 --
 
 *`mysql.slowlog.timestamp`*::
@@ -6446,6 +6456,126 @@ type: long
 The connection ID for the query.
 
 
+--
+
+*`mysql.slowlog.thread_id`*::
++
+--
+type: long
+
+ID of the thread executing this query.
+
+
+--
+
+*`mysql.slowlog.schema`*::
++
+--
+type: keyword
+
+The schema where the slow query was executed.
+
+
+--
+
+*`mysql.slowlog.current_user`*::
++
+--
+type: keyword
+
+Current authenticated user, used to determine access privileges. Can differ from the value for user.
+
+
+--
+
+*`mysql.slowlog.query_cache_hit`*::
++
+--
+type: keyword
+
+Whether the query cache was hit.
+
+
+--
+
+*`mysql.slowlog.tmp_table`*::
++
+--
+type: keyword
+
+Whether a temporary table was used to resolve the query.
+
+
+--
+
+*`mysql.slowlog.tmp_table_on_disk`*::
++
+--
+type: keyword
+
+Whether the query needed temporary tables on disk.
+
+
+--
+
+*`mysql.slowlog.filesort`*::
++
+--
+type: keyword
+
+Whether filesort optimization was used.
+
+
+--
+
+*`mysql.slowlog.filesort_on_disk`*::
++
+--
+type: keyword
+
+Whether filesort optimization was used and it needed temporary tables on disk.
+
+
+--
+
+*`mysql.slowlog.priority_queue`*::
++
+--
+type: keyword
+
+Whether a priority queue was used for filesort.
+
+
+--
+
+*`mysql.slowlog.full_scan`*::
++
+--
+type: keyword
+
+Whether a full table scan was needed for the slow query.
+
+
+--
+
+*`mysql.slowlog.full_join`*::
++
+--
+type: keyword
+
+Whether a full join was needed for the slow query (no indexes were used for joins).
+
+
+--
+
+*`mysql.slowlog.merge_passes`*::
++
+--
+type: long
+
+Number of merge passes executed for the query.
+
+
 --
 
 *`mysql.slowlog.user`*::

filebeat/docs/modules/mysql.asciidoc

@@ -16,7 +16,8 @@ include::../include/what-happens.asciidoc[]
 [float]
 === Compatibility
 
-The  +{modulename}+ module was tested with logs from versions 5.5 and 5.7.
+The  +{modulename}+ module was tested with logs from MySQL 5.5 and 5.7 and
+MariaDB 10.1 and 10.2.
 
 On Windows, the module was tested with MySQL installed from the Chocolatey
 repository.

filebeat/module/mysql/_meta/docs.asciidoc

@@ -11,7 +11,8 @@ include::../include/what-happens.asciidoc[]
 [float]
 === Compatibility
 
-The  +{modulename}+ module was tested with logs from versions 5.5 and 5.7.
+The  +{modulename}+ module was tested with logs from MySQL 5.5 and 5.7 and
+MariaDB 10.1 and 10.2.
 
 On Windows, the module was tested with MySQL installed from the Chocolatey
 repository.

filebeat/module/mysql/slowlog/_meta/fields.yml

@@ -20,6 +20,10 @@
       type: long
       description: >
         The number of rows scanned by the query.
+    - name: rows_affected
+      type: long
+      description: >
+        The number of rows modified by the query.
     - name: timestamp
       type: long
       description: >
@@ -31,6 +35,55 @@
       type: long
       description: >
         The connection ID for the query.
+    - name: thread_id
+      type: long
+      description: >
+        ID of the thread executing this query.
+    - name: schema
+      type: keyword
+      description: >
+        The schema where the slow query was executed.
+    - name: current_user
+      type: keyword
+      description: >
+        Current authenticated user, used to determine access privileges. Can differ from the value for user.
+
+    - name: query_cache_hit
+      type: keyword
+      description: >
+        Whether the query cache was hit.
+    - name: tmp_table
+      type: keyword
+      description: >
+        Whether a temporary table was used to resolve the query.
+    - name: tmp_table_on_disk
+      type: keyword
+      description: >
+        Whether the query needed temporary tables on disk.
+    - name: filesort
+      type: keyword
+      description: >
+        Whether filesort optimization was used.
+    - name: filesort_on_disk
+      type: keyword
+      description: >
+        Whether filesort optimization was used and it needed temporary tables on disk.
+    - name: priority_queue
+      type: keyword
+      description: >
+        Whether a priority queue was used for filesort.
+    - name: full_scan
+      type: keyword
+      description: >
+        Whether a full table scan was needed for the slow query.
+    - name: full_join
+      type: keyword
+      description: >
+        Whether a full join was needed for the slow query (no indexes were used for joins).
+    - name: merge_passes
+      type: long
+      description: >
+        Number of merge passes executed for the query.
 
     - name: user
       type: alias

filebeat/module/mysql/slowlog/ingest/pipeline.json

@@ -4,10 +4,11 @@
     "grok": {
       "field": "message",
       "patterns":[
-        "^# User@Host: %{USER:user.name}(\\[[^\\]]+\\])? @ (%{HOSTNAME:source.domain})? \\[(%{IP:source.ip})?\\](\\s*Id:\\s* %{NUMBER:mysql.slowlog.id:long})?\n# Query_time: %{NUMBER:mysql.slowlog.query_time.sec:float}\\s* Lock_time: %{NUMBER:mysql.slowlog.lock_time.sec:float}\\s* Rows_sent: %{NUMBER:mysql.slowlog.rows_sent:long}\\s* Rows_examined: %{NUMBER:mysql.slowlog.rows_examined:long}\n(SET timestamp=%{NUMBER:mysql.slowlog.timestamp:long};\n)?%{GREEDYMULTILINE:mysql.slowlog.query}"
+        "^# User@Host: %{USER:user.name}(\\[%{USER:mysql.slowlog.current_user}\\])? @ %{HOSTNAME:source.domain}? \\[%{IP:source.ip}?\\](\\s*Id:\\s* %{NUMBER:mysql.slowlog.id:long})?\n(# Thread_id:%{SPACE}%{NUMBER:mysql.slowlog.thread_id}%{METRICSPACE}Schema: %{WORD:mysql.slowlog.schema}?%{METRICSPACE}QC_hit: %{WORD:mysql.slowlog.query_cache_hit}\n)?%{METRICSPACE}Query_time: %{NUMBER:mysql.slowlog.query_time.sec:float}%{METRICSPACE}Lock_time: %{NUMBER:mysql.slowlog.lock_time.sec:float}%{METRICSPACE}Rows_sent: %{NUMBER:mysql.slowlog.rows_sent:long}%{METRICSPACE}Rows_examined: %{NUMBER:mysql.slowlog.rows_examined:long}%{METRICSPACE}(Rows_affected: %{NUMBER:mysql.slowlog.rows_affected:long}%{METRICSPACE})?(Full_scan: %{WORD:mysql.slowlog.full_scan}%{METRICSPACE})?(Full_join: %{WORD:mysql.slowlog.full_join}%{METRICSPACE})?(Tmp_table: %{WORD:mysql.slowlog.tmp_table}%{METRICSPACE})?(Tmp_table_on_disk: %{WORD:mysql.slowlog.tmp_table_on_disk}%{METRICSPACE})?(Filesort: %{WORD:mysql.slowlog.filesort}%{METRICSPACE})?(Filesort_on_disk: %{WORD:mysql.slowlog.filesort_on_disk}%{METRICSPACE})?(Merge_passes: %{NUMBER:mysql.slowlog.merge_passes}%{METRICSPACE})?(Priority_queue: %{WORD:mysql.slowlog.priority_queue}%{METRICSPACE})?(use %{WORD:mysql.slowlog.schema};\n)?SET timestamp=%{NUMBER:mysql.slowlog.timestamp:long};\n%{GREEDYMULTILINE:mysql.slowlog.query}"
         ],
       "pattern_definitions" : {
-        "GREEDYMULTILINE" : "(.|\n)*"
+        "GREEDYMULTILINE": "(.|\n)*",
+        "METRICSPACE": "([ #\n]*)"
       },
       "ignore_missing": true
     }

filebeat/module/mysql/slowlog/test/mariadb-10.1.21.log

@@ -0,0 +1,7 @@
+# Time: 180613 11:04:36
+# User@Host: root[root] @ localhost [121.0.0.1]
+# Thread_id: 5  Schema:   QC_hit: No
+# Query_time: 2.000652  Lock_time: 0.000000  Rows_sent: 1  Rows_examined: 0
+SET timestamp=1528898676;
+select sleep(2);
+

filebeat/module/mysql/slowlog/test/mariadb-10.1.21.log-expected.json

@@ -0,0 +1,28 @@
+[
+    {
+        "@timestamp": "2018-06-13T14:04:36.000Z",
+        "ecs.version": "1.0.0-beta2",
+        "event.dataset": "mysql.slowlog",
+        "event.duration": 2000652000,
+        "event.module": "mysql",
+        "fileset.name": "slowlog",
+        "input.type": "log",
+        "log.flags": [
+            "multiline"
+        ],
+        "log.offset": 24,
+        "mysql.slowlog.current_user": "root",
+        "mysql.slowlog.lock_time.sec": 0.0,
+        "mysql.slowlog.query": "select sleep(2);\n",
+        "mysql.slowlog.query_cache_hit": "No",
+        "mysql.slowlog.query_time.sec": 2.000652,
+        "mysql.slowlog.rows_examined": 0,
+        "mysql.slowlog.rows_sent": 1,
+        "mysql.slowlog.thread_id": "5",
+        "mysql.slowlog.timestamp": 1528898676,
+        "service.type": "mysql",
+        "source.domain": "localhost",
+        "source.ip": "121.0.0.1",
+        "user.name": "root"
+    }
+]

filebeat/module/mysql/slowlog/test/mariadb-10.2.12.log

@@ -0,0 +1,10 @@
+# Time: 181004 10:53:53
+# User@Host: root[root] @  [192.168.0.10]
+# Thread_id: 25844  Schema: blah  QC_hit: No
+# Query_time: 178.306017  Lock_time: 0.000000  Rows_sent: 0  Rows_examined: 53022772
+# Rows_affected: 3062
+# Full_scan: Yes  Full_join: No  Tmp_table: Yes  Tmp_table_on_disk: No
+# Filesort: Yes  Filesort_on_disk: No  Merge_passes: 0  Priority_queue: No
+SET timestamp=1538650433;
+call PROC('blah');
+

filebeat/module/mysql/slowlog/test/mariadb-10.2.12.log-expected.json

@@ -0,0 +1,37 @@
+[
+    {
+        "@timestamp": "2018-10-04T10:53:53.000Z",
+        "ecs.version": "1.0.0-beta2",
+        "event.dataset": "mysql.slowlog",
+        "event.duration": 178306016000,
+        "event.module": "mysql",
+        "fileset.name": "slowlog",
+        "input.type": "log",
+        "log.flags": [
+            "multiline"
+        ],
+        "log.offset": 24,
+        "mysql.slowlog.current_user": "root",
+        "mysql.slowlog.filesort": "Yes",
+        "mysql.slowlog.filesort_on_disk": "No",
+        "mysql.slowlog.full_join": "No",
+        "mysql.slowlog.full_scan": "Yes",
+        "mysql.slowlog.lock_time.sec": 0.0,
+        "mysql.slowlog.merge_passes": "0",
+        "mysql.slowlog.priority_queue": "No",
+        "mysql.slowlog.query": "call PROC('blah');\n",
+        "mysql.slowlog.query_cache_hit": "No",
+        "mysql.slowlog.query_time.sec": 178.30602,
+        "mysql.slowlog.rows_affected": 3062,
+        "mysql.slowlog.rows_examined": 53022772,
+        "mysql.slowlog.rows_sent": 0,
+        "mysql.slowlog.schema": "blah",
+        "mysql.slowlog.thread_id": "25844",
+        "mysql.slowlog.timestamp": 1538650433,
+        "mysql.slowlog.tmp_table": "Yes",
+        "mysql.slowlog.tmp_table_on_disk": "No",
+        "service.type": "mysql",
+        "source.ip": "192.168.0.10",
+        "user.name": "root"
+    }
+]

filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json

@@ -11,6 +11,7 @@
             "multiline"
         ],
         "log.offset": 41,
+        "mysql.slowlog.current_user": "root",
         "mysql.slowlog.id": 7234,
         "mysql.slowlog.lock_time.sec": 0.0,
         "mysql.slowlog.query": "select sleep(15);",
@@ -34,6 +35,7 @@
             "multiline"
         ],
         "log.offset": 254,
+        "mysql.slowlog.current_user": "debian-sys-maint",
         "mysql.slowlog.lock_time.sec": 6.1e-05,
         "mysql.slowlog.query": "SELECT count(*) FROM mysql.user WHERE user='root' and password='';",
         "mysql.slowlog.query_time.sec": 0.000153,
@@ -56,6 +58,7 @@
             "multiline"
         ],
         "log.offset": 526,
+        "mysql.slowlog.current_user": "appuser",
         "mysql.slowlog.id": 10997316,
         "mysql.slowlog.lock_time.sec": 0.000212,
         "mysql.slowlog.query": "SELECT mcu.mcu_guid, mcu.cus_guid, mcu.mcu_url, mcu.mcu_crawlelements, mcu.mcu_order, GROUP_CONCAT(mca.mca_guid SEPARATOR \";\") as mca_guid\n                    FROM kat_mailcustomerurl mcu, kat_customer cus, kat_mailcampaign mca\n                    WHERE cus.cus_guid = mcu.cus_guid\n                        AND cus.pro_code = 'CYB'\n                        AND cus.cus_offline = 0\n                        AND mca.cus_guid = cus.cus_guid\n                        AND (mcu.mcu_date IS NULL OR mcu.mcu_date < CURDATE())\n                        AND mcu.mcu_crawlelements IS NOT NULL\n                    GROUP BY mcu.mcu_guid\n                    ORDER BY mcu.mcu_order ASC\n                    LIMIT 1000;",
@@ -80,6 +83,7 @@
             "multiline"
         ],
         "log.offset": 1438,
+        "mysql.slowlog.current_user": "appuser",
         "mysql.slowlog.id": 10999834,
         "mysql.slowlog.lock_time.sec": 3.6e-05,
         "mysql.slowlog.query": "call load_stats(1, '2017-04-28 00:00:00');",