diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0909f2e7cdd..7bdff1ed2ab 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -308,6 +308,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add the `network.community_id` flow identifier to field to the IPTables, Suricata, and Zeek modules. {pull}11005[11005] - Add support for loading custom NetFlow and IPFIX field definitions to netflow input. {pull}10945[10945] - Added categorization fields for SSH login events in the system/auth fileset. {pull}11334[11334] +- Add support for MySQL 8.0 slow logs and tests also for Percona 8.0 and MariaDB 10.3. {pull}11417[11417] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index aec1f328044..72fbb9982b2 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -7273,7 +7273,19 @@ type: long format: bytes -The size of the query result. +The number of bytes sent to client. + + +-- + +*`mysql.slowlog.bytes_received`*:: ++ +-- +type: long + +format: bytes + +The number of bytes received from client. -- @@ -7454,6 +7466,46 @@ type: long Number of merge passes executed for the query. +-- + +*`mysql.slowlog.sort_merge_passes`*:: ++ +-- +type: long + +Number of merge passes that the sort algorithm has had to do. + + +-- + +*`mysql.slowlog.sort_range_count`*:: ++ +-- +type: long + +Number of sorts that were done using ranges. + + +-- + +*`mysql.slowlog.sort_rows`*:: ++ +-- +type: long + +Number of sorted rows. + + +-- + +*`mysql.slowlog.sort_scan_count`*:: ++ +-- +type: long + +Number of sorts that were done by scanning the table. + + -- *`mysql.slowlog.log_slow_rate_type`*:: @@ -7474,6 +7526,76 @@ type: keyword Slow log rate limit, a value of 100 means that one in a hundred queries or sessions are being logged. +-- + +*`mysql.slowlog.read_first`*:: ++ +-- +type: long + +The number of times the first entry in an index was read. + + +-- + +*`mysql.slowlog.read_last`*:: ++ +-- +type: long + +The number of times the last key in an index was read. + + +-- + +*`mysql.slowlog.read_key`*:: ++ +-- +type: long + +The number of requests to read a row based on a key. + + +-- + +*`mysql.slowlog.read_next`*:: ++ +-- +type: long + +The number of requests to read the next row in key order. + + +-- + +*`mysql.slowlog.read_prev`*:: ++ +-- +type: long + +The number of requests to read the previous row in key order. + + +-- + +*`mysql.slowlog.read_rnd`*:: ++ +-- +type: long + +The number of requests to read a row based on a fixed position. + + +-- + +*`mysql.slowlog.read_rnd_next`*:: ++ +-- +type: long + +The number of requests to read the next row in the data file. + + -- [float] diff --git a/filebeat/docs/modules/mysql.asciidoc b/filebeat/docs/modules/mysql.asciidoc index eeef90b2639..c04f8afa0b0 100644 --- a/filebeat/docs/modules/mysql.asciidoc +++ b/filebeat/docs/modules/mysql.asciidoc @@ -16,8 +16,8 @@ include::../include/what-happens.asciidoc[] [float] === Compatibility -The +{modulename}+ module was tested with logs from MySQL 5.5 and 5.7, -MariaDB 10.1 and 10.2, and Percona 5.7. +The +{modulename}+ module was tested with logs from MySQL 5.5, 5.7 and 8.0, +MariaDB 10.1, 10.2 and 10.3, and Percona 5.7 and 8.0. On Windows, the module was tested with MySQL installed from the Chocolatey repository. diff --git a/filebeat/module/mysql/_meta/docs.asciidoc b/filebeat/module/mysql/_meta/docs.asciidoc index 2737f441839..1ad7b8bd560 100644 --- a/filebeat/module/mysql/_meta/docs.asciidoc +++ b/filebeat/module/mysql/_meta/docs.asciidoc @@ -11,8 +11,8 @@ include::../include/what-happens.asciidoc[] [float] === Compatibility -The +{modulename}+ module was tested with logs from MySQL 5.5 and 5.7, -MariaDB 10.1 and 10.2, and Percona 5.7. +The +{modulename}+ module was tested with logs from MySQL 5.5, 5.7 and 8.0, +MariaDB 10.1, 10.2 and 10.3, and Percona 5.7 and 8.0. On Windows, the module was tested with MySQL installed from the Chocolatey repository. diff --git a/filebeat/module/mysql/error/test/mysql-ubuntu-8.0.15.log b/filebeat/module/mysql/error/test/mysql-ubuntu-8.0.15.log new file mode 100644 index 00000000000..3b05f21997b --- /dev/null +++ b/filebeat/module/mysql/error/test/mysql-ubuntu-8.0.15.log @@ -0,0 +1,12 @@ +2019-03-24T13:44:25.484123Z 0 [System] [MY-013169] [Server] /usr/sbin/mysqld (mysqld 8.0.15) initializing of server in progress as process 1640 +2019-03-24T13:44:27.924508Z 5 [Warning] [MY-010453] [Server] root@localhost is created with an empty password ! Please consider switching off the --initialize-insecure option. +2019-03-24T13:44:29.065309Z 0 [System] [MY-013170] [Server] /usr/sbin/mysqld (mysqld 8.0.15) initializing of server has completed +2019-03-24T13:44:31.085670Z 0 [System] [MY-010116] [Server] /usr/sbin/mysqld (mysqld 8.0.15) starting as process 1688 +2019-03-24T13:44:31.533096Z 0 [Warning] [MY-010068] [Server] CA certificate ca.pem is self signed. +2019-03-24T13:44:31.534587Z 0 [Warning] [MY-011810] [Server] Insecure configuration for --pid-file: Location '/tmp' in the path is accessible to all OS users. Consider choosing a different directory. +2019-03-24T13:44:31.555406Z 6 [System] [MY-013172] [Server] Received SHUTDOWN from user boot. Shutting down mysqld (Version: 8.0.15). +2019-03-24T13:44:33.236624Z 0 [System] [MY-010910] [Server] /usr/sbin/mysqld: Shutdown complete (mysqld 8.0.15) MySQL Community Server - GPL. +2019-03-24T13:44:34.072713Z 0 [System] [MY-010116] [Server] /usr/sbin/mysqld (mysqld 8.0.15) starting as process 1834 +2019-03-24T13:44:34.406962Z 0 [Warning] [MY-010068] [Server] CA certificate ca.pem is self signed. +2019-03-24T13:44:34.420123Z 0 [System] [MY-010931] [Server] /usr/sbin/mysqld: ready for connections. Version: '8.0.15' socket: '/var/run/mysqld/mysqld.sock' port: 3306 MySQL Community Server - GPL. +2019-03-24T13:44:34.572158Z 0 [System] [MY-011323] [Server] X Plugin ready for connections. Socket: '/var/run/mysqld/mysqlx.sock' bind-address: '::' port: 33060 diff --git a/filebeat/module/mysql/fields.go b/filebeat/module/mysql/fields.go index d81d7f79f7d..0e53ab01b33 100644 --- a/filebeat/module/mysql/fields.go +++ b/filebeat/module/mysql/fields.go @@ -32,5 +32,5 @@ func init() { // AssetMysql returns asset data. // This is the base64 encoded gzipped contents of module/mysql. func AssetMysql() string { - return "eJzEmM+P27gOx+/zVxB9l/eAqdN3eHhADgtsWyy2QLvAogX26DIWbWsji64oTyb96xeSnR+N5cSZTlofCnRi8fshLVIUX8KatktotvLF3AF47Q0t4cWH7cc/37+4A1AkhdOt12yX8MsdAMAHVp0hKNlBi060rcDXBHEJGK6g1IYkuwOQmp3PC7alrpbgXUd3AKUmo2QZTb0Eiw0d5MPjty0toXLctcNfEgzh+S0agtJxMwUQnmO9Y01fO0KVa7X/Zadt2FZHf5yQD8+nmqBga6kIvwK7wSq8exvjE7C+dOS22UienGM3kj52+4L2G7YetZXBwdM4RPshGpIdLTsNxqWAHMjQaJSTX1r09fDxsqnVja4c9vzDBhgrG3ogc6Wq4SpLrZuj15AIVnStn8lVU3o7LTG8MVzd8EsHhas+tOFinXvdUCZUJINQGkZ/8ssZQBhSARvurAcuIVg/7H7YoPak9ikRAMAzrAjwAbXBlaEsWBhZfUDTEWgBbUGoYKvkHlAAe8RQe1rW1oPtmhW5LOmv443kQvbUo2S2z3S11wuuBuvgyHfOkoLVNpX0Ixp6xEZbSqfasxBJgfYKICxLKvwtgRpWutSziFZbT9d9sZJdg35YeT2q6K8Ut+1+xzqSzvg0XnzjCSIhT894/RMKrxQ1NZiUXdN2w+7U6Bw3o03Y1OT6EnDwGzYoQI9UdJ5UOghF5xxZn3dC7vm43vRWATtfk/W6wFCOgsZ9+FeFYqTIkwtJCVgUJAKt0w/aUEWSwRu0oHRZkjtU3742hZoWDKXdMSg+J+csP58z71E8HI54IbJp8bU2ZiKjnxZFVvsscYTCFnT5TZWXQfPAM51AeYFFTXmt00m+YjaE9jrCv2ryNR31XRBFIlmtJ9LZN23uwxH0/BwInpqWHbotRIlIsttxjoTNA12qhnu+nG2utKxvGS9LpALdt9gCHPa/rC8Qntappx4ff+yPjhFI4QgPvYSWRDU+xgrQP5xtCNYJ46WvGw6h2ZBzD7wR/sfdUTcvsGnoeMFid4PM3VkGbr1u9Nd4gu2z5jzN7fLjPBWgVaD903KndZqd9tv8S0fdTUrQTgGiwoE6fOidYxOB7YzJQyN5C6xgfCiKQSJyDRHc3RMudUyR72/Wt+MLxs+Twb8tg7aKHklgE3qefXDDYvlPGr0hV1HeosgNClM0Dr3xfb81NZCAb26GVR5cyx16ygPIM/aG2zYWnt11FYIGGN1ofx+yJ+yBFcFnIRHN9vOutzi8NjKpBbBtTbhUtBQ6objyHtjB5+hlNKL98JbEt2a7n9L8Dv8/pvzGoYvkEv776hU0hFbA1+iBLYVLL0LdWeVIRW492iwAvHdcAB3BisKt2HBVTfZh/4La+1aWi8Vms8lacgVbzApuFoqLxfD/l0Lugdzif9n/F0pjZVm8LmQRA0SPnqwildW+MenbjLWsVsnwnY4+ZgTvdATiyKDXoXdieGctv30NZCttT/dragxyTOnd4/jmBBc+9Aze8HxyaAX7yeC7t5MImnOXczv+sGcqwUyCQ0VosYo9uwJuqb8WSnYeKdVZXIQ615bMhH4dlvasqnNxwnM9/Aa1T0y3LvLP4Ps9prGtQl3xzOu+k0cFaMzumFDosb8oimeHFU3jOiryOJD7EcjJiVxQPxPP2DX8HDrPQNYP95MhzfsmJnRc/QBxf7yxLcbzw/Bou7PcnfkOYZNJaB+9tsX4qPl+N39tW8ePuolNdrEblEbZYeIweNw7lDqqk4V2clIyPTiK84qw+gkjo5olfShOywl3rqBMcYOjVm2OpD49LGYKjtZNid39EwAA//+S3DeD" + return "eJzEmVuv3LgNx9/PpyC2Ly2QzNk+FAXOQ4FugqIBdgsUWaCPDseibXZk0ZHkueTTLyh7LvFlLsnMiR8C5HjE/4+USVHSW1jR7gXqXfhsnwAiR0sv8NNvu4///fWnJwBDIffcRBb3Av94AgD4TUxrCQrx0KAP7EqIFUEaAlZKKNhSWDwBhEp8zHJxBZcvEH1LTwAFkzXhJZl6Cw5rOsrrE3cNvUDppW36v0ww6POvZAgKL/UcgD6neqeasfKEJmNzeLPXtuLKkz/OyOvze0WQi3OU61sQ31uFD+9TfBTrc0t+txjJk/fiR9Knbl/QficuIrvQOziMQ7Kv0QiLk2HDYFwKyJEMLWMYvGkwVv3kLeZG11x67Pj7D2CsbGlN9kZVK+Viatw1ejWFgCXd6ufkqDm9vVawsrFSPnCmVeGmibaSr7LINS0C5ZNBKKxgHLw5Awh9KmAtrYsgBaj149cPG+RI5pASCgBRYEmAa2SLS0sLtTCyukbbEnAAdhAoF2fCG8AA2CFq7WmEXQTX1kvyi0l/vWxCFsgNPZrM9itd7fTUVbUOnmLrHRlY7qaSfkRDW6zZ0XSq3YUo5OhuAMKioDw+EqgWwwVfRbTcRbptxgrxNcZ+5PegJgOg0vqB5pbJxXOQnnLi9fVxuzPoXr6rCedwU7hvF0zF5cxU/YDVIuQV1Tgpu6LdRvzQ6DVuJpuwqch3devoN2wwAG0pbyOZ6SDkrffkYtYG8vfjetdZBWxjRS5yjlpDVeON/mv0AzUUyWslAcxzCgEaz2u2VFJYwDt0YLgoyB+XjK6gaiFWQ9PuWAwxI++d3M+ZXzFEOPYlgchNi6/Y2pl0+rYoiqG0IlUEnjCIAy6+WppCr3nkmU+gLMe8oqzi6cq0FLGE7jbC/1UUKzppFiGJJLKKZ9I51k0Wdd28PwdCpLoRj34HSSKR7L84T0Hsmi6V8ANfJi4zHFaPjJcjMkr3NXYA0e8/rC4QDuvUt655/znU5hFI7gmPDRCHiWp8iqXQr87WB2vAeGl2A3+5HvLaxW+E/5G/0A2BnYZOu0LxD8jcvWWQJnLNX9IKdsia8zSPy4/zVIDOAMdvy53Gs3iOu+xzS+1DStBeAZLCkVoneu/YTGBbazPtfh+Bpcb7oqgSiauP4H5zc6ljSnz/F34cnxo/TwZ/dgLsDG0pwEZ7nkNwdXD4yzR6Tb6krMEQHlCYknHojB/6rblTlFOqlEWvhxYrjF04NbfQlvqdVjVUulxj15XJGVKPrqQs153y3UFVoCdMs2rE6dTqNjnJhsV0b9OByeb+oVPLZNIO8ExMNJdeMyTLXbc/3p9dpoyeaYalzDRxMo+RMuW5485j16RlbX+CA6oBlmuOb7Q2a4VZEnwKFAKL+7TvXI8/G5nkANg0VvfZDWmfnUa+AfHwKeVQMsKx/1VIvzqTXF+7P6X5Hf5/nPIb+z2KFPDXn3+GmtD186czxw4QqtYZTyZx8yjfAeTgeAD0BEvSibZSlnOrcdqNFuzDY86IItepchAkDSAX/S754roynMq1QpzB023Zg+lUQqfyVrQVDY8W7nRwRJ9bCpq/koQAtZLAEkPXsKLCnuFytH3Qod8QTKOnaomPXQqieDN7JKl0jaf169GpGksbbiH07kEHghfnteCtljAJrBZnlq094w+cZ/2/wYipKZ05QfgTVDE24eX5ebPZLBryuThc5FI/G8mf+/+/DeTX5J//tvj7s2EsnYTIeXhOxZe2kZwhs6hibafP4ZwTs5yMwfCm4YogDG8cPFmMrLt+gQ/OyftfgFzJbrgWTt06nFJGvx2f+cGFReQKXn1+9+gCdhdxH97PIrBkPpNmvGic+WSuJDg2HQ2W1H0v0lB3oDlof0ZIU3vii1DnNtRXQv/SnyOjAdP6dKFyO/wGOU5cJl3kv4Lv36lFcKX2LFFkdSwZ1u43OF0CeqkhRPFY0jyupzxL91+vgTx5AabqZ+KZ9rs/hi6KNib9yVqf5t32G53p7+sOGzNx+fi6Tp++KKaB827qRxYywyGyy8dt7Pe7+c+m8bLlOh0P5ft7ySTbn5X3HncOTW0yJwvt7Bn//JVHOmnX0YO311x2VDLT9M3LBWl9TgsjNY4OGa6R5OFicaXgaNyc2NMfAQAA//9cPDl5" } diff --git a/filebeat/module/mysql/slowlog/_meta/fields.yml b/filebeat/module/mysql/slowlog/_meta/fields.yml index 8f3bb259018..fdd2d9f20e0 100644 --- a/filebeat/module/mysql/slowlog/_meta/fields.yml +++ b/filebeat/module/mysql/slowlog/_meta/fields.yml @@ -24,7 +24,12 @@ type: long format: bytes description: > - The size of the query result. + The number of bytes sent to client. + - name: bytes_received + type: long + format: bytes + description: > + The number of bytes received from client. - name: query description: > The slow query. @@ -98,6 +103,22 @@ type: long description: > Number of merge passes executed for the query. + - name: sort_merge_passes + type: long + description: > + Number of merge passes that the sort algorithm has had to do. + - name: sort_range_count + type: long + description: > + Number of sorts that were done using ranges. + - name: sort_rows + type: long + description: > + Number of sorted rows. + - name: sort_scan_count + type: long + description: > + Number of sorts that were done by scanning the table. - name: log_slow_rate_type type: keyword description: > @@ -108,6 +129,34 @@ description: > Slow log rate limit, a value of 100 means that one in a hundred queries or sessions are being logged. + - name: read_first + type: long + description: > + The number of times the first entry in an index was read. + - name: read_last + type: long + description: > + The number of times the last key in an index was read. + - name: read_key + type: long + description: > + The number of requests to read a row based on a key. + - name: read_next + type: long + description: > + The number of requests to read the next row in key order. + - name: read_prev + type: long + description: > + The number of requests to read the previous row in key order. + - name: read_rnd + type: long + description: > + The number of requests to read a row based on a fixed position. + - name: read_rnd_next + type: long + description: > + The number of requests to read the next row in the data file. # https://www.percona.com/doc/percona-server/5.7/diagnostics/slow_extended.html - name: innodb diff --git a/filebeat/module/mysql/slowlog/ingest/pipeline.json b/filebeat/module/mysql/slowlog/ingest/pipeline.json index 4849863bf05..9a3076f874f 100644 --- a/filebeat/module/mysql/slowlog/ingest/pipeline.json +++ b/filebeat/module/mysql/slowlog/ingest/pipeline.json @@ -4,7 +4,7 @@ "grok": { "field": "message", "patterns":[ - "^# User@Host: %{USER:user.name}(\\[%{USER:mysql.slowlog.current_user}\\])? @ %{HOSTNAME:source.domain}? \\[%{IP:source.ip}?\\]%{METRICSPACE}(Id:%{SPACE}%{NUMBER:mysql.thread_id:long}%{METRICSPACE})?(Thread_id:%{SPACE}%{NUMBER:mysql.thread_id}%{METRICSPACE})?(Schema:%{SPACE}%{WORD:mysql.slowlog.schema}?%{METRICSPACE})?(Last_errno: %{NUMBER:mysql.slowlog.last_errno:long}%{METRICSPACE})?(Killed: %{NUMBER:mysql.slowlog.killed:long}%{METRICSPACE})?(QC_hit: %{WORD:mysql.slowlog.query_cache_hit}%{METRICSPACE})?(Query_time: %{NUMBER:temp.duration:float}%{METRICSPACE})?(Lock_time: %{NUMBER:mysql.slowlog.lock_time.sec:float}%{METRICSPACE})?(Rows_sent: %{NUMBER:mysql.slowlog.rows_sent:long}%{METRICSPACE})?(Rows_examined: %{NUMBER:mysql.slowlog.rows_examined:long}%{METRICSPACE})?(Rows_affected: %{NUMBER:mysql.slowlog.rows_affected:long}%{METRICSPACE})?(Bytes_sent: %{NUMBER:mysql.slowlog.bytes_sent:long}%{METRICSPACE})?(Tmp_tables: %{NUMBER:mysql.slowlog.tmp_tables:long}%{METRICSPACE})?(Tmp_disk_tables: %{NUMBER:mysql.slowlog.tmp_disk_tables}%{METRICSPACE})?(Tmp_table_sizes: %{NUMBER:mysql.slowlog.tmp_table_sizes:long}%{METRICSPACE})?(InnoDB_trx_id: %{WORD:mysql.slowlog.innodb.trx_id}%{METRICSPACE})?(QC_Hit: %{WORD:mysql.slowlog.query_cache_hit}%{METRICSPACE})?(Full_scan: %{WORD:mysql.slowlog.full_scan}%{METRICSPACE})?(Full_join: %{WORD:mysql.slowlog.full_join}%{METRICSPACE})?(Tmp_table: %{WORD:mysql.slowlog.tmp_table}%{METRICSPACE})?(Tmp_table_on_disk: %{WORD:mysql.slowlog.tmp_table_on_disk}%{METRICSPACE})?(Filesort: %{WORD:mysql.slowlog.filesort}%{METRICSPACE})?(Filesort_on_disk: %{WORD:mysql.slowlog.filesort_on_disk}%{METRICSPACE})?(Merge_passes: %{NUMBER:mysql.slowlog.merge_passes:long}%{METRICSPACE})?(Priority_queue: %{WORD:mysql.slowlog.priority_queue}%{METRICSPACE})?(No InnoDB statistics available for this query%{METRICSPACE})?(InnoDB_IO_r_ops: %{NUMBER:mysql.slowlog.innodb.io_r_ops:long}%{METRICSPACE})?(InnoDB_IO_r_bytes: %{NUMBER:mysql.slowlog.innodb.io_r_bytes:long}%{METRICSPACE})?(InnoDB_IO_r_wait: %{NUMBER:mysql.slowlog.innodb.io_r_wait.sec:float}%{METRICSPACE})?(InnoDB_rec_lock_wait: %{NUMBER:mysql.slowlog.innodb.rec_lock_wait.sec:float}%{METRICSPACE})?(InnoDB_queue_wait: %{NUMBER:mysql.slowlog.innodb.queue_wait.sec:float}%{METRICSPACE})?(InnoDB_pages_distinct: %{NUMBER:mysql.slowlog.innodb.pages_distinct:long}%{METRICSPACE})?(Log_slow_rate_type: %{WORD:mysql.slowlog.log_slow_rate_type}%{METRICSPACE})?(Log_slow_rate_limit: %{NUMBER:mysql.slowlog.log_slow_rate_limit:long}%{METRICSPACE})?%{EXPLAIN}?(use %{WORD:mysql.slowlog.schema};\n)?SET timestamp=%{NUMBER:mysql.slowlog.timestamp:long};\n%{GREEDYMULTILINE:mysql.slowlog.query}" + "^# User@Host: %{USER:user.name}(\\[%{USER:mysql.slowlog.current_user}\\])? @ %{HOSTNAME:source.domain}? \\[%{IP:source.ip}?\\]%{METRICSPACE}(Id:%{SPACE}%{NUMBER:mysql.thread_id:long}%{METRICSPACE})?(Thread_id:%{SPACE}%{NUMBER:mysql.thread_id}%{METRICSPACE})?(Schema:%{SPACE}%{WORD:mysql.slowlog.schema}?%{METRICSPACE})?(Last_errno: %{NUMBER:mysql.slowlog.last_errno:long}%{METRICSPACE})?(Killed: %{NUMBER:mysql.slowlog.killed:long}%{METRICSPACE})?(QC_hit: %{WORD:mysql.slowlog.query_cache_hit}%{METRICSPACE})?(Query_time: %{NUMBER:temp.duration:float}%{METRICSPACE})?(Lock_time: %{NUMBER:mysql.slowlog.lock_time.sec:float}%{METRICSPACE})?(Rows_sent: %{NUMBER:mysql.slowlog.rows_sent:long}%{METRICSPACE})?(Rows_examined: %{NUMBER:mysql.slowlog.rows_examined:long}%{METRICSPACE})?(Rows_affected: %{NUMBER:mysql.slowlog.rows_affected:long}%{METRICSPACE})?(Thread_id: %{NUMBER:mysql.thread_id}%{METRICSPACE})?(Errno: %{NUMBER:mysql.slowlog.last_errno:long}%{METRICSPACE})?(Killed: %{NUMBER:mysql.slowlog.killed:long}%{METRICSPACE})?(Bytes_received: %{NUMBER:mysql.slowlog.bytes_received:long}%{METRICSPACE})?(Bytes_sent: %{NUMBER:mysql.slowlog.bytes_sent:long}%{METRICSPACE})?(Read_first: %{NUMBER:mysql.slowlog.read_first:long}%{METRICSPACE})?(Read_last: %{NUMBER:mysql.slowlog.read_last:long}%{METRICSPACE})?(Read_key: %{NUMBER:mysql.slowlog.read_key:long}%{METRICSPACE})?(Read_next: %{NUMBER:mysql.slowlog.read_next:long}%{METRICSPACE})?(Read_prev: %{NUMBER:mysql.slowlog.read_prev:long}%{METRICSPACE})?(Read_rnd: %{NUMBER:mysql.slowlog.read_rnd:long}%{METRICSPACE})?(Read_rnd_next: %{NUMBER:mysql.slowlog.read_rnd_next:long}%{METRICSPACE})?(Sort_merge_passes: %{NUMBER:mysql.slowlog.sort_merge_passes:long}%{METRICSPACE})?(Sort_range_count: %{NUMBER:mysql.slowlog.sort_range_count:long}%{METRICSPACE})?(Sort_rows: %{NUMBER:mysql.slowlog.sort_rows:long}%{METRICSPACE})?(Sort_scan_count: %{NUMBER:mysql.slowlog.sort_scan_count:long}%{METRICSPACE})?(Created_tmp_disk_tables: %{NUMBER:mysql.slowlog.tmp_disk_tables:long}%{METRICSPACE})?(Created_tmp_tables: %{NUMBER:mysql.slowlog.tmp_tables:long}%{METRICSPACE})?(Tmp_tables: %{NUMBER:mysql.slowlog.tmp_tables:long}%{METRICSPACE})?(Tmp_disk_tables: %{NUMBER:mysql.slowlog.tmp_disk_tables}%{METRICSPACE})?(Tmp_table_sizes: %{NUMBER:mysql.slowlog.tmp_table_sizes:long}%{METRICSPACE})?(Start: %{TIMESTAMP_ISO8601:event.start}%{METRICSPACE})?(End: %{TIMESTAMP_ISO8601:event.end}%{METRICSPACE})?(InnoDB_trx_id: %{WORD:mysql.slowlog.innodb.trx_id}%{METRICSPACE})?(QC_Hit: %{WORD:mysql.slowlog.query_cache_hit}%{METRICSPACE})?(Full_scan: %{WORD:mysql.slowlog.full_scan}%{METRICSPACE})?(Full_join: %{WORD:mysql.slowlog.full_join}%{METRICSPACE})?(Tmp_table: %{WORD:mysql.slowlog.tmp_table}%{METRICSPACE})?(Tmp_table_on_disk: %{WORD:mysql.slowlog.tmp_table_on_disk}%{METRICSPACE})?(Filesort: %{WORD:mysql.slowlog.filesort}%{METRICSPACE})?(Filesort_on_disk: %{WORD:mysql.slowlog.filesort_on_disk}%{METRICSPACE})?(Merge_passes: %{NUMBER:mysql.slowlog.merge_passes:long}%{METRICSPACE})?(Priority_queue: %{WORD:mysql.slowlog.priority_queue}%{METRICSPACE})?(No InnoDB statistics available for this query%{METRICSPACE})?(InnoDB_IO_r_ops: %{NUMBER:mysql.slowlog.innodb.io_r_ops:long}%{METRICSPACE})?(InnoDB_IO_r_bytes: %{NUMBER:mysql.slowlog.innodb.io_r_bytes:long}%{METRICSPACE})?(InnoDB_IO_r_wait: %{NUMBER:mysql.slowlog.innodb.io_r_wait.sec:float}%{METRICSPACE})?(InnoDB_rec_lock_wait: %{NUMBER:mysql.slowlog.innodb.rec_lock_wait.sec:float}%{METRICSPACE})?(InnoDB_queue_wait: %{NUMBER:mysql.slowlog.innodb.queue_wait.sec:float}%{METRICSPACE})?(InnoDB_pages_distinct: %{NUMBER:mysql.slowlog.innodb.pages_distinct:long}%{METRICSPACE})?(Log_slow_rate_type: %{WORD:mysql.slowlog.log_slow_rate_type}%{METRICSPACE})?(Log_slow_rate_limit: %{NUMBER:mysql.slowlog.log_slow_rate_limit:long}%{METRICSPACE})?%{EXPLAIN}?(use %{WORD:mysql.slowlog.schema};\n)?SET timestamp=%{NUMBER:mysql.slowlog.timestamp:long};\n%{GREEDYMULTILINE:mysql.slowlog.query}" ], "pattern_definitions" : { "GREEDYMULTILINE": "(.|\n)*", diff --git a/filebeat/module/mysql/slowlog/test/mariadb-10.3.13.log b/filebeat/module/mysql/slowlog/test/mariadb-10.3.13.log new file mode 100644 index 00000000000..b7d8ff94aa4 --- /dev/null +++ b/filebeat/module/mysql/slowlog/test/mariadb-10.3.13.log @@ -0,0 +1,18 @@ +/usr/sbin/mysqld, Version: 10.3.13-MariaDB-1:10.3.13+maria~bionic-log (mariadb.org binary distribution). started with: +Tcp port: 3306 Unix socket: /var/run/mysqld/mysqld.sock +Time Id Command Argument +# Time: 190324 16:03:00 +# User@Host: root[root] @ localhost [] +# Thread_id: 37 Schema: employees QC_hit: No +# Query_time: 2.461578 Lock_time: 0.000196 Rows_sent: 10 Rows_examined: 3145718 +# Rows_affected: 0 Bytes_sent: 319 +# Tmp_tables: 1 Tmp_disk_tables: 0 Tmp_table_sizes: 4026528 +# Full_scan: Yes Full_join: No Tmp_table: Yes Tmp_table_on_disk: No +# Filesort: Yes Filesort_on_disk: No Merge_passes: 0 Priority_queue: Yes +use employees; +SET timestamp=1553443380; +SELECT last_name, MAX(salary) AS salary FROM employees + INNER JOIN salaries ON employees.emp_no = salaries.emp_no + GROUP BY last_name + ORDER BY salary DESC + LIMIT 10; diff --git a/filebeat/module/mysql/slowlog/test/mysql-ubuntu-8.0.15.log b/filebeat/module/mysql/slowlog/test/mysql-ubuntu-8.0.15.log new file mode 100644 index 00000000000..2e70932a424 --- /dev/null +++ b/filebeat/module/mysql/slowlog/test/mysql-ubuntu-8.0.15.log @@ -0,0 +1,19 @@ +/usr/sbin/mysqld, Version: 8.0.15 (MySQL Community Server - GPL). started with: +Tcp port: 3306 Unix socket: /var/run/mysqld/mysqld.sock +Time Id Command Argument +# Time: 2019-03-24T14:01:47.811234Z +# User@Host: root[root] @ localhost [] Id: 14 +# Query_time: 2.475469 Lock_time: 0.000287 Rows_sent: 10 Rows_examined: 3145718 +use employees; +SET timestamp=1553436105; +SELECT last_name, MAX(salary) AS salary FROM employees INNER JOIN salaries ON employees.emp_no = salaries.emp_no GROUP BY last_name ORDER BY salary DESC LIMIT 10; +# Time: 2019-03-24T14:04:53.713951Z +# User@Host: root[root] @ localhost [] Id: 16 +# Query_time: 2.631844 Lock_time: 0.000145 Rows_sent: 10 Rows_examined: 3145718 Thread_id: 16 Errno: 0 Killed: 0 Bytes_received: 0 Bytes_sent: 312 Read_first: 1 Read_last: 0 Read_key: 3144072 Read_next: 2844047 Read_prev: 0 Read_rnd: 10 Read_rnd_next: 301663 Sort_merge_passes: 0 Sort_range_count: 0 Sort_rows: 10 Sort_scan_count: 1 Created_tmp_disk_tables: 0 Created_tmp_tables: 1 Start: 2019-03-24T14:04:51.082107Z End: 2019-03-24T14:04:53.713951Z +use employees; +SET timestamp=1553436291; +SELECT last_name, MAX(salary) AS salary FROM employees + INNER JOIN salaries ON employees.emp_no = salaries.emp_no + GROUP BY last_name + ORDER BY salary DESC + LIMIT 10; diff --git a/filebeat/module/mysql/slowlog/test/mysql-ubuntu-8.0.15.log-expected.json b/filebeat/module/mysql/slowlog/test/mysql-ubuntu-8.0.15.log-expected.json new file mode 100644 index 00000000000..7e81a656595 --- /dev/null +++ b/filebeat/module/mysql/slowlog/test/mysql-ubuntu-8.0.15.log-expected.json @@ -0,0 +1,67 @@ +[ + { + "@timestamp": "2019-03-24T14:01:45.000Z", + "ecs.version": "1.0.0", + "event.dataset": "mysql.slowlog", + "event.duration": 2475469000, + "event.module": "mysql", + "fileset.name": "slowlog", + "input.type": "log", + "log.flags": [ + "multiline" + ], + "log.offset": 217, + "mysql.slowlog.current_user": "root", + "mysql.slowlog.lock_time.sec": 0.000287, + "mysql.slowlog.query": "SELECT last_name, MAX(salary) AS salary FROM employees INNER JOIN salaries ON employees.emp_no = salaries.emp_no GROUP BY last_name ORDER BY salary DESC LIMIT 10;", + "mysql.slowlog.rows_examined": 3145718, + "mysql.slowlog.rows_sent": 10, + "mysql.slowlog.schema": "employees", + "mysql.thread_id": 14, + "service.type": "mysql", + "source.domain": "localhost", + "user.name": "root" + }, + { + "@timestamp": "2019-03-24T14:04:51.000Z", + "ecs.version": "1.0.0", + "event.dataset": "mysql.slowlog", + "event.duration": 2631844000, + "event.end": "2019-03-24T14:04:53.713951Z", + "event.module": "mysql", + "event.start": "2019-03-24T14:04:51.082107Z", + "fileset.name": "slowlog", + "input.type": "log", + "log.flags": [ + "multiline" + ], + "log.offset": 589, + "mysql.slowlog.bytes_received": 0, + "mysql.slowlog.bytes_sent": 312, + "mysql.slowlog.current_user": "root", + "mysql.slowlog.killed": 0, + "mysql.slowlog.last_errno": 0, + "mysql.slowlog.lock_time.sec": 0.000145, + "mysql.slowlog.query": "SELECT last_name, MAX(salary) AS salary FROM employees\n INNER JOIN salaries ON employees.emp_no = salaries.emp_no\n GROUP BY last_name\n ORDER BY salary DESC\n LIMIT 10;", + "mysql.slowlog.read_first": 1, + "mysql.slowlog.read_key": 3144072, + "mysql.slowlog.read_last": 0, + "mysql.slowlog.read_next": 2844047, + "mysql.slowlog.read_prev": 0, + "mysql.slowlog.read_rnd": 10, + "mysql.slowlog.read_rnd_next": 301663, + "mysql.slowlog.rows_examined": 3145718, + "mysql.slowlog.rows_sent": 10, + "mysql.slowlog.schema": "employees", + "mysql.slowlog.sort_merge_passes": 0, + "mysql.slowlog.sort_range_count": 0, + "mysql.slowlog.sort_rows": 10, + "mysql.slowlog.sort_scan_count": 1, + "mysql.slowlog.tmp_disk_tables": 0, + "mysql.slowlog.tmp_tables": 1, + "mysql.thread_id": "16", + "service.type": "mysql", + "source.domain": "localhost", + "user.name": "root" + } +] \ No newline at end of file diff --git a/filebeat/module/mysql/slowlog/test/percona-ubuntu-8.0.15.log b/filebeat/module/mysql/slowlog/test/percona-ubuntu-8.0.15.log new file mode 100644 index 00000000000..05fc70e1b7f --- /dev/null +++ b/filebeat/module/mysql/slowlog/test/percona-ubuntu-8.0.15.log @@ -0,0 +1,32 @@ +/usr/sbin/mysqld, Version: 8.0.15-5 (Percona Server (GPL), Release '5', Revision 'f8a9e99'). started with: +Tcp port: 3306 Unix socket: /var/run/mysqld/mysqld.sock +Time Id Command Argument +# Time: 2019-03-24T16:22:43.836524Z +# User@Host: root[root] @ localhost [] Id: 182 +# Schema: employees Last_errno: 0 Killed: 0 +# Query_time: 2.746607 Lock_time: 0.000138 Rows_sent: 10 Rows_examined: 3145718 Rows_affected: 0 +# Bytes_sent: 312 +use employees; +SET timestamp=1553444561; +SELECT last_name, MAX(salary) AS salary FROM employees INNER JOIN salaries ON employees.emp_no = salaries.emp_no GROUP BY last_name ORDER BY salary DESC LIMIT 10; +/usr/sbin/mysqld, Version: 8.0.15-5 (Percona Server (GPL), Release '5', Revision 'f8a9e99'). started with: +Tcp port: 0 Unix socket: /var/run/mysqld/mysqld.sock +Time Id Command Argument +# Time: 2019-03-24T16:26:04.482151Z +# User@Host: root[root] @ localhost [] Id: 8 +# Schema: employees Last_errno: 0 Killed: 0 +# Query_time: 3.133066 Lock_time: 0.000190 Rows_sent: 10 Rows_examined: 3145718 Rows_affected: 0 +# Bytes_sent: 312 Tmp_tables: 1 Tmp_disk_tables: 0 Tmp_table_sizes: 0 +# InnoDB_trx_id: 0 +# Full_scan: Yes Full_join: No Tmp_table: Yes Tmp_table_on_disk: No +# Filesort: Yes Filesort_on_disk: No Merge_passes: 0 +# InnoDB_IO_r_ops: 5491 InnoDB_IO_r_bytes: 89964544 InnoDB_IO_r_wait: 0.003183 +# InnoDB_rec_lock_wait: 0.000000 InnoDB_queue_wait: 0.000000 +# InnoDB_pages_distinct: 6122 +use employees; +SET timestamp=1553444761; +SELECT last_name, MAX(salary) AS salary FROM employees + INNER JOIN salaries ON employees.emp_no = salaries.emp_no + GROUP BY last_name + ORDER BY salary DESC + LIMIT 10; diff --git a/filebeat/module/mysql/slowlog/test/percona-ubuntu-8.0.15.log-expected.json b/filebeat/module/mysql/slowlog/test/percona-ubuntu-8.0.15.log-expected.json new file mode 100644 index 00000000000..555739ca182 --- /dev/null +++ b/filebeat/module/mysql/slowlog/test/percona-ubuntu-8.0.15.log-expected.json @@ -0,0 +1,73 @@ +[ + { + "@timestamp": "2019-03-24T16:22:41.000Z", + "ecs.version": "1.0.0", + "event.dataset": "mysql.slowlog", + "event.duration": 2746607000, + "event.module": "mysql", + "fileset.name": "slowlog", + "input.type": "log", + "log.flags": [ + "multiline" + ], + "log.offset": 244, + "mysql.slowlog.bytes_sent": 312, + "mysql.slowlog.current_user": "root", + "mysql.slowlog.killed": 0, + "mysql.slowlog.last_errno": 0, + "mysql.slowlog.lock_time.sec": 0.000138, + "mysql.slowlog.query": "SELECT last_name, MAX(salary) AS salary FROM employees INNER JOIN salaries ON employees.emp_no = salaries.emp_no GROUP BY last_name ORDER BY salary DESC LIMIT 10;\n/usr/sbin/mysqld, Version: 8.0.15-5 (Percona Server (GPL), Release '5', Revision 'f8a9e99'). started with:\nTcp port: 0 Unix socket: /var/run/mysqld/mysqld.sock\nTime Id Command Argument", + "mysql.slowlog.rows_affected": 0, + "mysql.slowlog.rows_examined": 3145718, + "mysql.slowlog.rows_sent": 10, + "mysql.slowlog.schema": "employees", + "mysql.thread_id": 182, + "service.type": "mysql", + "source.domain": "localhost", + "user.name": "root" + }, + { + "@timestamp": "2019-03-24T16:26:01.000Z", + "ecs.version": "1.0.0", + "event.dataset": "mysql.slowlog", + "event.duration": 3133066000, + "event.module": "mysql", + "fileset.name": "slowlog", + "input.type": "log", + "log.flags": [ + "multiline" + ], + "log.offset": 920, + "mysql.slowlog.bytes_sent": 312, + "mysql.slowlog.current_user": "root", + "mysql.slowlog.filesort": true, + "mysql.slowlog.filesort_on_disk": false, + "mysql.slowlog.full_join": false, + "mysql.slowlog.full_scan": true, + "mysql.slowlog.innodb.io_r_bytes": 89964544, + "mysql.slowlog.innodb.io_r_ops": 5491, + "mysql.slowlog.innodb.io_r_wait.sec": 0.003183, + "mysql.slowlog.innodb.pages_distinct": 6122, + "mysql.slowlog.innodb.queue_wait.sec": 0.0, + "mysql.slowlog.innodb.rec_lock_wait.sec": 0.0, + "mysql.slowlog.innodb.trx_id": "0", + "mysql.slowlog.killed": 0, + "mysql.slowlog.last_errno": 0, + "mysql.slowlog.lock_time.sec": 0.00019, + "mysql.slowlog.merge_passes": 0, + "mysql.slowlog.query": "SELECT last_name, MAX(salary) AS salary FROM employees\n INNER JOIN salaries ON employees.emp_no = salaries.emp_no\n GROUP BY last_name\n ORDER BY salary DESC\n LIMIT 10;", + "mysql.slowlog.rows_affected": 0, + "mysql.slowlog.rows_examined": 3145718, + "mysql.slowlog.rows_sent": 10, + "mysql.slowlog.schema": "employees", + "mysql.slowlog.tmp_disk_tables": "0", + "mysql.slowlog.tmp_table": true, + "mysql.slowlog.tmp_table_on_disk": false, + "mysql.slowlog.tmp_table_sizes": 0, + "mysql.slowlog.tmp_tables": 1, + "mysql.thread_id": 8, + "service.type": "mysql", + "source.domain": "localhost", + "user.name": "root" + } +] \ No newline at end of file