Support multiline logs in logstash/log fileset of Filebeat (#8562)

Multiline JDBC plugin logs were not parsed correctly. From now on the module is capable of aggregating log lines into a single multiline event and its pipeline can parse it correctly.
elastic · Oct 5, 2018 · 7271c55 · 7271c55
1 parent c67b88a
commit 7271c55
Show file tree

Hide file tree

Showing 5 changed files with 26 additions and 2 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -63,6 +63,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff]
 - Update CRI format to support partial/full tags. {pull}8265[8265]
 - Fix some errors happening when stopping syslog input. {pull}8347[8347]
 - Fix RFC3339 timezone and nanoseconds parsing with the syslog input. {pull}8346[8346]
+- Support multiline logs in logstash/log fileset of Filebeat. {pull}8562[8562]
 
 *Heartbeat*
 

diff --git a/filebeat/module/logstash/log/config/log.yml b/filebeat/module/logstash/log/config/log.yml
@@ -4,3 +4,7 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
+multiline:
+  pattern: ^\[[0-9]{4}-[0-9]{2}-[0-9]{2}
+  negate: true
+  match: after
diff --git a/filebeat/module/logstash/log/ingest/pipeline-plain.json b/filebeat/module/logstash/log/ingest/pipeline-plain.json
@@ -14,10 +14,11 @@
                 "field": "message",
                 "pattern_definitions": {
                     "LOGSTASH_CLASS_MODULE": "[\\w\\.]+\\s*",
-                    "LOGSTASH_LOGLEVEL": "INFO|ERROR|DEBUG|FATAL|WARN|TRACE"
+                    "LOGSTASH_LOGLEVEL": "INFO|ERROR|DEBUG|FATAL|WARN|TRACE",
+                    "GREEDYMULTILINE" : "(.|\n)*"
                 },
                 "patterns": [
-                    "\\[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\\]\\[%{LOGSTASH_LOGLEVEL:logstash.log.level}\\s?\\]\\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\\] %{GREEDYDATA:logstash.log.message}"
+                    "\\[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\\]\\[%{LOGSTASH_LOGLEVEL:logstash.log.level}\\s?\\]\\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\\] %{GREEDYMULTILINE:logstash.log.message}"
                 ]
             }
         },

diff --git a/filebeat/module/logstash/log/test/logstash-plain.log b/filebeat/module/logstash/log/test/logstash-plain.log
@@ -1 +1,5 @@
 [2017-10-23T14:20:12,046][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"}
+[2017-11-20T03:55:00,318][INFO ][logstash.inputs.jdbc     ] (0.058950s) Select Name as [person.name]
+, Address as [person.address]
+from people
+
diff --git a/filebeat/module/logstash/log/test/logstash-plain.log-expected.json b/filebeat/module/logstash/log/test/logstash-plain.log-expected.json
@@ -9,5 +9,19 @@
         "logstash.log.module": "logstash.modules.scaffold", 
         "offset": 0, 
         "prospector.type": "log"
+    }, 
+    {
+        "@timestamp": "2017-11-20T03:55:00,318", 
+        "fileset.module": "logstash", 
+        "fileset.name": "log", 
+        "input.type": "log", 
+        "log.flags": [
+            "multiline"
+        ], 
+        "logstash.log.level": "INFO", 
+        "logstash.log.message": "(0.058950s) Select Name as [person.name]\n, Address as [person.address]\nfrom people\n", 
+        "logstash.log.module": "logstash.inputs.jdbc     ", 
+        "offset": 175, 
+        "prospector.type": "log"
     }
 ]