Allow to test Logstash logs with json formatted (#9959)

elastic · Jan 9, 2019 · d6d85b7 · d6d85b7
1 parent 370ecd4
commit d6d85b7
Show file tree

Hide file tree

Showing 5 changed files with 61 additions and 0 deletions.
diff --git a/filebeat/module/logstash/log/test/logstash-json.log b/filebeat/module/logstash/log/test/logstash-json.log
@@ -0,0 +1,3 @@
+{"level":"INFO","loggerName":"logstash.agent","timeMillis":1546896321871,"thread":"Ruby-0-Thread-1: /Users/mat/work/elastic/releases/6.5.1/logstash/lib/bootstrap/environment.rb:6","logEvent":{"message":"Pipelines running","count":1,"running_pipelines":[{"metaClass":{"metaClass":{"metaClass":{"running_pipelines":"[:main]","non_running_pipelines":[]}}}}]}}
+{"level":"INFO","loggerName":"logstash.pipeline","timeMillis":1546896322538,"thread":"[main]>worker7","logEvent":{"message":"Pipeline has terminated","pipeline_id":"main","thread":"#<Thread:0x7d16ffef run>"}}
+{"level":"INFO","loggerName":"logstash.agent","timeMillis":1546896322594,"thread":"Api Webserver","logEvent":{"message":"Successfully started Logstash API endpoint","port":9600}}
diff --git a/filebeat/module/logstash/log/test/logstash-json.log-expected.json b/filebeat/module/logstash/log/test/logstash-json.log-expected.json
@@ -0,0 +1,31 @@
+[
+    {
+        "@timestamp": "2019-01-07T21:25:21.871Z",
+        "ecs.version": "1.0.0-beta2",
+        "event.dataset": "logstash.log",
+        "event.module": "logstash",
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.flags": [
+            "multiline"
+        ],
+        "log.offset": 0,
+        "logstash.log.level": "INFO",
+        "logstash.log.log_event.count": 1,
+        "logstash.log.log_event.running_pipelines": [
+            {
+                "metaClass": {
+                    "metaClass": {
+                        "metaClass": {
+                            "non_running_pipelines": [],
+                            "running_pipelines": "[:main]"
+                        }
+                    }
+                }
+            }
+        ],
+        "logstash.log.message": "Pipelines running",
+        "logstash.log.module": "logstash.agent",
+        "logstash.log.thread": "Ruby-0-Thread-1: /Users/mat/work/elastic/releases/6.5.1/logstash/lib/bootstrap/environment.rb:6"
+    }
+]
diff --git a/filebeat/module/logstash/slowlog/test/slowlog-json.log b/filebeat/module/logstash/slowlog/test/slowlog-json.log
@@ -0,0 +1 @@
+{"level":"INFO","loggerName":"slowlog.logstash.filters.ruby","timeMillis":1546959515605,"thread":"Ruby-0-Thread-6: :1","logEvent":{"message":"event processing time","plugin_params":{"code":"sleep(5)","id":"9746eeec777fcb4c7ea30366dab5445873a8560b7f1766ee1272e59b08d731fb"},"took_in_nanos":5026401704,"took_in_millis":5026,"event":"{\"sequence\":0,\"host\":\"matbook-pro.lan\",\"message\":\"Hello world!\",\"@version\":\"1\",\"@timestamp\":\"2019-01-08T14:58:30.399Z\"}"}}
diff --git a/filebeat/module/logstash/slowlog/test/slowlog-json.log-expected.json b/filebeat/module/logstash/slowlog/test/slowlog-json.log-expected.json
@@ -0,0 +1,21 @@
+[
+    {
+        "@timestamp": "2019-01-08T14:58:35.605Z",
+        "ecs.version": "1.0.0-beta2",
+        "event.dataset": "logstash.slowlog",
+        "event.module": "logstash",
+        "fileset.name": "slowlog",
+        "input.type": "log",
+        "log.offset": 0,
+        "logstash.slowlog.event": "{\"sequence\":0,\"host\":\"matbook-pro.lan\",\"message\":\"Hello world!\",\"@version\":\"1\",\"@timestamp\":\"2019-01-08T14:58:30.399Z\"}",
+        "logstash.slowlog.level": "INFO",
+        "logstash.slowlog.module": "slowlog.logstash.filters.ruby",
+        "logstash.slowlog.plugin_name": "ruby",
+        "logstash.slowlog.plugin_params_object.code": "sleep(5)",
+        "logstash.slowlog.plugin_params_object.id": "9746eeec777fcb4c7ea30366dab5445873a8560b7f1766ee1272e59b08d731fb",
+        "logstash.slowlog.plugin_type": "filters",
+        "logstash.slowlog.thread": "Ruby-0-Thread-6: :1",
+        "logstash.slowlog.took_in_millis": 5026,
+        "logstash.slowlog.took_in_nanos": 5026401704
+    }
+]
diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py
@@ -122,6 +122,11 @@ def run_on_file(self, module, fileset, test_file, cfgfile):
             "-M", "*.*.input.close_eof=true",
         ]
 
+        # Based on the convention that if a name contains -json the json format is needed. Currently used for LS.
+        if "-json" in test_file:
+            cmd.append("-M")
+            cmd.append("{module}.{fileset}.var.format=json".format(module=module, fileset=fileset))
+
         output_path = os.path.join(self.working_dir)
         output = open(os.path.join(output_path, "output.log"), "ab")
         output.write(" ".join(cmd) + "\n")