From d86d9fc9a96d48ea4c2168c096d1e41157a6c215 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 3 Jan 2019 13:21:17 -0700 Subject: [PATCH] Cherry-pick to 6.6: Ignore timestamp in redis, haproxy and system filebeat module (#9855) (#9858) * Ignore timestamp in redis, haproxy and system filebeat module (#9855) * Ignore timestamp in redis, haproxy and system filebeat module * Fixing formatting (cherry picked from commit a5f0323f475b336f8b814de6b6ba6b92a0a557a0) * Use event.dataset in if statement * Update golden files for icinga/startup test to exclude timestamp (#9506) The icing/startup log file does not contain a timestamp. Because of this the timestamp from filebeat is taken. During the generation of golden files still a timestamp was added but every skipped on comparison. Instead now the timestamp is not added to the generated file anymore to now show a diff each time GENERATE is run. * Regenerate expectations for logs without timestamps (#9862) * Rerun GENERATE=1 to update expected json files --- .../module/haproxy/log/test/default.log-expected.json | 1 - .../module/haproxy/log/test/haproxy.log-expected.json | 1 - .../module/haproxy/log/test/tcplog.log-expected.json | 1 - .../module/icinga/startup/test/test.log-expected.json | 2 -- filebeat/module/redis/log/test/test.log-expected.json | 4 ---- .../module/system/auth/test/test.log-expected.json | 10 ---------- .../syslog/test/darwin-syslog-sample.log-expected.json | 3 --- filebeat/tests/system/test_modules.py | 10 +++++----- 8 files changed, 5 insertions(+), 27 deletions(-) diff --git a/filebeat/module/haproxy/log/test/default.log-expected.json b/filebeat/module/haproxy/log/test/default.log-expected.json index 0ee6f23b074..5771d863b29 100644 --- a/filebeat/module/haproxy/log/test/default.log-expected.json +++ b/filebeat/module/haproxy/log/test/default.log-expected.json @@ -1,6 +1,5 @@ [ { - "@timestamp": "2018-09-20T15:42:59.000Z", "event.dataset": "haproxy.log", "fileset.module": "haproxy", "fileset.name": "log", diff --git a/filebeat/module/haproxy/log/test/haproxy.log-expected.json b/filebeat/module/haproxy/log/test/haproxy.log-expected.json index 96d6630de4e..6c4b0d4fa8e 100644 --- a/filebeat/module/haproxy/log/test/haproxy.log-expected.json +++ b/filebeat/module/haproxy/log/test/haproxy.log-expected.json @@ -1,6 +1,5 @@ [ { - "@timestamp": "2018-07-30T09:03:52.726Z", "event.dataset": "haproxy.log", "fileset.module": "haproxy", "fileset.name": "log", diff --git a/filebeat/module/haproxy/log/test/tcplog.log-expected.json b/filebeat/module/haproxy/log/test/tcplog.log-expected.json index b0a2ff08d92..3c84eec09b1 100644 --- a/filebeat/module/haproxy/log/test/tcplog.log-expected.json +++ b/filebeat/module/haproxy/log/test/tcplog.log-expected.json @@ -1,6 +1,5 @@ [ { - "@timestamp": "2018-09-20T15:44:23.285Z", "event.dataset": "haproxy.log", "fileset.module": "haproxy", "fileset.name": "log", diff --git a/filebeat/module/icinga/startup/test/test.log-expected.json b/filebeat/module/icinga/startup/test/test.log-expected.json index be7a016f527..04e34bd0814 100644 --- a/filebeat/module/icinga/startup/test/test.log-expected.json +++ b/filebeat/module/icinga/startup/test/test.log-expected.json @@ -1,6 +1,5 @@ [ { - "@timestamp": "2018-12-12T11:22:05.182Z", "event.dataset": "icinga.startup", "fileset.module": "icinga", "fileset.name": "startup", @@ -12,7 +11,6 @@ "prospector.type": "log" }, { - "@timestamp": "2018-12-12T11:22:05.182Z", "event.dataset": "icinga.startup", "fileset.module": "icinga", "fileset.name": "startup", diff --git a/filebeat/module/redis/log/test/test.log-expected.json b/filebeat/module/redis/log/test/test.log-expected.json index 649114bbf6e..1f576699308 100644 --- a/filebeat/module/redis/log/test/test.log-expected.json +++ b/filebeat/module/redis/log/test/test.log-expected.json @@ -1,6 +1,5 @@ [ { - "@timestamp": "2018-05-30T12:23:52.442Z", "event.dataset": "redis.log", "fileset.module": "redis", "fileset.name": "log", @@ -13,7 +12,6 @@ "redis.log.role": "master" }, { - "@timestamp": "2018-05-30T10:05:20.000Z", "event.dataset": "redis.log", "fileset.module": "redis", "fileset.name": "log", @@ -24,7 +22,6 @@ "redis.log.message": "0 clients connected (0 slaves), 618932 bytes in use, 0 shared objects." }, { - "@timestamp": "2018-05-31T04:32:08.000Z", "event.dataset": "redis.log", "fileset.module": "redis", "fileset.name": "log", @@ -35,7 +32,6 @@ "redis.log.message": "The server is now ready to accept connections on port 6379\"" }, { - "@timestamp": "2017-05-30T10:57:24.000Z", "event.dataset": "redis.log", "fileset.module": "redis", "fileset.name": "log", diff --git a/filebeat/module/system/auth/test/test.log-expected.json b/filebeat/module/system/auth/test/test.log-expected.json index 83018d65dbe..9b003fd4cdd 100644 --- a/filebeat/module/system/auth/test/test.log-expected.json +++ b/filebeat/module/system/auth/test/test.log-expected.json @@ -1,6 +1,5 @@ [ { - "@timestamp": "2018-02-21T21:54:44.000Z", "event.dataset": "system.auth", "fileset.module": "system", "fileset.name": "auth", @@ -18,7 +17,6 @@ "system.auth.user": "vagrant" }, { - "@timestamp": "2018-02-23T00:13:35.000Z", "event.dataset": "system.auth", "fileset.module": "system", "fileset.name": "auth", @@ -35,7 +33,6 @@ "system.auth.user": "vagrant" }, { - "@timestamp": "2018-02-21T21:56:12.000Z", "event.dataset": "system.auth", "fileset.module": "system", "fileset.name": "auth", @@ -50,7 +47,6 @@ "system.auth.user": "test" }, { - "@timestamp": "2018-02-20T08:35:22.000Z", "event.dataset": "system.auth", "fileset.module": "system", "fileset.name": "auth", @@ -73,7 +69,6 @@ "system.auth.user": "root" }, { - "@timestamp": "2018-02-21T23:35:33.000Z", "event.dataset": "system.auth", "fileset.module": "system", "fileset.name": "auth", @@ -89,7 +84,6 @@ "system.auth.user": "vagrant" }, { - "@timestamp": "2018-02-19T15:30:04.000Z", "event.dataset": "system.auth", "fileset.module": "system", "fileset.name": "auth", @@ -102,7 +96,6 @@ "system.auth.timestamp": "Feb 19 15:30:04" }, { - "@timestamp": "2018-02-23T00:08:48.000Z", "event.dataset": "system.auth", "fileset.module": "system", "fileset.name": "auth", @@ -118,7 +111,6 @@ "system.auth.user": "vagrant" }, { - "@timestamp": "2018-02-24T00:13:02.000Z", "event.dataset": "system.auth", "fileset.module": "system", "fileset.name": "auth", @@ -135,7 +127,6 @@ "system.auth.user": "tsg" }, { - "@timestamp": "2018-02-22T11:47:05.000Z", "event.dataset": "system.auth", "fileset.module": "system", "fileset.name": "auth", @@ -149,7 +140,6 @@ "system.auth.timestamp": "Feb 22 11:47:05" }, { - "@timestamp": "2018-02-22T11:47:05.000Z", "event.dataset": "system.auth", "fileset.module": "system", "fileset.name": "auth", diff --git a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json index a17409df62b..cc4364084b1 100644 --- a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json +++ b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json @@ -1,6 +1,5 @@ [ { - "@timestamp": "2018-12-13T11:35:28.000Z", "event.dataset": "system.syslog", "fileset.module": "system", "fileset.name": "syslog", @@ -17,7 +16,6 @@ "system.syslog.timestamp": "Dec 13 11:35:28" }, { - "@timestamp": "2018-12-13T11:35:28.000Z", "event.dataset": "system.syslog", "fileset.module": "system", "fileset.name": "syslog", @@ -31,7 +29,6 @@ "system.syslog.timestamp": "Dec 13 11:35:28" }, { - "@timestamp": "2018-04-04T03:39:57.000Z", "event.dataset": "system.syslog", "fileset.module": "system", "fileset.name": "syslog", diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index 4a3b39aa28a..ba5b4fd785a 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -159,6 +159,7 @@ def _test_expected_events(self, test_file, objects): for k, obj in enumerate(objects): objects[k] = self.flatten_object(obj, {}, "") clean_keys(objects[k]) + json.dump(objects, f, indent=4, sort_keys=True) with open(test_file + "-expected.json", "r") as f: @@ -175,11 +176,6 @@ def _test_expected_events(self, test_file, objects): obj = self.flatten_object(obj, {}, "") clean_keys(obj) - # Remove timestamp for comparison where timestamp is not part of the log line - if obj["fileset.module"] == "icinga" and obj["fileset.name"] == "startup": - delete_key(obj, "@timestamp") - delete_key(ev, "@timestamp") - if ev == obj: found = True break @@ -199,6 +195,10 @@ def clean_keys(obj): for key in host_keys + time_keys + other_keys: delete_key(obj, key) + # Remove timestamp for comparison where timestamp is not part of the log line + if obj["event.dataset"] in ["icinga.startup", "redis.log", "haproxy.log", "system.auth", "system.syslog"]: + delete_key(obj, "@timestamp") + def delete_key(obj, key): if key in obj: