Cherry-pick to 6.6: Ignore timestamp in redis, haproxy and system fil…

…ebeat module (elastic#9855) (elastic#9858) * Ignore timestamp in redis, haproxy and system filebeat module (elastic#9855) * Ignore timestamp in redis, haproxy and system filebeat module * Fixing formatting (cherry picked from commit b9a20f7) * Use event.dataset in if statement * Update golden files for icinga/startup test to exclude timestamp (elastic#9506) The icing/startup log file does not contain a timestamp. Because of this the timestamp from filebeat is taken. During the generation of golden files still a timestamp was added but every skipped on comparison. Instead now the timestamp is not added to the generated file anymore to now show a diff each time GENERATE is run. * Regenerate expectations for logs without timestamps (elastic#9862) * Rerun GENERATE=1 to update expected json files (cherry picked from commit 6defdd4)
jsoriano · Jan 15, 2019 · eec7e31 · eec7e31
1 parent 5a582bd
commit eec7e31
Show file tree

Hide file tree

Showing 8 changed files with 6 additions and 27 deletions.
diff --git a/filebeat/module/haproxy/log/test/default.log-expected.json b/filebeat/module/haproxy/log/test/default.log-expected.json
@@ -1,6 +1,5 @@
 [
     {
-        "@timestamp": "2018-09-20T15:42:59.000Z", 
         "fileset.module": "haproxy", 
         "fileset.name": "log", 
         "haproxy.client.ip": "1.2.3.4", 

diff --git a/filebeat/module/haproxy/log/test/haproxy.log-expected.json b/filebeat/module/haproxy/log/test/haproxy.log-expected.json
@@ -1,6 +1,5 @@
 [
     {
-        "@timestamp": "2018-07-30T09:03:52.726Z", 
         "fileset.module": "haproxy", 
         "fileset.name": "log", 
         "haproxy.backend_name": "docs_microservice", 

diff --git a/filebeat/module/haproxy/log/test/tcplog.log-expected.json b/filebeat/module/haproxy/log/test/tcplog.log-expected.json
@@ -1,6 +1,5 @@
 [
     {
-        "@timestamp": "2018-09-20T15:44:23.285Z", 
         "fileset.module": "haproxy", 
         "fileset.name": "log", 
         "haproxy.backend_name": "app", 

diff --git a/filebeat/module/icinga/startup/test/test.log-expected.json b/filebeat/module/icinga/startup/test/test.log-expected.json
@@ -1,6 +1,5 @@
 [
     {
-        "@timestamp": "2018-07-24T21:15:27.590Z", 
         "fileset.module": "icinga", 
         "fileset.name": "startup", 
         "icinga.startup.facility": "cli", 
@@ -11,7 +10,6 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-07-24T21:15:27.590Z", 
         "fileset.module": "icinga", 
         "fileset.name": "startup", 
         "icinga.startup.facility": "cli", 

diff --git a/filebeat/module/redis/log/test/test.log-expected.json b/filebeat/module/redis/log/test/test.log-expected.json
@@ -1,6 +1,5 @@
 [
     {
-        "@timestamp": "2018-05-30T12:23:52.442Z", 
         "fileset.module": "redis", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -12,7 +11,6 @@
         "redis.log.role": "master"
     }, 
     {
-        "@timestamp": "2018-05-30T10:05:20.000Z", 
         "fileset.module": "redis", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -22,7 +20,6 @@
         "redis.log.message": "0 clients connected (0 slaves), 618932 bytes in use, 0 shared objects."
     }, 
     {
-        "@timestamp": "2018-05-31T04:32:08.000Z", 
         "fileset.module": "redis", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -32,7 +29,6 @@
         "redis.log.message": "The server is now ready to accept connections on port 6379\""
     }, 
     {
-        "@timestamp": "2017-05-30T10:57:24.000Z", 
         "fileset.module": "redis", 
         "fileset.name": "log", 
         "input.type": "log", 

diff --git a/filebeat/module/system/auth/test/test.log-expected.json b/filebeat/module/system/auth/test/test.log-expected.json
@@ -1,6 +1,5 @@
 [
     {
-        "@timestamp": "2018-02-21T21:54:44.000Z", 
         "fileset.module": "system", 
         "fileset.name": "auth", 
         "input.type": "log", 
@@ -17,7 +16,6 @@
         "system.auth.user": "vagrant"
     }, 
     {
-        "@timestamp": "2018-02-23T00:13:35.000Z", 
         "fileset.module": "system", 
         "fileset.name": "auth", 
         "input.type": "log", 
@@ -33,7 +31,6 @@
         "system.auth.user": "vagrant"
     }, 
     {
-        "@timestamp": "2018-02-21T21:56:12.000Z", 
         "fileset.module": "system", 
         "fileset.name": "auth", 
         "input.type": "log", 
@@ -47,7 +44,6 @@
         "system.auth.user": "test"
     }, 
     {
-        "@timestamp": "2018-02-20T08:35:22.000Z", 
         "fileset.module": "system", 
         "fileset.name": "auth", 
         "input.type": "log", 
@@ -69,7 +65,6 @@
         "system.auth.user": "root"
     }, 
     {
-        "@timestamp": "2018-02-21T23:35:33.000Z", 
         "fileset.module": "system", 
         "fileset.name": "auth", 
         "input.type": "log", 
@@ -84,7 +79,6 @@
         "system.auth.user": "vagrant"
     }, 
     {
-        "@timestamp": "2018-02-19T15:30:04.000Z", 
         "fileset.module": "system", 
         "fileset.name": "auth", 
         "input.type": "log", 
@@ -96,7 +90,6 @@
         "system.auth.timestamp": "Feb 19 15:30:04"
     }, 
     {
-        "@timestamp": "2018-02-23T00:08:48.000Z", 
         "fileset.module": "system", 
         "fileset.name": "auth", 
         "input.type": "log", 
@@ -111,7 +104,6 @@
         "system.auth.user": "vagrant"
     }, 
     {
-        "@timestamp": "2018-02-24T00:13:02.000Z", 
         "fileset.module": "system", 
         "fileset.name": "auth", 
         "input.type": "log", 
@@ -127,7 +119,6 @@
         "system.auth.user": "tsg"
     }, 
     {
-        "@timestamp": "2018-02-22T11:47:05.000Z", 
         "fileset.module": "system", 
         "fileset.name": "auth", 
         "input.type": "log", 
@@ -140,7 +131,6 @@
         "system.auth.timestamp": "Feb 22 11:47:05"
     }, 
     {
-        "@timestamp": "2018-02-22T11:47:05.000Z", 
         "fileset.module": "system", 
         "fileset.name": "auth", 
         "input.type": "log", 

diff --git a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json
@@ -1,6 +1,5 @@
 [
     {
-        "@timestamp": "2018-12-13T11:35:28.000Z", 
         "fileset.module": "system", 
         "fileset.name": "syslog", 
         "input.type": "log", 
@@ -16,7 +15,6 @@
         "system.syslog.timestamp": "Dec 13 11:35:28"
     }, 
     {
-        "@timestamp": "2018-12-13T11:35:28.000Z", 
         "fileset.module": "system", 
         "fileset.name": "syslog", 
         "input.type": "log", 
@@ -29,7 +27,6 @@
         "system.syslog.timestamp": "Dec 13 11:35:28"
     }, 
     {
-        "@timestamp": "2018-04-04T03:39:57.000Z", 
         "fileset.module": "system", 
         "fileset.name": "syslog", 
         "input.type": "log", 

diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py
@@ -159,6 +159,7 @@ def _test_expected_events(self, test_file, objects):
                 for k, obj in enumerate(objects):
                     objects[k] = self.flatten_object(obj, {}, "")
                     clean_keys(objects[k])
+
                 json.dump(objects, f, indent=4, sort_keys=True)
 
         with open(test_file + "-expected.json", "r") as f:
@@ -175,11 +176,6 @@ def _test_expected_events(self, test_file, objects):
                 obj = self.flatten_object(obj, {}, "")
                 clean_keys(obj)
 
-                # Remove timestamp for comparison where timestamp is not part of the log line
-                if obj["fileset.module"] == "icinga" and obj["fileset.name"] == "startup":
-                    delete_key(obj, "@timestamp")
-                    delete_key(ev, "@timestamp")
-
                 if ev == obj:
                     found = True
                     break
@@ -199,6 +195,11 @@ def clean_keys(obj):
     for key in host_keys + time_keys + other_keys:
         delete_key(obj, key)
 
+    # Remove timestamp for comparison where timestamp is not part of the log line
+    dataset = "%s.%s" % (obj['fileset.module'], obj['fileset.name'])
+    if dataset in ["icinga.startup", "redis.log", "haproxy.log", "system.auth", "system.syslog"]:
+        delete_key(obj, "@timestamp")
+
 
 def delete_key(obj, key):
     if key in obj: