[7.6] Add pattern to handle logs from newer ES versions (#17406) (#17423

) * Add pattern to handle logs from newer ES versions (#17406) * Add pattern to handle logs from newer ES versions * Adding CHANGELOG entry * Removing fields not available in 7.6
elastic · Apr 3, 2020 · 2ae66ad · 2ae66ad
1 parent 176f95e
commit 2ae66ad
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 1 deletion.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -50,6 +50,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fixed a mapping exception when ingesting CEF logs that used the spriv or dpriv extensions. {issue}17216[17216] {pull}17220[17220]
 - Fix `elasticsearch.gc` fileset to not collect _all_ logs when Elasticsearch is running in Docker. {issue}13164[13164] {issue}16583[16583] {pull}17164[17164]
 - Fixed a mapping exception when ingesting Logstash plain logs (7.4+) with pipeline ids containing non alphanumeric chars. {issue}17242[17242] {pull}17243[17243]
+- Fix `elasticsearch.audit` data ingest pipeline to be more forgiving with date formats found in Elasticsearch audit logs. {pull}17406[17406]
 
 *Heartbeat*
 

diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.json b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.json
@@ -24,7 +24,8 @@
                 "field": "elasticsearch.audit.@timestamp",
                 "target_field": "elasticsearch.audit.@timestamp",
                 "formats": [
-                    "yyyy-MM-dd'T'HH:mm:ss,SSS"
+                    "yyyy-MM-dd'T'HH:mm:ss,SSS",
+                    "yyyy-MM-dd'T'HH:mm:ss,SSSZ"
                 ],
                 "timezone": "{{ event.timezone }}"
             }

diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-761.log b/filebeat/module/elasticsearch/audit/test/test-audit-761.log
@@ -0,0 +1 @@
+{"@timestamp":"2020-04-01T11:21:06,725+0200", "node.id":"vvj136QVQ2Ci2aXmrhyi3Q", "event.type":"transport", "event.action":"access_granted", "user.name":"logstash_manager", "user.realm":"native1", "user.roles":["logstash_admin","cluster_monitor"], "origin.type":"rest", "origin.address":"10.54.25.111:52148", "request.id":"rLBMfPM2Q9q-DQEB_g30ww", "action":"indices:data/read/mget[shard]", "request.name":"MultiGetShardRequest", "indices":[".logstash",".logstash",".logstash",".logstash",".logstash",".logstash",".logstash",".logstash"]}
diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-761.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-761.log-expected.json
@@ -0,0 +1,39 @@
+[
+    {
+        "@timestamp": "2020-04-01T13:21:06.725Z",
+        "elasticsearch.audit.action": "indices:data/read/mget[shard]",
+        "elasticsearch.audit.indices": [
+            ".logstash",
+            ".logstash",
+            ".logstash",
+            ".logstash",
+            ".logstash",
+            ".logstash",
+            ".logstash",
+            ".logstash"
+        ],
+        "elasticsearch.audit.layer": "transport",
+        "elasticsearch.audit.origin.type": "rest",
+        "elasticsearch.audit.request.id": "rLBMfPM2Q9q-DQEB_g30ww",
+        "elasticsearch.audit.request.name": "MultiGetShardRequest",
+        "elasticsearch.audit.user.realm": "native1",
+        "elasticsearch.audit.user.roles": [
+            "logstash_admin",
+            "cluster_monitor"
+        ],
+        "elasticsearch.node.id": "vvj136QVQ2Ci2aXmrhyi3Q",
+        "event.action": "access_granted",
+        "event.dataset": "elasticsearch.audit",
+        "event.module": "elasticsearch",
+        "event.timezone": "-02:00",
+        "fileset.name": "audit",
+        "input.type": "log",
+        "log.offset": 0,
+        "message": "{\"@timestamp\":\"2020-04-01T11:21:06,725+0200\", \"node.id\":\"vvj136QVQ2Ci2aXmrhyi3Q\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"logstash_manager\", \"user.realm\":\"native1\", \"user.roles\":[\"logstash_admin\",\"cluster_monitor\"], \"origin.type\":\"rest\", \"origin.address\":\"10.54.25.111:52148\", \"request.id\":\"rLBMfPM2Q9q-DQEB_g30ww\", \"action\":\"indices:data/read/mget[shard]\", \"request.name\":\"MultiGetShardRequest\", \"indices\":[\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\"]}",
+        "service.type": "elasticsearch",
+        "source.address": "10.54.25.111:52148",
+        "source.ip": "10.54.25.111",
+        "source.port": 52148,
+        "user.name": "logstash_manager"
+    }
+]