Add pattern to handle logs from newer ES versions (elastic#17406) (el…

…astic#17422) * Add pattern to handle logs from newer ES versions * Adding CHANGELOG entry
leweafan · Apr 3, 2020 · 7b11283 · 7b11283
1 parent 39c6a3d
commit 7b11283
Show file tree

Hide file tree

Showing 4 changed files with 49 additions and 0 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -134,6 +134,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fixed a mapping exception when ingesting CEF logs that used the spriv or dpriv extensions. {issue}17216[17216] {pull}17220[17220]
 - Fixed a mapping exception when ingesting Logstash plain logs (7.4+) with pipeline ids containing non alphanumeric chars. {issue}17242[17242] {pull}17243[17243]
 - Fixed MySQL slowlog module causing "regular expression has redundant nested repeat operator" warning in Elasticsearch. {issue}17086[17086] {pull}17156[17156]
+- Fix `elasticsearch.audit` data ingest pipeline to be more forgiving with date formats found in Elasticsearch audit logs. {pull}17406[17406]
 
 *Heartbeat*
 

diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml
@@ -14,6 +14,7 @@ processors:
     target_field: elasticsearch.audit.@timestamp
     formats:
     - yyyy-MM-dd'T'HH:mm:ss,SSS
+    - yyyy-MM-dd'T'HH:mm:ss,SSSZ
     timezone: '{{ event.timezone }}'
 - remove:
     if: ctx.elasticsearch.audit['@timestamp'] == null && ctx.event.timezone != null

diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-761.log b/filebeat/module/elasticsearch/audit/test/test-audit-761.log
@@ -0,0 +1 @@
+{"@timestamp":"2020-04-01T11:21:06,725+0200", "node.id":"vvj136QVQ2Ci2aXmrhyi3Q", "event.type":"transport", "event.action":"access_granted", "user.name":"logstash_manager", "user.realm":"native1", "user.roles":["logstash_admin","cluster_monitor"], "origin.type":"rest", "origin.address":"10.54.25.111:52148", "request.id":"rLBMfPM2Q9q-DQEB_g30ww", "action":"indices:data/read/mget[shard]", "request.name":"MultiGetShardRequest", "indices":[".logstash",".logstash",".logstash",".logstash",".logstash",".logstash",".logstash",".logstash"]}
diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-761.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-761.log-expected.json
@@ -0,0 +1,46 @@
+[
+    {
+        "@timestamp": "2020-04-01T13:21:06.725Z",
+        "elasticsearch.audit.action": "indices:data/read/mget[shard]",
+        "elasticsearch.audit.indices": [
+            ".logstash",
+            ".logstash",
+            ".logstash",
+            ".logstash",
+            ".logstash",
+            ".logstash",
+            ".logstash",
+            ".logstash"
+        ],
+        "elasticsearch.audit.layer": "transport",
+        "elasticsearch.audit.origin.type": "rest",
+        "elasticsearch.audit.request.id": "rLBMfPM2Q9q-DQEB_g30ww",
+        "elasticsearch.audit.request.name": "MultiGetShardRequest",
+        "elasticsearch.audit.user.realm": "native1",
+        "elasticsearch.audit.user.roles": [
+            "logstash_admin",
+            "cluster_monitor"
+        ],
+        "elasticsearch.node.id": "vvj136QVQ2Ci2aXmrhyi3Q",
+        "event.action": "access_granted",
+        "event.category": "database",
+        "event.dataset": "elasticsearch.audit",
+        "event.kind": "event",
+        "event.module": "elasticsearch",
+        "event.outcome": "success",
+        "event.timezone": "-02:00",
+        "fileset.name": "audit",
+        "host.id": "vvj136QVQ2Ci2aXmrhyi3Q",
+        "input.type": "log",
+        "log.offset": 0,
+        "message": "{\"@timestamp\":\"2020-04-01T11:21:06,725+0200\", \"node.id\":\"vvj136QVQ2Ci2aXmrhyi3Q\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"logstash_manager\", \"user.realm\":\"native1\", \"user.roles\":[\"logstash_admin\",\"cluster_monitor\"], \"origin.type\":\"rest\", \"origin.address\":\"10.54.25.111:52148\", \"request.id\":\"rLBMfPM2Q9q-DQEB_g30ww\", \"action\":\"indices:data/read/mget[shard]\", \"request.name\":\"MultiGetShardRequest\", \"indices\":[\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\"]}",
+        "related.user": [
+            "logstash_manager"
+        ],
+        "service.type": "elasticsearch",
+        "source.address": "10.54.25.111:52148",
+        "source.ip": "10.54.25.111",
+        "source.port": 52148,
+        "user.name": "logstash_manager"
+    }
+]