Fix date format in Mongodb Ingest pipeline (elastic#7974)

Previously, number of month was not parsed correctly in case of mongodb/log Filebeat module. Now this is fixed and the expected outputs are updated accordingly. Closes elastic#7958
kvch · Aug 15, 2018 · 87b9f32 · 87b9f32
1 parent 62f8c45
commit 87b9f32
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 35 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -47,6 +47,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff]
 *Filebeat*
 
 - Fixed a memory leak when harvesters are closed. {pull}7820[7820]
+- Fix date format in Mongodb Ingest pipeline. {pull}7974[7974]
 
 *Heartbeat*
 

diff --git a/filebeat/module/mongodb/log/ingest/pipeline.json b/filebeat/module/mongodb/log/ingest/pipeline.json
@@ -24,7 +24,7 @@
     "date": {
       "field": "mongodb.log.timestamp",
       "target_field": "@timestamp",
-      "formats": ["YYYY-MM-DD'T'HH:mm:ss.SSSZZ"]
+      "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSZZ"]
     }
   },
   {

diff --git a/filebeat/module/mongodb/log/test/mongodb-debian-3.2.11.log-expected.json b/filebeat/module/mongodb/log/test/mongodb-debian-3.2.11.log-expected.json
@@ -1,6 +1,6 @@
 [
     {
-        "@timestamp": "2018-01-05T12:44:56.657Z", 
+        "@timestamp": "2018-02-05T12:44:56.657Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -12,7 +12,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T12:44:56.657Z", 
+        "@timestamp": "2018-02-05T12:44:56.657Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -24,7 +24,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T12:44:56.657Z", 
+        "@timestamp": "2018-02-05T12:44:56.657Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -36,7 +36,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T12:44:56.677Z", 
+        "@timestamp": "2018-02-05T12:44:56.677Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -48,7 +48,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T12:44:56.724Z", 
+        "@timestamp": "2018-02-05T12:44:56.724Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -60,7 +60,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T12:44:56.724Z", 
+        "@timestamp": "2018-02-05T12:44:56.724Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -72,7 +72,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T12:44:56.744Z", 
+        "@timestamp": "2018-02-05T12:44:56.744Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -84,7 +84,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T12:50:55.170Z", 
+        "@timestamp": "2018-02-05T12:50:55.170Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -96,7 +96,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T12:50:55.487Z", 
+        "@timestamp": "2018-02-05T12:50:55.487Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -108,7 +108,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T13:49:45.606Z", 
+        "@timestamp": "2018-02-05T13:49:45.606Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -120,7 +120,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T13:49:45.606Z", 
+        "@timestamp": "2018-02-05T13:49:45.606Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -132,7 +132,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T13:49:45.606Z", 
+        "@timestamp": "2018-02-05T13:49:45.606Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -144,7 +144,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T13:49:45.606Z", 
+        "@timestamp": "2018-02-05T13:49:45.606Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -156,7 +156,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T13:49:45.606Z", 
+        "@timestamp": "2018-02-05T13:49:45.606Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -168,7 +168,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T13:49:45.688Z", 
+        "@timestamp": "2018-02-05T13:49:45.688Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -180,7 +180,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T12:44:56.657Z", 
+        "@timestamp": "2018-02-05T12:44:56.657Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -192,7 +192,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T12:44:56.657Z", 
+        "@timestamp": "2018-02-05T12:44:56.657Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -204,7 +204,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T12:44:56.657Z", 
+        "@timestamp": "2018-02-05T12:44:56.657Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -216,7 +216,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T12:44:56.657Z", 
+        "@timestamp": "2018-02-05T12:44:56.657Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -228,7 +228,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T12:50:55.170Z", 
+        "@timestamp": "2018-02-05T12:50:55.170Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -240,7 +240,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T12:50:56.180Z", 
+        "@timestamp": "2018-02-05T12:50:56.180Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -252,7 +252,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T13:15:42.095Z", 
+        "@timestamp": "2018-02-05T13:15:42.095Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -264,7 +264,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T13:49:45.606Z", 
+        "@timestamp": "2018-02-05T13:49:45.606Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -276,7 +276,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T13:49:45.606Z", 
+        "@timestamp": "2018-02-05T13:49:45.606Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -288,7 +288,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T13:49:45.688Z", 
+        "@timestamp": "2018-02-05T13:49:45.688Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -300,7 +300,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T12:44:56.657Z", 
+        "@timestamp": "2018-02-05T12:44:56.657Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -312,7 +312,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T12:44:56.657Z", 
+        "@timestamp": "2018-02-05T12:44:56.657Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -324,7 +324,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T12:44:56.657Z", 
+        "@timestamp": "2018-02-05T12:44:56.657Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -336,7 +336,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T12:50:55.487Z", 
+        "@timestamp": "2018-02-05T12:50:55.487Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -348,7 +348,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T12:50:56.180Z", 
+        "@timestamp": "2018-02-05T12:50:56.180Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -360,7 +360,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T13:11:41.401Z", 
+        "@timestamp": "2018-02-05T13:11:41.401Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -372,7 +372,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T13:49:45.605Z", 
+        "@timestamp": "2018-02-05T13:49:45.605Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -384,7 +384,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T13:49:45.605Z", 
+        "@timestamp": "2018-02-05T13:49:45.605Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -396,7 +396,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-01-05T13:49:45.606Z", 
+        "@timestamp": "2018-02-05T13:49:45.606Z", 
         "fileset.module": "mongodb", 
         "fileset.name": "log", 
         "input.type": "log",