Add convert_timezone to nginx module (elastic#10148)

* Add convert_timezone to nginx module

* Run mage fmt update on x-pack filebeat

CHANGELOG.next.asciidoc

@@ -150,6 +150,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Elasticsearch module's slowlog now populates `event.duration` (ECS). {pull}9293[9293]
 - HAProxy module now populates `event.duration` and `http.response.bytes` (ECS). {pull}10143[10143]
 - Teach elasticsearch/audit fileset to parse out some more fields. {issue}10134[10134] {pull}10137[10137]
+- Add convert_timezone to nginx module. {issue}9839[9839] {pull}10148[10148]
 
 *Heartbeat*
 

filebeat/filebeat.reference.yml

@@ -304,6 +304,9 @@ filebeat.modules:
     # can be added under this section.
     #input:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
+
   # Error logs
   #error:
     #enabled: true
@@ -316,6 +319,9 @@ filebeat.modules:
     # can be added under this section.
     #input:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
+
 #------------------------------- Osquery Module -------------------------------
 - module: osquery
   result:

filebeat/module/nginx/_meta/config.reference.yml

@@ -11,6 +11,9 @@
     # can be added under this section.
     #input:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
+
   # Error logs
   #error:
     #enabled: true
@@ -22,3 +25,6 @@
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
+
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false

filebeat/module/nginx/_meta/config.yml

@@ -7,10 +7,16 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: true
+
   # Error logs
   error:
     enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
+
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: true

filebeat/module/nginx/access/config/nginx-access.yml

@@ -4,3 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
+
+{{ if .convert_timezone }}
+processors:
+- add_locale: ~
+{{ end }}

filebeat/module/nginx/access/ingest/default.json

@@ -66,7 +66,9 @@
                 "target_field": "@timestamp",
                 "formats": [
                     "dd/MMM/YYYY:H:m:s Z"
-                ]
+                ],
+                {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
+                "ignore_failure": true
             }
         },
         {

filebeat/module/nginx/access/manifest.yml

@@ -8,6 +8,13 @@ var:
       - /usr/local/var/log/nginx/access.log*
     os.windows:
       - c:/programdata/nginx/logs/*access.log*
+  - name: convert_timezone
+    default: false
+    # if ES < 6.1.0, this flag switches to false automatically when evaluating the
+    # pipeline
+    min_elasticsearch_version:
+      version: 6.1.0
+      value: false
 
 ingest_pipeline: ingest/default.json
 input: config/nginx-access.yml

filebeat/module/nginx/error/config/nginx-error.yml

@@ -4,3 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
+
+{{ if .convert_timezone }}
+processors:
+- add_locale: ~
+{{ end }}

filebeat/module/nginx/error/ingest/pipeline.json

@@ -17,7 +17,9 @@
     "date": {
       "field": "nginx.error.time",
       "target_field": "@timestamp",
-      "formats": ["YYYY/MM/dd H:m:s"]
+      "formats": ["YYYY/MM/dd H:m:s"],
+      {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
+      "ignore_failure": true
     }
   }, {
     "remove": {

filebeat/module/nginx/error/manifest.yml

@@ -8,6 +8,13 @@ var:
       - /usr/local/var/log/nginx/error.log*
     os.windows:
       - c:/programdata/nginx/logs/error.log*
+  - name: convert_timezone
+    default: false
+    # if ES < 6.1.0, this flag switches to false automatically when evaluating the
+    # pipeline
+    min_elasticsearch_version:
+      version: 6.1.0
+      value: false
 
 ingest_pipeline: ingest/pipeline.json
 input: config/nginx-error.yml

filebeat/modules.d/nginx.yml.disabled

@@ -10,10 +10,16 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: true
+
   # Error logs
   error:
     enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
+
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: true

x-pack/filebeat/filebeat.reference.yml

@@ -304,6 +304,9 @@ filebeat.modules:
     # can be added under this section.
     #input:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
+
   # Error logs
   #error:
     #enabled: true
@@ -316,6 +319,9 @@ filebeat.modules:
     # can be added under this section.
     #input:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
+
 #------------------------------- Osquery Module -------------------------------
 - module: osquery
   result: