Make HAProxy filebeat module follow ECS more closely (#10143)

- Convert one more field under `haproxy.*` to ECS. Previous field names are field aliases towards the new corresponding ECS field: - haproxy.http.response.status_code => http.response.status_code - Populate event.duration
elastic · Jan 17, 2019 · 74076cc · 74076cc
1 parent cdb6c1a
commit 74076cc
Show file tree

Hide file tree

Showing 10 changed files with 59 additions and 27 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -31,6 +31,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Rename `read_timestamp` to `event.created` for Redis input. {pull}9924[9924]
 - Rename a few `elasticsearch.audit.*` fields to map to ECS. {pull}9293[9293]
 - Rename many `iis.error.*` fields to map to ECS. {pull}9955[9955]
+- Adjust fileset `haproxy.log` to map to ECS. {pull}10143[10143]
 - Rename a few `logstash.*` fields to map to ECS, remove logstash.slowlog.message. {pull}9935[9935]
 - Rename a few `mongodb.*` fields to map to ECS. {pull}10009[10009]
 - Rename a few `mysql.*` fields to map to ECS. {pull}10008[10008]
@@ -132,6 +133,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for iis 7.5 log format. {issue}9753[9753] {pull}9967[9967]
 - Add service.type field to all Modules. By default the field is set with the module name. It can be overwritten with `service.type` config. {pull}10042[10042]
 - Elasticsearch module's slowlog now populates `event.duration` (ECS). {pull}9293[9293]
+- HAProxy module now populates `event.duration` and `http.response.bytes` (ECS). {pull}10143[10143]
 
 *Heartbeat*
 

diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml
@@ -448,6 +448,10 @@
   to: destination.port
   alias: true
 
+- from: haproxy.http.response.status_code
+  to: http.response.status_code
+  alias: true
+
 - from: haproxy.geoip.continent_name
   to: source.geo.continent_name
   alias: true
@@ -472,10 +476,6 @@
   to: source.geo.region_iso_code
   alias: true
 
-- from: nginx.access.remote_ip_list
-  to: network.forwarded_ip
-  alias: true
-
 ## Kafka module
 
 - from: kafka.log.level
@@ -644,6 +644,10 @@
   to: message
   alias: true
 
+- from: nginx.access.remote_ip_list
+  to: network.forwarded_ip
+  alias: true
+
 ## PostgreSQL module
 
 - from: postgresql.log.timezone

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -4840,15 +4840,6 @@ Please add description
 Fields related to the HTTP response
 
 
-*`haproxy.http.response.status_code`*::
-+
---
-type: long
-
-HTTP status code returned to the client.
-
---
-
 *`haproxy.http.response.captured_cookie`*::
 +
 --
@@ -4865,6 +4856,15 @@ type: text
 List of headers captured in the response due to the presence of the "capture response header" statement in the frontend.
 
 
+--
+
+*`haproxy.http.response.status_code`*::
++
+--
+type: alias
+
+alias to: http.response.status_code
+
 --
 
 [float]

diff --git a/filebeat/module/haproxy/fields.go b/filebeat/module/haproxy/fields.go
diff --git a/filebeat/module/haproxy/log/_meta/fields.yml b/filebeat/module/haproxy/log/_meta/fields.yml
@@ -1,16 +1,12 @@
 - name: http
   description: Please add description
   type: group
-  fields: 
+  fields:
 
   - name: response
     description: Fields related to the HTTP response
     type: group
     fields:
-    - name: status_code
-      description: HTTP status code returned to the client.
-      type: long
-
     - name: captured_cookie
       description: >
         Optional "name=value" entry indicating that the client had this cookie in the response.
@@ -20,6 +16,11 @@
         List of headers captured in the response due to the presence of the "capture response header" statement in the frontend.
       type: text
 
+    - name: status_code
+      type: alias
+      path: http.response.status_code
+      migration: true
+
   - name: request
     description: Fields related to the HTTP request
     type: group
@@ -36,7 +37,7 @@
     - name: raw_request_line
       description: Complete HTTP request line, including the method, request and HTTP version string.
       type: text
-      
+
     - name: time_active_ms
       description: Time the request remained active in haproxy, which is the total time in milliseconds elapsed between the first byte of the request was received and the last byte of response was sent.
       type: long

diff --git a/filebeat/module/haproxy/log/ingest/pipeline.json b/filebeat/module/haproxy/log/ingest/pipeline.json
@@ -7,7 +7,7 @@
                 "patterns": [
                     "%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{GREEDYDATA} %{IPORHOST:haproxy.client.ip}:%{POSINT:source.port:long} %{WORD} %{IPORHOST:destination.ip}:%{POSINT:destination.port:long} \\(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\\)",
 
-                    "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:haproxy.http.request.time_active_ms:long} %{NUMBER:haproxy.http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} |\\{%{DATA}\\} )?\"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"",
+                    "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:haproxy.http.request.time_active_ms:long} %{NUMBER:http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} |\\{%{DATA}\\} )?\"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"",
 
                     "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name} %{GREEDYDATA:haproxy.error_message}",
 
@@ -68,6 +68,22 @@
                 "separator": "\\|",
                 "ignore_failure": true
             }
+        },
+
+        {
+            "script": {
+                "lang": "painless",
+                "source": "if (ctx.haproxy.http?.request?.time_active_ms != null) { ctx.event.duration = ctx.haproxy.http.request.time_active_ms * 1000000 } else { if (ctx.haproxy.tcp?.processing_time_ms != null) { ctx.event.duration = ctx.haproxy.tcp.processing_time_ms * 1000000 } }"
+            }
+        },
+
+        {
+            "convert": {
+                "field": "haproxy.bytes_read",
+                "target_field": "http.response.bytes",
+                "type": "long",
+                "if": "ctx.containsKey('http')"
+            }
         }
     ],
     "on_failure": [

diff --git a/filebeat/module/haproxy/log/test/haproxy.log-expected.json b/filebeat/module/haproxy/log/test/haproxy.log-expected.json
@@ -2,6 +2,7 @@
     {
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "haproxy.log",
+        "event.duration": 2000000,
         "event.module": "haproxy",
         "fileset.name": "log",
         "haproxy.backend_name": "docs_microservice",
@@ -25,11 +26,12 @@
         "haproxy.http.request.time_wait_without_data_ms": 0,
         "haproxy.http.response.captured_cookie": "-",
         "haproxy.http.response.captured_headers": [],
-        "haproxy.http.response.status_code": 304,
         "haproxy.server_name": "docs",
         "haproxy.server_queue": 0,
         "haproxy.termination_state": "----",
         "haproxy.total_waiting_time_ms": 0,
+        "http.response.bytes": 168,
+        "http.response.status_code": 304,
         "input.type": "log",
         "log.offset": 0,
         "process.name": "haproxy",

diff --git a/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json b/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json
@@ -2,6 +2,7 @@
     {
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "haproxy.log",
+        "event.duration": 0,
         "event.module": "haproxy",
         "fileset.name": "log",
         "haproxy.backend_name": "http-webservices",
@@ -21,11 +22,12 @@
         "haproxy.http.request.time_wait_ms": 0,
         "haproxy.http.request.time_wait_without_data_ms": -1,
         "haproxy.http.response.captured_cookie": "-",
-        "haproxy.http.response.status_code": 503,
         "haproxy.server_name": "<NOSRV>",
         "haproxy.server_queue": 0,
         "haproxy.termination_state": "SC--",
         "haproxy.total_waiting_time_ms": -1,
+        "http.response.bytes": 213,
+        "http.response.status_code": 503,
         "input.type": "log",
         "log.offset": 0,
         "process.name": "haproxy",
@@ -37,6 +39,7 @@
     {
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "haproxy.log",
+        "event.duration": 0,
         "event.module": "haproxy",
         "fileset.name": "log",
         "haproxy.backend_name": "http-webservices",
@@ -56,11 +59,12 @@
         "haproxy.http.request.time_wait_ms": 0,
         "haproxy.http.request.time_wait_without_data_ms": -1,
         "haproxy.http.response.captured_cookie": "-",
-        "haproxy.http.response.status_code": 503,
         "haproxy.server_name": "<NOSRV>",
         "haproxy.server_queue": 0,
         "haproxy.termination_state": "SC--",
         "haproxy.total_waiting_time_ms": -1,
+        "http.response.bytes": 213,
+        "http.response.status_code": 503,
         "input.type": "log",
         "log.offset": 186,
         "process.name": "haproxy",
@@ -72,6 +76,7 @@
     {
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "haproxy.log",
+        "event.duration": 0,
         "event.module": "haproxy",
         "fileset.name": "log",
         "haproxy.backend_name": "http-webservices",
@@ -95,11 +100,12 @@
         "haproxy.http.request.time_wait_without_data_ms": -1,
         "haproxy.http.response.captured_cookie": "-",
         "haproxy.http.response.captured_headers": [],
-        "haproxy.http.response.status_code": 503,
         "haproxy.server_name": "<NOSRV>",
         "haproxy.server_queue": 0,
         "haproxy.termination_state": "SC--",
         "haproxy.total_waiting_time_ms": -1,
+        "http.response.bytes": 213,
+        "http.response.status_code": 503,
         "input.type": "log",
         "log.offset": 394,
         "process.name": "haproxy",

diff --git a/filebeat/module/haproxy/log/test/tcplog.log b/filebeat/module/haproxy/log/test/tcplog.log
@@ -1 +1 @@
-Sep 20 15:44:23 127.0.0.1 haproxy[25457]: 127.0.0.1:40962 [20/Sep/2018:15:44:23.285] main app/<NOSRV> -1/-1/0 212 SC 1/1/0/0/0 0/0
+Sep 20 15:44:23 127.0.0.1 haproxy[25457]: 127.0.0.1:40962 [20/Sep/2018:15:44:23.285] main app/<NOSRV> -1/-1/1 212 SC 1/1/0/0/0 0/0
diff --git a/filebeat/module/haproxy/log/test/tcplog.log-expected.json b/filebeat/module/haproxy/log/test/tcplog.log-expected.json
@@ -2,6 +2,7 @@
     {
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "haproxy.log",
+        "event.duration": 1000000,
         "event.module": "haproxy",
         "fileset.name": "log",
         "haproxy.backend_name": "app",
@@ -18,7 +19,7 @@
         "haproxy.server_name": "<NOSRV>",
         "haproxy.server_queue": 0,
         "haproxy.source": "127.0.0.1",
-        "haproxy.tcp.processing_time_ms": 0,
+        "haproxy.tcp.processing_time_ms": 1,
         "haproxy.termination_state": "SC",
         "haproxy.total_waiting_time_ms": -1,
         "input.type": "log",