diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 74e777d2f5b..6eb0839fede 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -58,6 +58,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `convert_timezone` option to Elasticsearch module to convert dates to UTC. {issue}9756[9756] {pull}9761[9761] - Support IPv6 addresses with zone id in IIS ingest pipeline. {issue}9836[9836] {pull}9869[9869] +- Support haproxy log lines without captured headers. {issue}9463[9463] {pull}9958[9958] *Heartbeat* diff --git a/filebeat/module/haproxy/log/ingest/pipeline.json b/filebeat/module/haproxy/log/ingest/pipeline.json index e55c38c7a86..2ad541d985c 100644 --- a/filebeat/module/haproxy/log/ingest/pipeline.json +++ b/filebeat/module/haproxy/log/ingest/pipeline.json @@ -7,7 +7,7 @@ "patterns": [ "%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{GREEDYDATA} %{IPORHOST:haproxy.client.ip}:%{POSINT:source.port:long} %{WORD} %{IPORHOST:destination.ip}:%{POSINT:destination.port:long} \\(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\\)", - "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:haproxy.http.request.time_active_ms:long} %{NUMBER:haproxy.http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} \\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} \"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"", + "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:haproxy.http.request.time_active_ms:long} %{NUMBER:haproxy.http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} |\\{%{DATA}\\} )?\"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"", "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name} %{GREEDYDATA:haproxy.error_message}", diff --git a/filebeat/module/haproxy/log/test/httplog-no-headers.log b/filebeat/module/haproxy/log/test/httplog-no-headers.log new file mode 100644 index 00000000000..e6d4f96f4b7 --- /dev/null +++ b/filebeat/module/haproxy/log/test/httplog-no-headers.log @@ -0,0 +1,4 @@ +Dec 10 12:01:46 voyager haproxy[19312]: 127.0.0.1:35982 [10/Dec/2018:12:01:46.395] http-webservices http-webservices/ 0/-1/-1/-1/0 503 213 - - SC-- 1/1/0/0/0 0/0 "GET / HTTP/1.1" +Dec 10 15:46:49 voyager haproxy[29785]: 127.0.0.1:43738 [10/Dec/2018:15:46:49.497] http-webservices http-webservices/ 0/-1/-1/-1/0 503 213 - - SC-- 1/1/0/0/0 0/0 {localhost:8888||} "GET /foo HTTP/1.1" +Dec 10 15:48:56 voyager haproxy[7873]: 127.0.0.1:44542 [10/Dec/2018:15:48:56.017] http-webservices http-webservices/ 0/-1/-1/-1/0 503 213 - - SC-- 1/1/0/0/0 0/0 {localhost:8888||} {|} "GET /foo HTTP/1.1" + diff --git a/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json b/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json new file mode 100644 index 00000000000..ab6c3a66884 --- /dev/null +++ b/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json @@ -0,0 +1,108 @@ +[ + { + "ecs.version": "1.0.0-beta2", + "event.dataset": "haproxy.log", + "event.module": "haproxy", + "fileset.name": "log", + "haproxy.backend_name": "http-webservices", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 213, + "haproxy.client.ip": "127.0.0.1", + "haproxy.connection_wait_time_ms": -1, + "haproxy.connections.active": 1, + "haproxy.connections.backend": 0, + "haproxy.connections.frontend": 1, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "http-webservices", + "haproxy.http.request.captured_cookie": "-", + "haproxy.http.request.raw_request_line": "GET / HTTP/1.1", + "haproxy.http.request.time_active_ms": 0, + "haproxy.http.request.time_wait_ms": 0, + "haproxy.http.request.time_wait_without_data_ms": -1, + "haproxy.http.response.captured_cookie": "-", + "haproxy.http.response.status_code": 503, + "haproxy.server_name": "", + "haproxy.server_queue": 0, + "haproxy.termination_state": "SC--", + "haproxy.total_waiting_time_ms": -1, + "input.type": "log", + "log.offset": 0, + "process.name": "haproxy", + "process.pid": 19312, + "source.ip": "127.0.0.1", + "source.port": 35982 + }, + { + "ecs.version": "1.0.0-beta2", + "event.dataset": "haproxy.log", + "event.module": "haproxy", + "fileset.name": "log", + "haproxy.backend_name": "http-webservices", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 213, + "haproxy.client.ip": "127.0.0.1", + "haproxy.connection_wait_time_ms": -1, + "haproxy.connections.active": 1, + "haproxy.connections.backend": 0, + "haproxy.connections.frontend": 1, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "http-webservices", + "haproxy.http.request.captured_cookie": "-", + "haproxy.http.request.raw_request_line": "GET /foo HTTP/1.1", + "haproxy.http.request.time_active_ms": 0, + "haproxy.http.request.time_wait_ms": 0, + "haproxy.http.request.time_wait_without_data_ms": -1, + "haproxy.http.response.captured_cookie": "-", + "haproxy.http.response.status_code": 503, + "haproxy.server_name": "", + "haproxy.server_queue": 0, + "haproxy.termination_state": "SC--", + "haproxy.total_waiting_time_ms": -1, + "input.type": "log", + "log.offset": 186, + "process.name": "haproxy", + "process.pid": 29785, + "source.ip": "127.0.0.1", + "source.port": 43738 + }, + { + "ecs.version": "1.0.0-beta2", + "event.dataset": "haproxy.log", + "event.module": "haproxy", + "fileset.name": "log", + "haproxy.backend_name": "http-webservices", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 213, + "haproxy.client.ip": "127.0.0.1", + "haproxy.connection_wait_time_ms": -1, + "haproxy.connections.active": 1, + "haproxy.connections.backend": 0, + "haproxy.connections.frontend": 1, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "http-webservices", + "haproxy.http.request.captured_cookie": "-", + "haproxy.http.request.captured_headers": [ + "localhost:8888" + ], + "haproxy.http.request.raw_request_line": "GET /foo HTTP/1.1", + "haproxy.http.request.time_active_ms": 0, + "haproxy.http.request.time_wait_ms": 0, + "haproxy.http.request.time_wait_without_data_ms": -1, + "haproxy.http.response.captured_cookie": "-", + "haproxy.http.response.captured_headers": [], + "haproxy.http.response.status_code": 503, + "haproxy.server_name": "", + "haproxy.server_queue": 0, + "haproxy.termination_state": "SC--", + "haproxy.total_waiting_time_ms": -1, + "input.type": "log", + "log.offset": 394, + "process.name": "haproxy", + "process.pid": 7873, + "source.ip": "127.0.0.1", + "source.port": 44542 + } +] \ No newline at end of file