From c86f937feaf0f5d4699c258d3ec08b63f16db677 Mon Sep 17 00:00:00 2001 From: Mario Castro Date: Fri, 1 Feb 2019 13:48:59 +0100 Subject: [PATCH] Move dissect tokenizing from Beat to ES in Traefik Filebeat Access Fileset (#10442) Traefik access Metricset were parsing incoming log lines by using a mixed of dissect processing on the Beat and a Grok processing on ES. This PR moves all parsing to ES --- CHANGELOG.next.asciidoc | 1 + filebeat/module/traefik/access/config/traefik-access.yml | 9 --------- filebeat/module/traefik/access/ingest/pipeline.json | 6 ++++++ 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 26bd7efceac..2e3aa7f784c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -72,6 +72,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Migrate Elasticsearch audit logs fields to ECS {pull}10352[10352] - Several text fields in the Logstash module are now indexed as `keyword` fields with `text` multi-fields (ECS). {pull}10417[10417] - Several text fields in the Elasticsearch module are now indexed as `keyword` fields with `text` multi-fields (ECS). {pull}10414[10414] +- Move dissect pattern for traefik.access fileset from Filbeat to Elasticsearch. {pull}10442[10442] *Heartbeat* diff --git a/filebeat/module/traefik/access/config/traefik-access.yml b/filebeat/module/traefik/access/config/traefik-access.yml index 8b640efa575..0afd17317d4 100644 --- a/filebeat/module/traefik/access/config/traefik-access.yml +++ b/filebeat/module/traefik/access/config/traefik-access.yml @@ -4,12 +4,3 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] - -processors: -- dissect: - tokenizer: '%{source.address} %{traefik.access.user_identifier} %{user.name} [%{traefik.access.time}] - "%{http.request.method} %{url.original} HTTP/%{http.version}" - %{http.response.status_code} %{traefik.access.message}' - - field: "message" - target_prefix: "" diff --git a/filebeat/module/traefik/access/ingest/pipeline.json b/filebeat/module/traefik/access/ingest/pipeline.json index ff88d43ea31..5c9a3060239 100644 --- a/filebeat/module/traefik/access/ingest/pipeline.json +++ b/filebeat/module/traefik/access/ingest/pipeline.json @@ -1,6 +1,12 @@ { "description": "Pipeline for parsing Traefik access logs. Requires the geoip and user_agent plugins.", "processors": [ + { + "dissect": { + "field": "message", + "pattern": "%{source.address} %{traefik.access.user_identifier} %{user.name} [%{traefik.access.time}] \"%{http.request.method} %{url.original} HTTP/%{http.version}\" %{http.response.status_code} %{traefik.access.message}" + } + }, { "grok": { "field": "traefik.access.message",