From 4460501ee3299ac144a16f6e78b1d99f5e16357e Mon Sep 17 00:00:00 2001 From: "Lee E. Hinman" Date: Mon, 24 Feb 2020 15:55:10 -0600 Subject: [PATCH] Improve ECS categorization field mapping in icinga - event.kind - event.type Closes #16164 --- CHANGELOG.next.asciidoc | 1 + .../module/icinga/debug/ingest/pipeline.json | 36 ----------------- .../module/icinga/debug/ingest/pipeline.yml | 39 +++++++++++++++++++ filebeat/module/icinga/debug/manifest.yml | 2 +- .../icinga/debug/test/test.log-expected.json | 6 +++ .../module/icinga/main/ingest/pipeline.json | 36 ----------------- .../module/icinga/main/ingest/pipeline.yml | 39 +++++++++++++++++++ filebeat/module/icinga/main/manifest.yml | 2 +- .../icinga/main/test/test.log-expected.json | 6 +++ .../icinga/startup/ingest/pipeline.json | 21 ---------- .../module/icinga/startup/ingest/pipeline.yml | 29 ++++++++++++++ filebeat/module/icinga/startup/manifest.yml | 2 +- .../startup/test/test.log-expected.json | 4 ++ 13 files changed, 127 insertions(+), 96 deletions(-) delete mode 100644 filebeat/module/icinga/debug/ingest/pipeline.json create mode 100644 filebeat/module/icinga/debug/ingest/pipeline.yml delete mode 100644 filebeat/module/icinga/main/ingest/pipeline.json create mode 100644 filebeat/module/icinga/main/ingest/pipeline.yml delete mode 100644 filebeat/module/icinga/startup/ingest/pipeline.json create mode 100644 filebeat/module/icinga/startup/ingest/pipeline.yml diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5ad027d18be..ad31342baa9 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -134,6 +134,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve ECS categorization, container & process field mappings in auditd module. {issue}16153[16153] {pull}16280[16280] - Improve ECS field mappings in aws module. {issue}16154[16154] {pull}16307[16307] - Improve ECS categorization field mappings in googlecloud module. {issue}16030[16030] {pull}16500[16500] +- Improve ECS categorization field mapping in icinga module. {issue}16164[16164] {pull}16533[16533] *Heartbeat* diff --git a/filebeat/module/icinga/debug/ingest/pipeline.json b/filebeat/module/icinga/debug/ingest/pipeline.json deleted file mode 100644 index 65abfffca9f..00000000000 --- a/filebeat/module/icinga/debug/ingest/pipeline.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "description": "Pipeline for parsing icinga debug logs", - "processors": [{ - "grok": { - "field": "message", - "patterns":[ - "\\[%{TIMESTAMP:icinga.debug.timestamp}\\] %{WORD:log.level}/%{WORD:icinga.debug.facility}: %{GREEDYMULTILINE:message}" - ], - "ignore_missing": true, - "pattern_definitions": { - "TIMESTAMP": "%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{ISO8601_TIMEZONE}", - "GREEDYMULTILINE": "(.|\n)*" - } - } - }, - { - "date": { - "field": "icinga.debug.timestamp", - "target_field": "@timestamp", - "formats": ["yyyy-MM-dd HH:mm:ss Z"], - "ignore_failure": true - } - }, - { - "remove": { - "field": "icinga.debug.timestamp" - } - - }], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/filebeat/module/icinga/debug/ingest/pipeline.yml b/filebeat/module/icinga/debug/ingest/pipeline.yml new file mode 100644 index 00000000000..ee25b38e90e --- /dev/null +++ b/filebeat/module/icinga/debug/ingest/pipeline.yml @@ -0,0 +1,39 @@ +description: Pipeline for parsing icinga debug logs +processors: +- grok: + field: message + patterns: + - '\[%{TIMESTAMP:icinga.debug.timestamp}\] %{WORD:log.level}/%{WORD:icinga.debug.facility}: + %{GREEDYMULTILINE:message}' + ignore_missing: true + pattern_definitions: + TIMESTAMP: '%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{ISO8601_TIMEZONE}' + GREEDYMULTILINE: |- + (.| + )* +- date: + field: icinga.debug.timestamp + target_field: '@timestamp' + formats: + - yyyy-MM-dd HH:mm:ss Z + ignore_failure: true +- remove: + field: icinga.debug.timestamp +- set: + field: event.kind + value: event +- script: + lang: painless + source: >- + def errorLevels = ["warning", "critical"]; + if (ctx?.log?.level != null) { + if (errorLevels.contains(ctx.log.level)) { + ctx.event.type = "error"; + } else { + ctx.event.type = "info"; + } + } +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/icinga/debug/manifest.yml b/filebeat/module/icinga/debug/manifest.yml index 50881b6d36f..a42eed6c725 100644 --- a/filebeat/module/icinga/debug/manifest.yml +++ b/filebeat/module/icinga/debug/manifest.yml @@ -9,5 +9,5 @@ var: os.windows: - c:/programdata/icinga2/var/log/icinga2/debug.log* -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/debug.yml diff --git a/filebeat/module/icinga/debug/test/test.log-expected.json b/filebeat/module/icinga/debug/test/test.log-expected.json index bca53f276a3..dea469224b5 100644 --- a/filebeat/module/icinga/debug/test/test.log-expected.json +++ b/filebeat/module/icinga/debug/test/test.log-expected.json @@ -2,7 +2,9 @@ { "@timestamp": "2017-04-04T11:43:09.000Z", "event.dataset": "icinga.debug", + "event.kind": "event", "event.module": "icinga", + "event.type": "info", "fileset.name": "debug", "icinga.debug.facility": "GraphiteWriter", "input.type": "log", @@ -14,7 +16,9 @@ { "@timestamp": "2017-04-04T11:43:09.000Z", "event.dataset": "icinga.debug", + "event.kind": "event", "event.module": "icinga", + "event.type": "info", "fileset.name": "debug", "icinga.debug.facility": "IdoMysqlConnection", "input.type": "log", @@ -26,7 +30,9 @@ { "@timestamp": "2017-04-04T11:43:11.000Z", "event.dataset": "icinga.debug", + "event.kind": "event", "event.module": "icinga", + "event.type": "info", "fileset.name": "debug", "icinga.debug.facility": "Process", "input.type": "log", diff --git a/filebeat/module/icinga/main/ingest/pipeline.json b/filebeat/module/icinga/main/ingest/pipeline.json deleted file mode 100644 index b11b0cabfbf..00000000000 --- a/filebeat/module/icinga/main/ingest/pipeline.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "description": "Pipeline for parsing icinga main logs", - "processors": [{ - "grok": { - "field": "message", - "patterns":[ - "\\[%{TIMESTAMP:icinga.main.timestamp}\\] %{WORD:log.level}/%{WORD:icinga.main.facility}: %{GREEDYMULTILINE:message}" - ], - "ignore_missing": true, - "pattern_definitions": { - "TIMESTAMP": "%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{ISO8601_TIMEZONE}", - "GREEDYMULTILINE": "(.|\n)*" - } - } - }, - { - "date": { - "field": "icinga.main.timestamp", - "target_field": "@timestamp", - "formats": ["yyyy-MM-dd HH:mm:ss Z"], - "ignore_failure": true - } - }, - { - "remove": { - "field": "icinga.main.timestamp" - } - - }], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/filebeat/module/icinga/main/ingest/pipeline.yml b/filebeat/module/icinga/main/ingest/pipeline.yml new file mode 100644 index 00000000000..5db480e07ab --- /dev/null +++ b/filebeat/module/icinga/main/ingest/pipeline.yml @@ -0,0 +1,39 @@ +description: Pipeline for parsing icinga main logs +processors: +- grok: + field: message + patterns: + - '\[%{TIMESTAMP:icinga.main.timestamp}\] %{WORD:log.level}/%{WORD:icinga.main.facility}: + %{GREEDYMULTILINE:message}' + ignore_missing: true + pattern_definitions: + TIMESTAMP: '%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{ISO8601_TIMEZONE}' + GREEDYMULTILINE: |- + (.| + )* +- date: + field: icinga.main.timestamp + target_field: '@timestamp' + formats: + - yyyy-MM-dd HH:mm:ss Z + ignore_failure: true +- remove: + field: icinga.main.timestamp +- set: + field: event.kind + value: event +- script: + lang: painless + source: >- + def errorLevels = ["warning", "critical"]; + if (ctx?.log?.level != null) { + if (errorLevels.contains(ctx.log.level)) { + ctx.event.type = "error"; + } else { + ctx.event.type = "info"; + } + } +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/icinga/main/manifest.yml b/filebeat/module/icinga/main/manifest.yml index 5ab760dff61..55e5c38e69b 100644 --- a/filebeat/module/icinga/main/manifest.yml +++ b/filebeat/module/icinga/main/manifest.yml @@ -9,5 +9,5 @@ var: os.windows: - c:/programdata/icinga2/var/log/icinga2/icinga2.log* -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/main.yml diff --git a/filebeat/module/icinga/main/test/test.log-expected.json b/filebeat/module/icinga/main/test/test.log-expected.json index 2abcb8dfa0f..033457dd19e 100644 --- a/filebeat/module/icinga/main/test/test.log-expected.json +++ b/filebeat/module/icinga/main/test/test.log-expected.json @@ -2,7 +2,9 @@ { "@timestamp": "2017-04-04T09:16:34.000Z", "event.dataset": "icinga.main", + "event.kind": "event", "event.module": "icinga", + "event.type": "info", "fileset.name": "main", "icinga.main.facility": "Notification", "input.type": "log", @@ -14,7 +16,9 @@ { "@timestamp": "2017-04-04T09:16:34.000Z", "event.dataset": "icinga.main", + "event.kind": "event", "event.module": "icinga", + "event.type": "error", "fileset.name": "main", "icinga.main.facility": "PluginNotificationTask", "input.type": "log", @@ -29,7 +33,9 @@ { "@timestamp": "2017-04-04T09:16:48.000Z", "event.dataset": "icinga.main", + "event.kind": "event", "event.module": "icinga", + "event.type": "info", "fileset.name": "main", "icinga.main.facility": "IdoMysqlConnection", "input.type": "log", diff --git a/filebeat/module/icinga/startup/ingest/pipeline.json b/filebeat/module/icinga/startup/ingest/pipeline.json deleted file mode 100644 index 9528010c716..00000000000 --- a/filebeat/module/icinga/startup/ingest/pipeline.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "description": "Pipeline for parsing icinga startup logs", - "processors": [{ - "grok": { - "field": "message", - "patterns":[ - "%{WORD:log.level}/%{WORD:icinga.startup.facility}: %{GREEDYMULTILINE:message}" - ], - "ignore_missing": true, - "pattern_definitions": { - "GREEDYMULTILINE": "(.|\n)*" - } - } - }], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/filebeat/module/icinga/startup/ingest/pipeline.yml b/filebeat/module/icinga/startup/ingest/pipeline.yml new file mode 100644 index 00000000000..61e0e6fef27 --- /dev/null +++ b/filebeat/module/icinga/startup/ingest/pipeline.yml @@ -0,0 +1,29 @@ +description: Pipeline for parsing icinga startup logs +processors: +- grok: + field: message + patterns: + - '%{WORD:log.level}/%{WORD:icinga.startup.facility}: %{GREEDYMULTILINE:message}' + ignore_missing: true + pattern_definitions: + GREEDYMULTILINE: |- + (.| + )* +- set: + field: event.kind + value: event +- script: + lang: painless + source: >- + def errorLevels = ["warning", "critical"]; + if (ctx?.log?.level != null) { + if (errorLevels.contains(ctx.log.level)) { + ctx.event.type = "error"; + } else { + ctx.event.type = "info"; + } + } +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/icinga/startup/manifest.yml b/filebeat/module/icinga/startup/manifest.yml index b749ff46be1..c60a22f4dd4 100644 --- a/filebeat/module/icinga/startup/manifest.yml +++ b/filebeat/module/icinga/startup/manifest.yml @@ -9,5 +9,5 @@ var: os.windows: - c:/programdata/icinga2/var/log/icinga2/startup.log -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/startup.yml diff --git a/filebeat/module/icinga/startup/test/test.log-expected.json b/filebeat/module/icinga/startup/test/test.log-expected.json index 4f7b859a1de..fa620293b5c 100644 --- a/filebeat/module/icinga/startup/test/test.log-expected.json +++ b/filebeat/module/icinga/startup/test/test.log-expected.json @@ -1,7 +1,9 @@ [ { "event.dataset": "icinga.startup", + "event.kind": "event", "event.module": "icinga", + "event.type": "info", "fileset.name": "startup", "icinga.startup.facility": "cli", "input.type": "log", @@ -12,7 +14,9 @@ }, { "event.dataset": "icinga.startup", + "event.kind": "event", "event.module": "icinga", + "event.type": "info", "fileset.name": "startup", "icinga.startup.facility": "cli", "input.type": "log",