From b85a764a55ae97bfe11d9e7ad37259b41115a742 Mon Sep 17 00:00:00 2001 From: Lee Hinman <57081003+leehinman@users.noreply.github.com> Date: Thu, 25 Jun 2020 09:22:26 -0500 Subject: [PATCH] [Filebeat] Explicitly set ECS version in Filebeat modules. (#19198) (#19223) * Explicitly set ECS version in Filebeat modules. - Add test to check if ecs.version is set - add_fields to azure/activitylogs - add_fields to azure/auditlogs - add_fields to azure/signinlogs - add_fields to checkpoint/firewall - add_fields to crowdstrike/falcon - add_fields to fortinet/firewall - add_fields to traefik/access Relates #17688 (cherry picked from commit 81b0c3adbbde1f31fd0107d9d35d21b0486bc482) --- CHANGELOG.next.asciidoc | 1 + .../traefik/access/config/traefik-access.yml | 5 +++ filebeat/tests/system/test_modules.py | 2 ++ libbeat/tests/system/beat/beat.py | 31 +++++++++++++++++++ .../activitylogs/config/azure-eventhub.yml | 5 +++ .../module/azure/activitylogs/config/file.yml | 6 ++++ .../azure/auditlogs/config/azure-eventhub.yml | 5 +++ .../module/azure/auditlogs/config/file.yml | 5 +++ .../signinlogs/config/azure-eventhub.yml | 5 +++ .../module/azure/signinlogs/config/file.yml | 5 +++ .../checkpoint/firewall/config/firewall.yml | 4 +++ .../crowdstrike/falcon/config/falcon.yml | 4 +++ .../fortinet/firewall/config/firewall.yml | 4 +++ 13 files changed, 82 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ee8433325a1..1c5ae2ac160 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -480,6 +480,7 @@ field. You can revert this change by configuring tags for the module and omittin - Improved performance of PANW sample dashboards. {issue}19031[19031] {pull}19032[19032] - Add support for v1 consumer API in Cloud Foundry input, use it by default. {pull}19125[19125] - Add new mode to multiline reader to aggregate constant number of lines {pull}18352[18352] +- Explicitly set ECS version in all Filebeat modules. {pull}19198[19198] *Heartbeat* diff --git a/filebeat/module/traefik/access/config/traefik-access.yml b/filebeat/module/traefik/access/config/traefik-access.yml index 0afd17317d4..183de629867 100644 --- a/filebeat/module/traefik/access/config/traefik-access.yml +++ b/filebeat/module/traefik/access/config/traefik-access.yml @@ -4,3 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index 0479868ccc0..9cdaf78b2e8 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -108,6 +108,8 @@ def test_fileset_file(self, module, fileset, test_file): def run_on_file(self, module, fileset, test_file, cfgfile): print("Testing {}/{} on {}".format(module, fileset, test_file)) + self.assert_explicit_ecs_version_set(module, fileset) + try: self.es.indices.delete(index=self.index_name) except: diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 9d0bf471635..23a0bdbb4ba 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -11,6 +11,7 @@ import yaml import hashlib import re +import glob from datetime import datetime, timedelta from .compose import ComposeMixin @@ -722,3 +723,33 @@ def get_beat_version(self): proc.wait() return self.get_log_lines(logfile="version")[0].split()[2] + + def assert_explicit_ecs_version_set(self, module, fileset): + """ + Assert that the module explicitly sets the ECS version field. + """ + def get_config_paths(modules_path, module, fileset): + pathname = os.path.abspath(modules_path + + "/" + + module + + "/" + + fileset + + "/" + + "config/*.yml") + return glob.glob(pathname) + + def is_ecs_version_set(path): + # parsing the yml file would be better but go templates in + # the file make that difficult + with open(path) as fhandle: + for line in fhandle: + if re.search("ecs\.version", line): + return True + return False + + errors = [] + for cfg_path in get_config_paths(self.modules_path, module, fileset): + if not is_ecs_version_set(cfg_path): + errors.append("{}".format(cfg_path)) + if len(errors) > 0: + raise Exception("{}/{} ecs.version not explicitly set in:\n{}".format(module, fileset, '\n'.join(errors))) diff --git a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml index 9f24f85e3eb..8c4c42d60cf 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml @@ -8,3 +8,8 @@ resource_manager_endpoint: {{ .resource_manager_endpoint }} tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/azure/activitylogs/config/file.yml b/x-pack/filebeat/module/azure/activitylogs/config/file.yml index 456cc5dce7c..498a7906457 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/file.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/file.yml @@ -6,3 +6,9 @@ paths: exclude_files: [".gz$"] tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml index f8b88d18a4a..239e7ef2249 100644 --- a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml @@ -7,3 +7,8 @@ storage_account_key: {{ .storage_account_key }} resource_manager_endpoint: {{ .resource_manager_endpoint }} tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/azure/auditlogs/config/file.yml b/x-pack/filebeat/module/azure/auditlogs/config/file.yml index 456cc5dce7c..937446eb523 100644 --- a/x-pack/filebeat/module/azure/auditlogs/config/file.yml +++ b/x-pack/filebeat/module/azure/auditlogs/config/file.yml @@ -6,3 +6,8 @@ paths: exclude_files: [".gz$"] tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml index f8b88d18a4a..239e7ef2249 100644 --- a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml @@ -7,3 +7,8 @@ storage_account_key: {{ .storage_account_key }} resource_manager_endpoint: {{ .resource_manager_endpoint }} tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/azure/signinlogs/config/file.yml b/x-pack/filebeat/module/azure/signinlogs/config/file.yml index 456cc5dce7c..937446eb523 100644 --- a/x-pack/filebeat/module/azure/signinlogs/config/file.yml +++ b/x-pack/filebeat/module/azure/signinlogs/config/file.yml @@ -6,3 +6,8 @@ paths: exclude_files: [".gz$"] tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml b/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml index 637a2899315..12440f8fffe 100644 --- a/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml +++ b/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml @@ -20,3 +20,7 @@ publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: - add_locale: ~ + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml b/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml index 9382953fd37..e9e9d253ca8 100644 --- a/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml +++ b/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml @@ -17,3 +17,7 @@ processors: lang: javascript id: crowdstrike_falcon file: ${path.home}/module/crowdstrike/falcon/config/pipeline.js +- add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml b/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml index 3cf11a126a0..725ad622fa0 100644 --- a/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml +++ b/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml @@ -24,3 +24,7 @@ tags: {{.tags}} processors: - add_locale: ~ + - add_fields: + target: '' + fields: + ecs.version: 1.5.0