elastic#25061: Add ZooKeeper Module

CHANGELOG.next.asciidoc

@@ -852,6 +852,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `awsfargate` module to collect container logs from Amazon ECS on Fargate. {pull}25041[25041]
 - New module `cyberarkpas` for CyberArk Privileged Access Security audit logs. {pull}24803[24803]
 - Add `uri_parts` processor to Apache, Nginx, IIS, Traefik, S3Access, Cisco, F5, Fortinet, Google Workspace, Imperva, Microsoft, Netscout, O365, Sophos, Squid, Suricata, Zeek, Zia, Zoom, and ZScaler modules ingest pipelines. {issue}19088[19088] {pull}24699[24699]
+- Add ZooKeeper Module with fileset for audit logs {issue}25061[25061] {pull}25128[25128]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -91,6 +91,7 @@ grouped in the following categories:
 * <<exported-fields-tomcat>>
 * <<exported-fields-traefik>>
 * <<exported-fields-zeek>>
+* <<exported-fields-zookeeper>>
 * <<exported-fields-zoom>>
 * <<exported-fields-zscaler>>
 
@@ -163715,6 +163716,76 @@ type: boolean
 
 --
 
+[[exported-fields-zookeeper]]
+== ZooKeeper fields
+
+ZooKeeper Module
+
+
+
+[float]
+=== zookeeper
+
+
+
+
+[float]
+=== audit
+
+ZooKeeper Audit logs.
+
+
+
+*`zookeeper.audit.session`*::
++
+--
+Client session id
+
+
+type: keyword
+
+--
+
+*`zookeeper.audit.znode`*::
++
+--
+Path of the znode
+
+
+type: keyword
+
+--
+
+*`zookeeper.audit.znode_type`*::
++
+--
+Type of znode in case of creation operation
+
+
+type: keyword
+
+--
+
+*`zookeeper.audit.acl`*::
++
+--
+String representation of znode ACL like cdrwa(create, delete,read, write, admin). This is logged only for setAcl operation
+
+
+type: keyword
+
+--
+
+*`zookeeper.audit.result`*::
++
+--
+Result of the operation. Possible values are (success/failure/invoked). Result "invoked" is used for serverStop operation because stop is logged before ensuring that server actually stopped.
+
+
+type: keyword
+
+--
+
 [[exported-fields-zoom]]
 == Zoom fields
 

filebeat/docs/modules/zookeeper.asciidoc

@@ -0,0 +1,77 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[filebeat-module-zookeeper]]
+:modulename: zookeeper
+:has-dashboards: true
+
+== ZooKeeper module
+
+The +{modulename}+ module collects and parses the logs created by https://zookeeper.apache.org/[Apache ZooKeeper]
+
+include::../include/what-happens.asciidoc[]
+
+include::../include/gs-link.asciidoc[]
+
+[float]
+=== Compatibility
+
+The +{modulename}+ module was tested with logs from versions 3.7.0.
+
+include::../include/configuring-intro.asciidoc[]
+
+The following example shows how to set paths in the +modules.d/{modulename}.yml+
+file to override the default paths for logs:
+
+[source,yaml]
+-----
+- module: zookeeper
+  audit:
+    enabled: true
+    var.paths:
+      - "/path/to/logs/audit.log*"
+-----
+
+
+To specify the same settings at the command line, you use:
+
+[source,yaml]
+-----
+-M "zookeeper.audit.var.paths=[/path/to/logs/audit.log*]"
+-----
+
+
+//set the fileset name used in the included example
+:fileset_ex: audit
+
+include::../include/config-option-intro.asciidoc[]
+
+[float]
+==== `audit` fileset settings
+
+include::../include/var-paths.asciidoc[]
+
+include::../include/timezone-support.asciidoc[]
+
+[float]
+=== Example dashboard
+
+This module comes with a sample dashboard to see ZooKeeper Audit logs.
+
+[role="screenshot"]
+image::./images/filebeat-kafka-logs-overview.png[]
+
+:has-dashboards!:
+
+:fileset_ex!:
+
+:modulename!:
+
+
+[float]
+=== Fields
+
+For a description of each field in the module, see the
+<<exported-fields-zookeeper,exported fields>> section.
+

filebeat/docs/modules_list.asciidoc

@@ -70,6 +70,7 @@ This file is generated! See scripts/docs_collector.py
   * <<filebeat-module-tomcat>>
   * <<filebeat-module-traefik>>
   * <<filebeat-module-zeek>>
+  * <<filebeat-module-zookeeper>>
   * <<filebeat-module-zoom>>
   * <<filebeat-module-zscaler>>
 
@@ -144,5 +145,6 @@ include::modules/threatintel.asciidoc[]
 include::modules/tomcat.asciidoc[]
 include::modules/traefik.asciidoc[]
 include::modules/zeek.asciidoc[]
+include::modules/zookeeper.asciidoc[]
 include::modules/zoom.asciidoc[]
 include::modules/zscaler.asciidoc[]

filebeat/filebeat.reference.yml

@@ -403,6 +403,16 @@ filebeat.modules:
     # can be added under this section.
     #input:
 
+#------------------------------ ZooKeeper Module ------------------------------
+- module: zookeeper
+  # All logs
+  audit:
+    enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
 
 
 #=========================== Filebeat inputs =============================

filebeat/module/zookeeper/_meta/config.yml

@@ -0,0 +1,8 @@
+- module: zookeeper
+  # All logs
+  audit:
+    enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:

filebeat/module/zookeeper/_meta/docs.asciidoc

@@ -0,0 +1,64 @@
+:modulename: zookeeper
+:has-dashboards: true
+
+== ZooKeeper module
+
+The +{modulename}+ module collects and parses the logs created by https://zookeeper.apache.org/[Apache ZooKeeper]
+
+include::../include/what-happens.asciidoc[]
+
+include::../include/gs-link.asciidoc[]
+
+[float]
+=== Compatibility
+
+The +{modulename}+ module was tested with logs from versions 3.7.0.
+
+include::../include/configuring-intro.asciidoc[]
+
+The following example shows how to set paths in the +modules.d/{modulename}.yml+
+file to override the default paths for logs:
+
+[source,yaml]
+-----
+- module: zookeeper
+  audit:
+    enabled: true
+    var.paths:
+      - "/path/to/logs/audit.log*"
+-----
+
+
+To specify the same settings at the command line, you use:
+
+[source,yaml]
+-----
+-M "zookeeper.audit.var.paths=[/path/to/logs/audit.log*]"
+-----
+
+
+//set the fileset name used in the included example
+:fileset_ex: audit
+
+include::../include/config-option-intro.asciidoc[]
+
+[float]
+==== `audit` fileset settings
+
+include::../include/var-paths.asciidoc[]
+
+include::../include/timezone-support.asciidoc[]
+
+[float]
+=== Example dashboard
+
+This module comes with a sample dashboard to see ZooKeeper Audit logs.
+
+[role="screenshot"]
+image::./images/filebeat-kafka-logs-overview.png[]
+
+:has-dashboards!:
+
+:fileset_ex!:
+
+:modulename!:

filebeat/module/zookeeper/_meta/fields.yml

@@ -0,0 +1,9 @@
+- key: zookeeper
+  title: "ZooKeeper"
+  description: >
+    ZooKeeper Module
+  fields:
+    - name: zookeeper
+      type: group
+      description: >
+      fields:

filebeat/module/zookeeper/audit/_meta/fields.yml

@@ -0,0 +1,37 @@
+- name: audit
+  type: group
+  description: >
+    ZooKeeper Audit logs.
+  fields:
+    - name: session
+      type: keyword
+      description: >
+        Client session id
+    - name: znode
+      type: keyword
+      description: >
+        Path of the znode
+    - name: znode_type
+      type: keyword
+      description: >
+        Type of znode in case of creation operation
+    - name: acl
+      type: keyword
+      description: >
+        String representation of znode ACL like cdrwa(create, delete,read, write, admin). This is logged only for setAcl operation
+    - name: result
+      type: keyword
+      description: >
+        Result of the operation. Possible values are (success/failure/invoked). Result "invoked" is used for serverStop operation because stop is logged before ensuring that server actually stopped.
+    - name: user
+      type: keyword
+      description: >
+        XX
+    - name: ip
+      type: keyword
+      description: >
+        XX
+    - name: operation
+      type: keyword
+      description: >
+        XX

filebeat/module/zookeeper/audit/config/audit.yml

@@ -0,0 +1,12 @@
+type: log
+paths:
+{{ range $i, $path := .paths }}
+ - {{$path}}
+{{ end }}
+exclude_files: [".gz$"]
+processors:
+  - add_locale: ~
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.9.0

filebeat/module/zookeeper/audit/ingest/pipeline.yml

@@ -0,0 +1,45 @@
+description: Pipeline for parsing ZooKeeper audit messages
+processors:
+- set:
+    field: event.ingested
+    value: '{{_ingest.timestamp}}'
+- kv:
+    field: message
+    field_split: "\\s+"
+    value_split: "="
+    target_field: zookeeper.audit
+- rename:
+    field: message
+    target_field: log.original
+    ignore_missing: true
+- rename:
+    field: zookeeper.audit.operation
+    target_field: event.action
+    ignore_missing: true
+- set:
+    field: event.outcome
+    value: "{{zookeeper.audit.result}}"
+    ignore_empty_value: true
+    if: '["success","failure"].contains(ctx.zookeeper?.audit?.result)'
+- rename:
+    field: zookeeper.audit.ip
+    target_field: client.address
+    ignore_missing: true
+- convert:
+    field: client.address
+    target_field: client.ip
+    type: ip
+    ignore_missing: true
+- split:
+    field: zookeeper.audit.user
+    separator: ","
+    ignore_missing: true
+- rename:
+    field: zookeeper.audit.user
+    target_field: user.id
+    ignore_missing: true
+
+on_failure:
+- set:
+    field: error.log
+    value: '{{ _ingest.on_failure_message }}'