[Filebeat] Allow cef and checkpoint modules to override network direc…

…tionality based off of zones (elastic#23066) * [Filebeat] Allow cef and checkpoint modules to override network directionality based off of zones * Remove _temp_ * Add changelog entry * run mage update and add variable reference * Don't override categorization if no zone set * Update cef pipeline (cherry picked from commit f52e452)
andrewstucki · Dec 10, 2020 · 5d4665f · 5d4665f
1 parent 62467ea
commit 5d4665f
Show file tree

Hide file tree

Showing 13 changed files with 206 additions and 0 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -499,6 +499,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add logic for external network.direction in sophos xg fileset {pull}22973[22973]
 - Add top_level_domain enrichment for suricata/eve fileset. {pull}23046[23046]
 - Add top_level_domain enrichment for zeek/dns fileset. {pull}23046[23046]
+- Add `network.direction` to netflow/log fileset. {pull}23052[23052]
+- Allow cef and checkpoint modules to override network directionality based off of zones {pull}23066[23066]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -461,6 +461,14 @@ filebeat.modules:
       syslog_host: localhost
       syslog_port: 9003
 
+      # Set internal security zones. used to override parsed network.direction
+      # based on zone egress and ingress
+      #var.internal_zones: [ "Internal" ]
+
+      # Set external security zones. used to override parsed network.direction
+      # based on zone egress and ingress
+      #var.external_zones: [ "External" ]
+
 #------------------------------ Checkpoint Module ------------------------------
 - module: checkpoint
   firewall:
@@ -476,6 +484,14 @@ filebeat.modules:
     # The UDP port to listen for syslog traffic. Defaults to 9001.
     #var.syslog_port: 9001
 
+    # Set internal security zones. used to override parsed network.direction
+    # based on zone egress and ingress
+    #var.internal_zones: [ "Internal" ]
+
+    # Set external security zones. used to override parsed network.direction
+    # based on zone egress and ingress
+    #var.external_zones: [ "External" ]
+
 #-------------------------------- Cisco Module --------------------------------
 - module: cisco
   asa:

diff --git a/x-pack/filebeat/module/cef/_meta/config.yml b/x-pack/filebeat/module/cef/_meta/config.yml
@@ -4,3 +4,11 @@
     var:
       syslog_host: localhost
       syslog_port: 9003
+
+      # Set internal security zones. used to override parsed network.direction
+      # based on zone egress and ingress
+      #var.internal_zones: [ "Internal" ]
+
+      # Set external security zones. used to override parsed network.direction
+      # based on zone egress and ingress
+      #var.external_zones: [ "External" ]
diff --git a/x-pack/filebeat/module/cef/log/config/input.yml b/x-pack/filebeat/module/cef/log/config/input.yml
@@ -29,3 +29,17 @@ processors:
       target: ''
       fields:
         ecs.version: 1.7.0
+
+{{ if .external_zones }}
+  - add_fields:
+      target: _temp_
+      fields:
+        external_zones: {{ .external_zones | tojson }}
+{{ end }}
+
+{{ if .internal_zones }}
+  - add_fields:
+      target: _temp_
+      fields:
+        internal_zones: {{ .internal_zones | tojson }}
+{{ end }}
diff --git a/x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml b/x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml
@@ -337,3 +337,64 @@ processors:
       field: event.category
       value: intrusion_detection
       if: 'ctx.event?.category != "malware" && (ctx.checkpoint?.protection_type != null || ctx.cef.extensions?.flexString2Label == "Attack Information")'
+
+  # Handle zone-based network directionality
+  - set:
+      field: network.direction
+      value: inbound
+      if: >
+        ctx?._temp_?.external_zones != null &&
+        ctx?._temp_?.internal_zones != null &&
+        ctx?.observer?.ingress?.zone != null &&
+        ctx?.observer?.egress?.zone != null &&
+        ctx._temp_.external_zones.contains(ctx.observer.ingress.zone) &&
+        ctx._temp_.internal_zones.contains(ctx.observer.egress.zone)
+  - set:
+      field: network.direction
+      value: outbound
+      if: >
+        ctx?._temp_?.external_zones != null &&
+        ctx?._temp_?.internal_zones != null &&
+        ctx?.observer?.ingress?.zone != null &&
+        ctx?.observer?.egress?.zone != null &&
+        ctx._temp_.external_zones.contains(ctx.observer.egress.zone) &&
+        ctx._temp_.internal_zones.contains(ctx.observer.ingress.zone)
+  - set:
+      field: network.direction
+      value: internal
+      if: >
+        ctx?._temp_?.external_zones != null &&
+        ctx?._temp_?.internal_zones != null &&
+        ctx?.observer?.ingress?.zone != null &&
+        ctx?.observer?.egress?.zone != null &&
+        ctx._temp_.internal_zones.contains(ctx.observer.egress.zone) &&
+        ctx._temp_.internal_zones.contains(ctx.observer.ingress.zone)
+  - set:
+      field: network.direction
+      value: external
+      if: >
+        ctx?._temp_?.external_zones != null &&
+        ctx?._temp_?.internal_zones != null &&
+        ctx?.observer?.ingress?.zone != null &&
+        ctx?.observer?.egress?.zone != null &&
+        ctx._temp_.external_zones.contains(ctx.observer.egress.zone) &&
+        ctx._temp_.external_zones.contains(ctx.observer.ingress.zone)
+  - set:
+      field: network.direction
+      value: unknown
+      if: >
+        ctx?._temp_?.external_zones != null &&
+        ctx?._temp_?.internal_zones != null &&
+        ctx?.observer?.ingress?.zone != null &&
+        ctx?.observer?.egress?.zone != null &&
+        (
+          (
+            !ctx._temp_.external_zones.contains(ctx.observer.egress.zone) &&
+            !ctx._temp_.internal_zones.contains(ctx.observer.egress.zone)
+          ) ||
+          (
+            !ctx._temp_.external_zones.contains(ctx.observer.ingress.zone) &&
+            !ctx._temp_.internal_zones.contains(ctx.observer.ingress.zone)
+          )
+        )
+
diff --git a/x-pack/filebeat/module/cef/log/ingest/pipeline.yml b/x-pack/filebeat/module/cef/log/ingest/pipeline.yml
@@ -87,6 +87,10 @@ processors:
   - pipeline:
         name: '{< IngestPipeline "cp-pipeline" >}'
         if: "ctx.cef?.device?.vendor == 'Check Point'"
+  - remove:
+      field:
+        - _temp_
+      ignore_missing: true
 on_failure:
   - set:
         field: error.message

diff --git a/x-pack/filebeat/module/cef/log/manifest.yml b/x-pack/filebeat/module/cef/log/manifest.yml
@@ -12,6 +12,8 @@ var:
     default: 9003
   - name: input
     default: syslog
+  - name: internal_zones
+  - name: external_zones
 
 ingest_pipeline:
   - ingest/pipeline.yml

diff --git a/x-pack/filebeat/module/checkpoint/_meta/config.yml b/x-pack/filebeat/module/checkpoint/_meta/config.yml
@@ -11,3 +11,11 @@
 
     # The UDP port to listen for syslog traffic. Defaults to 9001.
     #var.syslog_port: 9001
+
+    # Set internal security zones. used to override parsed network.direction
+    # based on zone egress and ingress
+    #var.internal_zones: [ "Internal" ]
+
+    # Set external security zones. used to override parsed network.direction
+    # based on zone egress and ingress
+    #var.external_zones: [ "External" ]
diff --git a/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml b/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml
@@ -29,3 +29,16 @@ processors:
       target: ''
       fields:
         ecs.version: 1.7.0
+{{ if .external_zones }}
+  - add_fields:
+      target: _temp_
+      fields:
+        external_zones: {{ .external_zones | tojson }}
+{{ end }}
+
+{{ if .internal_zones }}
+  - add_fields:
+      target: _temp_
+      fields:
+        internal_zones: {{ .internal_zones | tojson }}
+{{ end }}
diff --git a/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml
@@ -781,6 +781,65 @@ processors:
     field: destination.as.organization_name
     target_field: destination.as.organization.name
     ignore_missing: true
+# Handle zone-based network directionality
+- set:
+    field: network.direction
+    value: inbound
+    if: >
+      ctx?._temp_?.external_zones != null &&
+      ctx?._temp_?.internal_zones != null &&
+      ctx?.observer?.ingress?.zone != null &&
+      ctx?.observer?.egress?.zone != null &&
+      ctx._temp_.external_zones.contains(ctx.observer.ingress.zone) &&
+      ctx._temp_.internal_zones.contains(ctx.observer.egress.zone)
+- set:
+    field: network.direction
+    value: outbound
+    if: >
+      ctx?._temp_?.external_zones != null &&
+      ctx?._temp_?.internal_zones != null &&
+      ctx?.observer?.ingress?.zone != null &&
+      ctx?.observer?.egress?.zone != null &&
+      ctx._temp_.external_zones.contains(ctx.observer.egress.zone) &&
+      ctx._temp_.internal_zones.contains(ctx.observer.ingress.zone)
+- set:
+    field: network.direction
+    value: internal
+    if: >
+      ctx?._temp_?.external_zones != null &&
+      ctx?._temp_?.internal_zones != null &&
+      ctx?.observer?.ingress?.zone != null &&
+      ctx?.observer?.egress?.zone != null &&
+      ctx._temp_.internal_zones.contains(ctx.observer.egress.zone) &&
+      ctx._temp_.internal_zones.contains(ctx.observer.ingress.zone)
+- set:
+    field: network.direction
+    value: external
+    if: >
+      ctx?._temp_?.external_zones != null &&
+      ctx?._temp_?.internal_zones != null &&
+      ctx?.observer?.ingress?.zone != null &&
+      ctx?.observer?.egress?.zone != null &&
+      ctx._temp_.external_zones.contains(ctx.observer.egress.zone) &&
+      ctx._temp_.external_zones.contains(ctx.observer.ingress.zone)
+- set:
+    field: network.direction
+    value: unknown
+    if: >
+      ctx?._temp_?.external_zones != null &&
+      ctx?._temp_?.internal_zones != null &&
+      ctx?.observer?.ingress?.zone != null &&
+      ctx?.observer?.egress?.zone != null &&
+      (
+        (
+          !ctx._temp_.external_zones.contains(ctx.observer.egress.zone) &&
+          !ctx._temp_.internal_zones.contains(ctx.observer.egress.zone)
+        ) ||
+        (
+          !ctx._temp_.external_zones.contains(ctx.observer.ingress.zone) &&
+          !ctx._temp_.internal_zones.contains(ctx.observer.ingress.zone)
+        )
+      )
 - remove:
     field:
     - checkpoint.client_outbound_packets
@@ -801,6 +860,7 @@ processors:
     - checkpoint.uid
     - checkpoint.time
     - syslog5424_ts
+    - _temp_
     ignore_missing: true
 on_failure:
 - set:

diff --git a/x-pack/filebeat/module/checkpoint/firewall/manifest.yml b/x-pack/filebeat/module/checkpoint/firewall/manifest.yml
@@ -10,6 +10,8 @@ var:
   - name: input
     default: syslog
   - name: ssl
+  - name: internal_zones
+  - name: external_zones
 
 ingest_pipeline:
   - ingest/pipeline.yml

diff --git a/x-pack/filebeat/modules.d/cef.yml.disabled b/x-pack/filebeat/modules.d/cef.yml.disabled
@@ -7,3 +7,11 @@
     var:
       syslog_host: localhost
       syslog_port: 9003
+
+      # Set internal security zones. used to override parsed network.direction
+      # based on zone egress and ingress
+      #var.internal_zones: [ "Internal" ]
+
+      # Set external security zones. used to override parsed network.direction
+      # based on zone egress and ingress
+      #var.external_zones: [ "External" ]
diff --git a/x-pack/filebeat/modules.d/checkpoint.yml.disabled b/x-pack/filebeat/modules.d/checkpoint.yml.disabled
@@ -14,3 +14,11 @@
 
     # The UDP port to listen for syslog traffic. Defaults to 9001.
     #var.syslog_port: 9001
+
+    # Set internal security zones. used to override parsed network.direction
+    # based on zone egress and ingress
+    #var.internal_zones: [ "Internal" ]
+
+    # Set external security zones. used to override parsed network.direction
+    # based on zone egress and ingress
+    #var.external_zones: [ "External" ]