Added support for intel.log zeek module

CHANGELOG.next.asciidoc

@@ -401,6 +401,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `index` option to all inputs to directly set a per-input index value. {pull}14010[14010]
 - Remove beta flag for some filebeat modules. {pull}14374[14374]
 - Add attack_pattern_kql field to MISP threat indicators. {pull}14470[14470]
+- Add fileset to the Zeek module for the intel.log. {pull}14404[14404]
 
 *Heartbeat*
 - Add non-privileged icmp on linux and darwin(mac). {pull}13795[13795] {issue}11498[11498]

x-pack/filebeat/filebeat.reference.yml

@@ -627,6 +627,8 @@ filebeat.modules:
     enabled: true
   http:
     enabled: true
+  intel:
+    enabled: true    
   irc:
     enabled: true
   kerberos:

x-pack/filebeat/module/zeek/_meta/config.yml

@@ -19,6 +19,8 @@
     enabled: true
   http:
     enabled: true
+  intel:
+    enabled: true    
   irc:
     enabled: true
   kerberos:

x-pack/filebeat/module/zeek/intel/_meta/fields.yml

@@ -0,0 +1,80 @@
+- name: intel
+  type: group
+  default_field: false
+  description: >
+    Fields exported by the Zeek Intel log.
+  fields:     
+
+    - name: seen
+      type: group
+      fields:
+        - name: indicator
+          type: keyword
+          description: >
+            The intelligence indicator.
+
+        - name: indicator_type
+          type: keyword
+          description: >
+            The type of data the indicator represents.
+
+        - name: host
+          type: keyword
+          description: >
+            If the indicator type was Intel::ADDR, then this field will be present.
+
+        - name: conn
+          type: keyword
+          description: >
+            If the data was discovered within a connection, the connection record should go here to give context to the data.
+
+        - name: where
+          type: keyword
+          description: >
+            Where the data was discovered.
+
+        - name: node
+          type: keyword
+          description: >
+            The name of the node where the match was discovered.
+
+        - name: uid
+          type: keyword
+          description: >
+            If the data was discovered within a connection, the connection uid should go here to give context to the data. If the conn field is provided, this will be automatically filled out.
+
+        - name: fa_file
+          type: keyword
+          description: >
+            If the data was discovered within a file, the file record should go here to provide context to the data.
+
+        - name: fuid
+          type: keyword
+          description: >
+            If the data was discovered within a file, the file uid should go here to provide context to the data. If the file record f is provided, this will be automatically filled out.
+
+
+    - name: matched
+      type: keyword
+      description: >
+        Event to represent a match in the intelligence data from data that was seen.
+
+    - name: sources
+      type: keyword
+      description: >
+        Sources which supplied data for this match.
+
+    - name: fuid
+      type: keyword
+      description: >
+        If a file was associated with this intelligence hit, this is the uid for the file.
+
+    - name: file_mime_type
+      type: keyword
+      description: >
+        A mime type if the intelligence hit is related to a file. If the $f field is provided this will be automatically filled out.
+
+    - name: file_desc
+      type: keyword
+      description: >
+        Frequently files can be described to give a bit more context. If the $f field is provided this field will be automatically filled out.

x-pack/filebeat/module/zeek/intel/config/intel.yml

@@ -0,0 +1,33 @@
+type: log
+paths:
+{{ range $i, $path := .paths }}
+ - {{$path}}
+{{ end }}
+exclude_files: [".gz$"]
+tags: {{.tags}}
+
+json.keys_under_root: false
+
+processors:
+  - rename:
+      fields:
+        - from: "json"
+          to: "zeek.intel"
+
+        - from: "zeek.intel.uid"
+          to: "zeek.session_id"
+
+        - from: "zeek.intel.id.orig_h"
+          to: "source.address"
+
+        - from: "zeek.intel.id.orig_p"
+          to: "source.port"
+
+        - from: "zeek.intel.id.resp_h"
+          to: "destination.address"
+
+        - from: "zeek.intel.id.resp_p"
+          to: "destination.port"
+
+      ignore_missing: true
+      fail_on_error: false

x-pack/filebeat/module/zeek/intel/ingest/pipeline.json

@@ -0,0 +1,104 @@
+{
+  "description": "Pipeline for normalizing Zeek intel.log",
+  "processors": [
+    {
+      "script": {
+        "lang": "painless",
+        "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['intel']['ts'] * 1000; ctx.zeek.intel.remove('ts');"
+      }
+    },
+    {
+      "set": {
+        "field": "destination.ip",
+        "value": "{{destination.address}}",
+        "if": "ctx.destination?.address != null"
+      }
+    },
+    {
+      "set": {
+        "field": "source.ip",
+        "value": "{{source.address}}",
+        "if": "ctx.source?.address != null"
+      }
+    },
+    {
+      "set": {
+        "field": "event.id",
+        "value": "{{zeek.session_id}}",
+        "if": "ctx.zeek.session_id != null"
+      }
+    },
+    {
+      "geoip": {
+        "field": "destination.ip",
+        "target_field": "destination.geo",
+        "ignore_missing": true
+      }
+    },
+    {
+      "geoip": {
+        "field": "source.ip",
+        "target_field": "source.geo",
+        "ignore_missing": true
+      }
+    },
+    {
+      "geoip": {
+        "database_file": "GeoLite2-ASN.mmdb",
+        "field": "source.ip",
+        "target_field": "source.as",
+        "properties": [
+          "asn",
+          "organization_name"
+        ],
+        "ignore_missing": true
+      }
+    },
+    {
+      "geoip": {
+        "database_file": "GeoLite2-ASN.mmdb",
+        "field": "destination.ip",
+        "target_field": "destination.as",
+        "properties": [
+          "asn",
+          "organization_name"
+        ],
+        "ignore_missing": true
+      }
+    },
+    {
+      "rename": {
+        "field": "source.as.asn",
+        "target_field": "source.as.number",
+        "ignore_missing": true
+      }
+    },
+    {
+      "rename": {
+        "field": "source.as.organization_name",
+        "target_field": "source.as.organization.name",
+        "ignore_missing": true
+      }
+    },
+    {
+      "rename": {
+        "field": "destination.as.asn",
+        "target_field": "destination.as.number",
+        "ignore_missing": true
+      }
+    },
+    {
+      "rename": {
+        "field": "destination.as.organization_name",
+        "target_field": "destination.as.organization.name",
+        "ignore_missing": true
+      }
+    }
+  ],
+  "on_failure" : [{ 
+    "set" : { 
+      "field" : "error.message", 
+      "value" : "{{ _ingest.on_failure_message }}" 
+    } 
+  }]
+}

x-pack/filebeat/module/zeek/intel/manifest.yml

@@ -0,0 +1,21 @@
+module_version: 1.0
+
+var:
+  - name: paths
+    default:
+      - /var/log/bro/current/intel.log
+    os.linux:
+      - /var/log/bro/current/intel.log
+    os.darwin:
+      - /usr/local/var/logs/current/intel.log
+  - name: tags
+    default: [zeek.intel]
+  - name: community_id
+    default: true
+
+ingest_pipeline: ingest/pipeline.json
+input: config/intel.yml
+
+requires.processors:
+- name: geoip
+  plugin: ingest-geoip

x-pack/filebeat/module/zeek/intel/test/intel-json.log

@@ -0,0 +1 @@
+{"ts":1573030980.989353,"uid":"Ctefoj1tgOPt4D0EK2","id.orig_h":"192.168.1.1","id.orig_p":37598,"id.resp_h":"198.41.0.4","id.resp_p":53,"seen.indicator":"198.41.0.4","seen.indicator_type":"Intel::ADDR","seen.where":"Conn::IN_RESP","seen.node":"worker-1-2","matched":["Intel::ADDR"],"sources":["ETPRO Rep: AbusedTLD Score: 127"]}

x-pack/filebeat/module/zeek/intel/test/intel-json.log-expected.json

@@ -0,0 +1,38 @@
+[
+    {
+        "@timestamp": 1573030980000,
+        "destination.address": "198.41.0.4",
+        "destination.as.number": 20431,
+        "destination.as.organization.name": "VeriSign Global Registry Services",
+        "destination.geo.continent_name": "North America",
+        "destination.geo.country_iso_code": "US",
+        "destination.geo.location.lat": 37.751,
+        "destination.geo.location.lon": -97.822,
+        "destination.ip": "198.41.0.4",
+        "destination.port": 53,
+        "event.dataset": "zeek.intel",
+        "event.id": "Ctefoj1tgOPt4D0EK2",
+        "event.module": "zeek",
+        "fileset.name": "intel",
+        "input.type": "log",
+        "log.offset": 0,
+        "service.type": "zeek",
+        "source.address": "192.168.1.1",
+        "source.ip": "192.168.1.1",
+        "source.port": 37598,
+        "tags": [
+            "zeek.intel"
+        ],
+        "zeek.intel.matched": [
+            "Intel::ADDR"
+        ],
+        "zeek.intel.seen.indicator": "198.41.0.4",
+        "zeek.intel.seen.indicator_type": "Intel::ADDR",
+        "zeek.intel.seen.node": "worker-1-2",
+        "zeek.intel.seen.where": "Conn::IN_RESP",
+        "zeek.intel.sources": [
+            "ETPRO Rep: AbusedTLD Score: 127"
+        ],
+        "zeek.session_id": "Ctefoj1tgOPt4D0EK2"
+    }
+]

x-pack/filebeat/modules.d/zeek.yml.disabled

@@ -22,6 +22,8 @@
     enabled: true
   http:
     enabled: true
+  intel:
+    enabled: true    
   irc:
     enabled: true
   kerberos: