[Filebeat][Fortinet]Converting certain fields to type long (elastic#1…

…8736) * certain fields was set to string instead of long, causing issues with painless and did not conform with the ECS standards
leehinman · May 27, 2020 · 306a5a8 · 306a5a8
1 parent b772f2a
commit 306a5a8
Show file tree

Hide file tree

Showing 4 changed files with 168 additions and 84 deletions.
diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/event.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/event.yml
@@ -61,19 +61,25 @@ processors:
     field: fortinet.firewall.remip
     target_field: destination.ip
     ignore_missing: true
-    if: "ctx.destination?.ip == null"
-- rename:
+    if: "ctx.destination?.ip == null"  
+- convert:
     field: fortinet.firewall.dstport
     target_field: destination.port
+    type: long
+    ignore_failure: true
     ignore_missing: true
-- rename:
+- convert:
     field: fortinet.firewall.remport
     target_field: destination.port
+    type: long
+    ignore_failure: true
     ignore_missing: true
     if: "ctx.destination?.port == null"
-- rename:
+- convert:
     field: fortinet.firewall.rcvdbyte
     target_field: destination.bytes
+    type: long
+    ignore_failure: true
     ignore_missing: true
 - rename:
     field: fortinet.firewall.daddr
@@ -93,9 +99,11 @@ processors:
     field: fortinet.firewall.group
     target_field: source.user.group.name
     ignore_missing: true
-- rename:
+- convert:
     field: fortinet.firewall.sentbyte
     target_field: source.bytes
+    type: long
+    ignore_failure: true
     ignore_missing: true
 - rename:
     field: fortinet.firewall.srcip
@@ -115,13 +123,17 @@ processors:
     target_field: source.mac
     ignore_missing: true
     if: "ctx.source?.mac == null"
-- rename:
+- convert:
     field: fortinet.firewall.srcport
     target_field: source.port
+    type: long
+    ignore_failure: true
     ignore_missing: true
-- rename:
+- convert:
     field: fortinet.firewall.locport
     target_field: source.port
+    type: long
+    ignore_failure: true
     ignore_missing: true
     if: "ctx.source?.port == null"
 - rename:
@@ -140,9 +152,11 @@ processors:
     field: fortinet.firewall.file
     target_field: file.name
     ignore_missing: true
-- rename:
+- convert:
     field: fortinet.firewall.filesize
     target_field: file.size
+    type: long
+    ignore_failure: true
     ignore_missing: true
 - rename:
     field: fortinet.firewall.level
@@ -197,9 +211,11 @@ processors:
     field: fortinet.firewall.url
     target_field: url.path
     ignore_missing: true
-- rename:
+- convert:
     field: fortinet.firewall.sess_duration
+    type: long
     target_field: event.duration
+    ignore_failure: true
     ignore_missing: true
     if: "ctx.event?.duration == null"
 - geoip:
@@ -289,6 +305,17 @@ processors:
     field: related.user
     value: "{{source.user.name}}"
     if: "ctx.source?.user?.name != null"
+- remove:
+    field:
+    - fortinet.firewall.dstport
+    - fortinet.firewall.remport
+    - fortinet.firewall.rcvdbyte
+    - fortinet.firewall.sentbyte
+    - fortinet.firewall.srcport
+    - fortinet.firewall.locport
+    - fortinet.firewall.filesize
+    - fortinet.firewall.sess_duration
+    ignore_missing: true
 on_failure:
 - set:
     field: error.message

diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/traffic.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/traffic.yml
@@ -45,21 +45,29 @@ processors:
     field: fortinet.firewall.tranip
     target_field: destination.nat.ip
     ignore_missing: true
-- rename:
+- convert:
     field: fortinet.firewall.dstport
     target_field: destination.port
+    type: long
+    ignore_failure: true
     ignore_missing: true
-- rename:
+- convert:
     field: fortinet.firewall.tranport
     target_field: destination.nat.port
+    type: long
+    ignore_failure: true
     ignore_missing: true
-- rename:
+- convert:
     field: fortinet.firewall.rcvdbyte
     target_field: destination.bytes
+    type: long
+    ignore_failure: true
     ignore_missing: true
-- rename:
+- convert:
     field: fortinet.firewall.rcvdpkt
     target_field: destination.packets
+    type: long
+    ignore_failure: true
     ignore_missing: true
 - rename:
     field: fortinet.firewall.dstcollectedemail
@@ -77,9 +85,11 @@ processors:
     field: fortinet.firewall.group
     target_field: source.user.group.name
     ignore_missing: true
-- rename:
+- convert:
     field: fortinet.firewall.sentbyte
     target_field: source.bytes
+    type: long
+    ignore_failure: true
     ignore_missing: true
 - rename:
     field: fortinet.firewall.srcdomain
@@ -93,9 +103,11 @@ processors:
     field: fortinet.firewall.srcmac
     target_field: source.mac
     ignore_missing: true
-- rename:
+- convert:
     field: fortinet.firewall.srcport
     target_field: source.port
+    type: long
+    ignore_failure: true
     ignore_missing: true
 - rename:
     field: fortinet.firewall.unauthuser
@@ -110,17 +122,21 @@ processors:
     field: fortinet.firewall.collectedemail
     target_field: source.user.email
     ignore_missing: true
-- rename:
+- convert:
     field: fortinet.firewall.sentpkt
     target_field: source.packets
+    type: long
+    ignore_failure: true
     ignore_missing: true
 - rename:
     field: fortinet.firewall.transip
     target_field: source.nat.ip
     ignore_missing: true
-- rename:
+- convert:
     field: fortinet.firewall.transport
     target_field: source.nat.port
+    type: long
+    ignore_failure: true
     ignore_missing: true
 - rename:
     field: fortinet.firewall.app
@@ -280,6 +296,17 @@ processors:
     field: related.user
     value: "{{destination.user.name}}"
     if: "ctx.destination?.user?.name != null"
+- remove:
+    field:
+    - fortinet.firewall.dstport
+    - fortinet.firewall.tranport
+    - fortinet.firewall.rcvdbyte
+    - fortinet.firewall.rcvdpkt
+    - fortinet.firewall.sentbyte
+    - fortinet.firewall.srcport
+    - fortinet.firewall.sentpkt
+    - fortinet.firewall.transport
+    ignore_missing: true
 on_failure:
 - set:
     field: error.message

diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/utm.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/utm.yml
@@ -30,24 +30,32 @@ processors:
     field: fortinet.firewall.remip
     target_field: destination.ip
     ignore_missing: true
-    if: "ctx.destination?.ip == null"
-- rename:
+    if: "ctx.destination?.ip == null"  
+- convert:
     field: fortinet.firewall.dst_port
     target_field: destination.port
+    type: long
+    ignore_failure: true
     ignore_missing: true
-- rename:
+- convert:
     field: fortinet.firewall.remport
     target_field: destination.port
+    type: long
+    ignore_failure: true
     ignore_missing: true
     if: "ctx.destination?.port == null"
-- rename:
+- convert:
     field: fortinet.firewall.dstport
     target_field: destination.port
+    type: long
+    ignore_failure: true
     ignore_missing: true
     if: "ctx.destination?.port == null"
-- rename:
+- convert:
     field: fortinet.firewall.rcvdbyte
     target_field: destination.bytes
+    type: long
+    ignore_failure: true
     ignore_missing: true
 - rename:
     field: fortinet.firewall.recipient
@@ -61,18 +69,31 @@ processors:
     field: fortinet.firewall.locip
     target_field: source.ip
     ignore_missing: true
-- rename:
+- convert:
     field: fortinet.firewall.locport
     target_field: source.port
+    type: long
+    ignore_failure: true
     ignore_missing: true
-- rename:
+- convert:
     field: fortinet.firewall.src_port
     target_field: source.port
+    type: long
+    ignore_failure: true
     ignore_missing: true
     if: "ctx.source?.port == null"
-- rename:
+- convert:
+    field: fortinet.firewall.srcport
+    target_field: source.port
+    type: long
+    ignore_failure: true
+    ignore_missing: true
+    if: "ctx.source?.port == null"
+- convert:
     field: fortinet.firewall.sentbyte
     target_field: source.bytes
+    type: long
+    ignore_failure: true
     ignore_missing: true
 - rename:
     field: fortinet.firewall.srcdomain
@@ -87,11 +108,6 @@ processors:
     field: fortinet.firewall.srcmac
     target_field: source.mac
     ignore_missing: true
-- rename:
-    field: fortinet.firewall.srcport
-    target_field: source.port
-    ignore_missing: true
-    if: "ctx.source?.port == null"
 - rename:
     field: fortinet.firewall.unauthuser
     target_field: source.user.name
@@ -171,9 +187,11 @@ processors:
     field: fortinet.firewall.filename
     target_field: file.name
     ignore_missing: true
-- rename:
+- convert:
     field: fortinet.firewall.filesize
     target_field: file.size
+    type: long
+    ignore_failure: true
     ignore_missing: true
 - rename:
     field: fortinet.firewall.filetype
@@ -390,6 +408,18 @@ processors:
     field: related.hash
     value: "{{fortinet.file.hash.crc32}}"
     if: "ctx.fortinet?.file?.hash?.crc32 != null"
+- remove:
+    field:
+    - fortinet.firewall.dst_port
+    - fortinet.firewall.remport
+    - fortinet.firewall.dstport
+    - fortinet.firewall.rcvdbyte
+    - fortinet.firewall.locport
+    - fortinet.firewall.src_port
+    - fortinet.firewall.srcport
+    - fortinet.firewall.sentbyte
+    - fortinet.firewall.filesize
+    ignore_missing: true
 on_failure:
 - set:
     field: error.message