Skip to content

Commit

Permalink
[Filebeat] Improve ECS categorization field mappings for netflow modu…
Browse files Browse the repository at this point in the history 
…le (#18108) (#18173)

* Improve ECS categorization field mappings for netflow module

- event.category : make array and add network
- event.type
- related.ip

Closes #16135

(cherry picked from commit e6d4787)
  • Loading branch information
@leehinman
leehinman committed May 4, 2020
1 parent d034fae commit 65e7348
Show file tree
Hide file tree
Showing 45 changed files with 5,572 additions and 861 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Improve ECS categorization field mappings in postgresql module. {issue}16177[16177] {pull}17914[17914]
- Improve ECS categorization field mappings for nginx module. {issue}16174[16174] {pull}17844[17844]
- Improve ECS categorization field mappings for zeek module. {issue}16029[16029] {pull}17738[17738]
- Improve ECS categorization field mappings for netflow module. {issue}16135[16135] {pull}18108[18108]

*Heartbeat*

Expand Down
13 changes: 11 additions & 2 deletions x-pack/filebeat/input/netflow/convert.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,12 @@ func toBeatEventCommon(flow record.Record) (event beat.Event) {
ecsEvent := common.MapStr{
"created": flow.Timestamp,
"kind": "event",
"category": "network_traffic",
"category": []string{"network_traffic", "network"},
"action": flow.Fields["type"],
}
if ecsEvent["action"] == "netflow_flow" {
ecsEvent["type"] = []string{"connection"}
}
// ECS Fields -- device
ecsDevice := common.MapStr{}
if exporter, ok := getKeyString(flow.Exporter, "address"); ok {
Expand Down Expand Up @@ -155,9 +158,10 @@ func flowToBeatEvent(flow record.Record) (event beat.Event) {
}

flowDirection, hasFlowDirection := getKeyUint64(flow.Fields, "flowDirection")
// ECS Fields -- source and destination
// ECS Fields -- source, destination & related.ip
ecsSource := common.MapStr{}
ecsDest := common.MapStr{}
var relatedIP []net.IP

// Populate first with WLAN fields
if hasFlowDirection {
Expand Down Expand Up @@ -189,6 +193,7 @@ func flowToBeatEvent(flow record.Record) (event beat.Event) {
// Regular IPv4 fields
if ip, found := getKeyIP(flow.Fields, "sourceIPv4Address"); found {
ecsSource["ip"] = ip
relatedIP = append(relatedIP, ip)
ecsSource["locality"] = getIPLocality(ip).String()
}
if sourcePort, found := getKeyUint64(flow.Fields, "sourceTransportPort"); found {
Expand All @@ -201,6 +206,7 @@ func flowToBeatEvent(flow record.Record) (event beat.Event) {
// ECS Fields -- destination
if ip, found := getKeyIP(flow.Fields, "destinationIPv4Address"); found {
ecsDest["ip"] = ip
relatedIP = append(relatedIP, ip)
ecsDest["locality"] = getIPLocality(ip).String()
}
if destPort, found := getKeyUint64(flow.Fields, "destinationTransportPort"); found {
Expand Down Expand Up @@ -313,6 +319,9 @@ func flowToBeatEvent(flow record.Record) (event beat.Event) {
if len(ecsNetwork) > 0 {
event.Fields["network"] = ecsNetwork
}
if len(relatedIP) > 0 {
event.Fields["related"] = common.MapStr{"ip": relatedIP}
}
return
}

Expand Down
32 changes: 28 additions & 4 deletions ...t/input/netflow/testdata/golden/IPFIX-Barracuda-extended-uniflow-template-256.golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,16 @@
},
"event": {
"action": "netflow_flow",
"category": "network_traffic",
"category": [
"network_traffic",
"network"
],
"created": "2018-04-18T08:16:47Z",
"duration": 0,
"kind": "event"
"kind": "event",
"type": [
"connection"
]
},
"flow": {
"id": "kSpZ1WuBhjc",
Expand Down Expand Up @@ -70,6 +76,12 @@
"observer": {
"ip": "192.0.2.1"
},
"related": {
"ip": [
"10.236.5.4",
"64.235.151.76"
]
},
"source": {
"bytes": 0,
"ip": "10.236.5.4",
Expand All @@ -93,10 +105,16 @@
},
"event": {
"action": "netflow_flow",
"category": "network_traffic",
"category": [
"network_traffic",
"network"
],
"created": "2018-04-18T08:16:47Z",
"duration": 0,
"kind": "event"
"kind": "event",
"type": [
"connection"
]
},
"flow": {
"id": "kSpZ1WuBhjc",
Expand Down Expand Up @@ -151,6 +169,12 @@
"observer": {
"ip": "192.0.2.1"
},
"related": {
"ip": [
"64.235.151.76",
"10.236.5.4"
]
},
"source": {
"bytes": 0,
"ip": "64.235.151.76",
Expand Down
128 changes: 112 additions & 16 deletions x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-firewall.golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,16 @@
},
"event": {
"action": "netflow_flow",
"category": "network_traffic",
"category": [
"network_traffic",
"network"
],
"created": "2017-06-29T13:58:28Z",
"duration": 20269000000,
"kind": "event"
"kind": "event",
"type": [
"connection"
]
},
"flow": {
"id": "2vFIarATx_4",
Expand Down Expand Up @@ -58,6 +64,12 @@
"observer": {
"ip": "192.0.2.1"
},
"related": {
"ip": [
"10.99.130.239",
"10.99.252.50"
]
},
"source": {
"bytes": 0,
"ip": "10.99.130.239",
Expand All @@ -81,10 +93,16 @@
},
"event": {
"action": "netflow_flow",
"category": "network_traffic",
"category": [
"network_traffic",
"network"
],
"created": "2017-06-29T13:58:28Z",
"duration": 20269000000,
"kind": "event"
"kind": "event",
"type": [
"connection"
]
},
"flow": {
"id": "2vFIarATx_4",
Expand Down Expand Up @@ -127,6 +145,12 @@
"observer": {
"ip": "192.0.2.1"
},
"related": {
"ip": [
"10.99.252.50",
"10.99.130.239"
]
},
"source": {
"bytes": 81,
"ip": "10.99.252.50",
Expand All @@ -150,10 +174,16 @@
},
"event": {
"action": "netflow_flow",
"category": "network_traffic",
"category": [
"network_traffic",
"network"
],
"created": "2017-06-29T13:58:28Z",
"duration": 20306000000,
"kind": "event"
"kind": "event",
"type": [
"connection"
]
},
"flow": {
"id": "wU3G8idsscw",
Expand Down Expand Up @@ -196,6 +226,12 @@
"observer": {
"ip": "192.0.2.1"
},
"related": {
"ip": [
"10.99.130.239",
"10.98.243.20"
]
},
"source": {
"bytes": 0,
"ip": "10.99.130.239",
Expand All @@ -219,10 +255,16 @@
},
"event": {
"action": "netflow_flow",
"category": "network_traffic",
"category": [
"network_traffic",
"network"
],
"created": "2017-06-29T13:58:28Z",
"duration": 20306000000,
"kind": "event"
"kind": "event",
"type": [
"connection"
]
},
"flow": {
"id": "wU3G8idsscw",
Expand Down Expand Up @@ -265,6 +307,12 @@
"observer": {
"ip": "192.0.2.1"
},
"related": {
"ip": [
"10.98.243.20",
"10.99.130.239"
]
},
"source": {
"bytes": 81,
"ip": "10.98.243.20",
Expand All @@ -288,10 +336,16 @@
},
"event": {
"action": "netflow_flow",
"category": "network_traffic",
"category": [
"network_traffic",
"network"
],
"created": "2017-06-29T13:58:28Z",
"duration": 20317000000,
"kind": "event"
"kind": "event",
"type": [
"connection"
]
},
"flow": {
"id": "rOmj8EdZ2dc",
Expand Down Expand Up @@ -334,6 +388,12 @@
"observer": {
"ip": "192.0.2.1"
},
"related": {
"ip": [
"10.99.168.140",
"10.98.243.20"
]
},
"source": {
"bytes": 0,
"ip": "10.99.168.140",
Expand All @@ -357,10 +417,16 @@
},
"event": {
"action": "netflow_flow",
"category": "network_traffic",
"category": [
"network_traffic",
"network"
],
"created": "2017-06-29T13:58:28Z",
"duration": 20317000000,
"kind": "event"
"kind": "event",
"type": [
"connection"
]
},
"flow": {
"id": "rOmj8EdZ2dc",
Expand Down Expand Up @@ -403,6 +469,12 @@
"observer": {
"ip": "192.0.2.1"
},
"related": {
"ip": [
"10.98.243.20",
"10.99.168.140"
]
},
"source": {
"bytes": 113,
"ip": "10.98.243.20",
Expand All @@ -426,10 +498,16 @@
},
"event": {
"action": "netflow_flow",
"category": "network_traffic",
"category": [
"network_traffic",
"network"
],
"created": "2017-06-29T13:58:28Z",
"duration": 20368000000,
"kind": "event"
"kind": "event",
"type": [
"connection"
]
},
"flow": {
"id": "JE7pThaMwJY",
Expand Down Expand Up @@ -472,6 +550,12 @@
"observer": {
"ip": "192.0.2.1"
},
"related": {
"ip": [
"10.99.168.140",
"10.98.243.20"
]
},
"source": {
"bytes": 0,
"ip": "10.99.168.140",
Expand All @@ -495,10 +579,16 @@
},
"event": {
"action": "netflow_flow",
"category": "network_traffic",
"category": [
"network_traffic",
"network"
],
"created": "2017-06-29T13:58:28Z",
"duration": 20368000000,
"kind": "event"
"kind": "event",
"type": [
"connection"
]
},
"flow": {
"id": "JE7pThaMwJY",
Expand Down Expand Up @@ -541,6 +631,12 @@
"observer": {
"ip": "192.0.2.1"
},
"related": {
"ip": [
"10.98.243.20",
"10.99.168.140"
]
},
"source": {
"bytes": 113,
"ip": "10.98.243.20",
Expand Down
Loading

0 comments on commit 65e7348

Please sign in to comment.