Skip to content

Commit

Permalink
Netflow: fix field name conversion to snake case
Browse files Browse the repository at this point in the history
Original field name conversion was buggy.
  • Loading branch information
adriansr committed Feb 27, 2019
1 parent 84603c2 commit 42957dc
Show file tree
Hide file tree
Showing 8 changed files with 220 additions and 178 deletions.
14 changes: 10 additions & 4 deletions x-pack/filebeat/input/netflow/case.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,12 @@ import (
)

var fieldNameConverter = caseConverter{
conversion: make(map[string]string),
conversion: map[string]string{
// Special handled fields

// VRFname should be VRFName
"VRFname": "vrf_name",
},
}

type caseConverter struct {
Expand Down Expand Up @@ -50,15 +55,15 @@ func (c *caseConverter) ToSnakeCase(orig record.Map) common.MapStr {
// format. This function is tailored to some specifics of NetFlow field names.
// Don't reuse it.
func CamelCaseToSnakeCase(in string) string {
// Lowercase those few fields that are already snake-cased
// skip those few fields that are already snake-cased
if strings.ContainsRune(in, '_') {
return strings.ToLower(in)
}

out := make([]rune, 0, len(in)+4)
runes := []rune(in)
upperStrike := 1
for pos, r := range runes {
for _, r := range runes {
lr := unicode.ToLower(r)
isUpper := lr != r
if isUpper {
Expand All @@ -77,7 +82,8 @@ func CamelCaseToSnakeCase(in string) string {
// postNATSourceIPv4Address : post_nat_source_ipv4_address
// selectorIDTotalFlowsObserved : selector_id_total_flows_...
out = append(out, '_')
out[pos], out[pos-1] = out[pos-1], out[pos]
n := len(out) - 1
out[n], out[n-1] = out[n-1], out[n]
}
upperStrike = 0
}
Expand Down
36 changes: 36 additions & 0 deletions x-pack/filebeat/input/netflow/case_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
// or more contributor license agreements. Licensed under the Elastic License;
// you may not use this file except in compliance with the Elastic License.

package netflow

import (
"testing"

"github.com/stretchr/testify/assert"
)

func TestCamelCaseToSnakeCase(t *testing.T) {
for _, testCase := range [][2]string{
{"aBCDe", "a_bc_de"},
{"postNATSourceIPv4Address", "post_nat_source_ipv4_address"},
{"selectorIDTotalFlowsObserved", "selector_id_total_flows_observed"},
{"engineId", "engine_id"},
{"samplerRandomInterval", "sampler_random_interval"},
{"dot1qVlanId", "dot1q_vlan_id"},
{"messageMD5Checksum", "message_md5_checksum"},
{"hashIPPayloadSize", "hash_ip_payload_size"},
{"upperCILimit", "upper_ci_limit"},
{"virtualStationUUID", "virtual_station_uuid"},
{"selectorIDTotalFlowsObserved", "selector_id_total_flows_observed"},
{"postMCastLayer2OctetDeltaCount", "post_mcast_layer2_octet_delta_count"},
{"IPSecSPI", "ip_sec_spi"},
{"VRFname", "vrf_name"},
} {
s, found := fieldNameConverter.conversion[testCase[0]]
if !found {
s = CamelCaseToSnakeCase(testCase[0])
}
assert.Equal(t, testCase[1], s)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@
"ip_version": 4,
"octet_delta_count": 152,
"packet_delta_count": 2,
"post_nadt_estination_ipv4_address": "192.168.128.17",
"post_nast_ource_ipv4_address": "192.168.230.216",
"post_nat_destination_ipv4_address": "192.168.128.17",
"post_nat_source_ipv4_address": "192.168.230.216",
"protocol_identifier": 17,
"source_ipv4_address": "10.10.8.197",
"source_transport_port": 123,
Expand Down Expand Up @@ -102,8 +102,8 @@
"ip_version": 4,
"octet_delta_count": 502,
"packet_delta_count": 8,
"post_nadt_estination_ipv4_address": "10.10.6.11",
"post_nast_ource_ipv4_address": "192.168.35.143",
"post_nat_destination_ipv4_address": "10.10.6.11",
"post_nat_source_ipv4_address": "192.168.35.143",
"protocol_identifier": 6,
"source_ipv4_address": "192.168.35.143",
"source_transport_port": 46518,
Expand Down Expand Up @@ -166,8 +166,8 @@
"ip_version": 4,
"octet_delta_count": 2233,
"packet_delta_count": 8,
"post_nadt_estination_ipv4_address": "192.168.35.143",
"post_nast_ource_ipv4_address": "192.168.230.216",
"post_nat_destination_ipv4_address": "192.168.35.143",
"post_nat_source_ipv4_address": "192.168.230.216",
"protocol_identifier": 6,
"source_ipv4_address": "10.10.6.11",
"source_transport_port": 80,
Expand Down Expand Up @@ -230,8 +230,8 @@
"ip_version": 4,
"octet_delta_count": 152,
"packet_delta_count": 2,
"post_nadt_estination_ipv4_address": "10.10.8.197",
"post_nast_ource_ipv4_address": "192.168.128.17",
"post_nat_destination_ipv4_address": "10.10.8.197",
"post_nat_source_ipv4_address": "192.168.128.17",
"protocol_identifier": 17,
"source_ipv4_address": "192.168.128.17",
"source_transport_port": 123,
Expand Down Expand Up @@ -294,8 +294,8 @@
"ip_version": 4,
"octet_delta_count": 79724,
"packet_delta_count": 57,
"post_nadt_estination_ipv4_address": "172.20.5.191",
"post_nast_ource_ipv4_address": "10.10.8.220",
"post_nat_destination_ipv4_address": "172.20.5.191",
"post_nat_source_ipv4_address": "10.10.8.220",
"protocol_identifier": 6,
"source_ipv4_address": "10.10.8.220",
"source_transport_port": 80,
Expand Down Expand Up @@ -358,8 +358,8 @@
"ip_version": 4,
"octet_delta_count": 161,
"packet_delta_count": 3,
"post_nadt_estination_ipv4_address": "172.20.4.1",
"post_nast_ource_ipv4_address": "172.20.4.199",
"post_nat_destination_ipv4_address": "172.20.4.1",
"post_nat_source_ipv4_address": "172.20.4.199",
"protocol_identifier": 17,
"source_ipv4_address": "172.20.4.199",
"source_transport_port": 10240,
Expand Down Expand Up @@ -422,8 +422,8 @@
"ip_version": 4,
"octet_delta_count": 245,
"packet_delta_count": 3,
"post_nadt_estination_ipv4_address": "172.20.4.199",
"post_nast_ource_ipv4_address": "172.20.4.1",
"post_nat_destination_ipv4_address": "172.20.4.199",
"post_nat_source_ipv4_address": "172.20.4.1",
"protocol_identifier": 17,
"source_ipv4_address": "172.20.4.1",
"source_transport_port": 53,
Expand Down Expand Up @@ -486,8 +486,8 @@
"ip_version": 4,
"octet_delta_count": 504,
"packet_delta_count": 6,
"post_nadt_estination_ipv4_address": "10.10.8.34",
"post_nast_ource_ipv4_address": "172.20.4.30",
"post_nat_destination_ipv4_address": "10.10.8.34",
"post_nat_source_ipv4_address": "172.20.4.30",
"protocol_identifier": 1,
"source_ipv4_address": "172.20.4.30",
"source_transport_port": 0,
Expand Down Expand Up @@ -550,8 +550,8 @@
"ip_version": 4,
"octet_delta_count": 784,
"packet_delta_count": 6,
"post_nadt_estination_ipv4_address": "172.20.4.30",
"post_nast_ource_ipv4_address": "10.10.8.105",
"post_nat_destination_ipv4_address": "172.20.4.30",
"post_nat_source_ipv4_address": "10.10.8.105",
"protocol_identifier": 6,
"source_ipv4_address": "10.10.8.105",
"source_transport_port": 22,
Expand Down Expand Up @@ -614,8 +614,8 @@
"ip_version": 4,
"octet_delta_count": 433,
"packet_delta_count": 8,
"post_nadt_estination_ipv4_address": "10.10.8.105",
"post_nast_ource_ipv4_address": "172.20.4.30",
"post_nat_destination_ipv4_address": "10.10.8.105",
"post_nat_source_ipv4_address": "172.20.4.30",
"protocol_identifier": 6,
"source_ipv4_address": "172.20.4.30",
"source_transport_port": 59571,
Expand Down Expand Up @@ -678,8 +678,8 @@
"ip_version": 4,
"octet_delta_count": 196,
"packet_delta_count": 3,
"post_nadt_estination_ipv4_address": "192.168.183.199",
"post_nast_ource_ipv4_address": "192.168.230.216",
"post_nat_destination_ipv4_address": "192.168.183.199",
"post_nat_source_ipv4_address": "192.168.230.216",
"protocol_identifier": 6,
"source_ipv4_address": "10.10.7.11",
"source_transport_port": 48378,
Expand Down Expand Up @@ -742,8 +742,8 @@
"ip_version": 4,
"octet_delta_count": 206,
"packet_delta_count": 3,
"post_nadt_estination_ipv4_address": "10.10.7.11",
"post_nast_ource_ipv4_address": "192.168.183.199",
"post_nat_destination_ipv4_address": "10.10.7.11",
"post_nat_source_ipv4_address": "192.168.183.199",
"protocol_identifier": 6,
"source_ipv4_address": "192.168.183.199",
"source_transport_port": 6667,
Expand Down Expand Up @@ -806,8 +806,8 @@
"ip_version": 4,
"octet_delta_count": 504,
"packet_delta_count": 6,
"post_nadt_estination_ipv4_address": "172.20.4.30",
"post_nast_ource_ipv4_address": "10.10.8.34",
"post_nat_destination_ipv4_address": "172.20.4.30",
"post_nat_source_ipv4_address": "10.10.8.34",
"protocol_identifier": 1,
"source_ipv4_address": "10.10.8.34",
"source_transport_port": 0,
Expand Down Expand Up @@ -870,8 +870,8 @@
"ip_version": 4,
"octet_delta_count": 3539,
"packet_delta_count": 58,
"post_nadt_estination_ipv4_address": "10.10.8.220",
"post_nast_ource_ipv4_address": "172.20.5.191",
"post_nat_destination_ipv4_address": "10.10.8.220",
"post_nat_source_ipv4_address": "172.20.5.191",
"protocol_identifier": 6,
"source_ipv4_address": "172.20.5.191",
"source_transport_port": 42502,
Expand Down Expand Up @@ -934,8 +934,8 @@
"ip_version": 4,
"octet_delta_count": 495,
"packet_delta_count": 3,
"post_nadt_estination_ipv4_address": "255.255.255.255",
"post_nast_ource_ipv4_address": "172.20.4.1",
"post_nat_destination_ipv4_address": "255.255.255.255",
"post_nat_source_ipv4_address": "172.20.4.1",
"protocol_identifier": 17,
"source_ipv4_address": "172.20.4.1",
"source_transport_port": 33332,
Expand Down Expand Up @@ -998,8 +998,8 @@
"ip_version": 4,
"octet_delta_count": 330,
"packet_delta_count": 2,
"post_nadt_estination_ipv4_address": "255.255.255.255",
"post_nast_ource_ipv4_address": "172.20.4.1",
"post_nat_destination_ipv4_address": "255.255.255.255",
"post_nat_source_ipv4_address": "172.20.4.1",
"protocol_identifier": 17,
"source_ipv4_address": "172.20.4.1",
"source_transport_port": 33332,
Expand Down Expand Up @@ -1062,8 +1062,8 @@
"ip_version": 4,
"octet_delta_count": 435,
"packet_delta_count": 3,
"post_nadt_estination_ipv4_address": "255.255.255.255",
"post_nast_ource_ipv4_address": "172.30.0.1",
"post_nat_destination_ipv4_address": "255.255.255.255",
"post_nat_source_ipv4_address": "172.30.0.1",
"protocol_identifier": 17,
"source_ipv4_address": "172.30.0.1",
"source_transport_port": 53298,
Expand Down Expand Up @@ -1126,8 +1126,8 @@
"ip_version": 4,
"octet_delta_count": 290,
"packet_delta_count": 2,
"post_nadt_estination_ipv4_address": "255.255.255.255",
"post_nast_ource_ipv4_address": "172.30.0.1",
"post_nat_destination_ipv4_address": "255.255.255.255",
"post_nat_source_ipv4_address": "172.30.0.1",
"protocol_identifier": 17,
"source_ipv4_address": "172.30.0.1",
"source_transport_port": 53298,
Expand Down Expand Up @@ -1190,8 +1190,8 @@
"ip_version": 4,
"octet_delta_count": 495,
"packet_delta_count": 3,
"post_nadt_estination_ipv4_address": "255.255.255.255",
"post_nast_ource_ipv4_address": "10.10.6.1",
"post_nat_destination_ipv4_address": "255.255.255.255",
"post_nat_source_ipv4_address": "10.10.6.1",
"protocol_identifier": 17,
"source_ipv4_address": "10.10.6.1",
"source_transport_port": 48172,
Expand Down Expand Up @@ -1254,8 +1254,8 @@
"ip_version": 4,
"octet_delta_count": 330,
"packet_delta_count": 2,
"post_nadt_estination_ipv4_address": "255.255.255.255",
"post_nast_ource_ipv4_address": "10.10.6.1",
"post_nat_destination_ipv4_address": "255.255.255.255",
"post_nat_source_ipv4_address": "10.10.6.1",
"protocol_identifier": 17,
"source_ipv4_address": "10.10.6.1",
"source_transport_port": 48172,
Expand Down Expand Up @@ -1318,8 +1318,8 @@
"ip_version": 4,
"octet_delta_count": 495,
"packet_delta_count": 3,
"post_nadt_estination_ipv4_address": "255.255.255.255",
"post_nast_ource_ipv4_address": "10.10.7.1",
"post_nat_destination_ipv4_address": "255.255.255.255",
"post_nat_source_ipv4_address": "10.10.7.1",
"protocol_identifier": 17,
"source_ipv4_address": "10.10.7.1",
"source_transport_port": 48935,
Expand Down Expand Up @@ -1382,8 +1382,8 @@
"ip_version": 4,
"octet_delta_count": 330,
"packet_delta_count": 2,
"post_nadt_estination_ipv4_address": "255.255.255.255",
"post_nast_ource_ipv4_address": "10.10.7.1",
"post_nat_destination_ipv4_address": "255.255.255.255",
"post_nat_source_ipv4_address": "10.10.7.1",
"protocol_identifier": 17,
"source_ipv4_address": "10.10.7.1",
"source_transport_port": 48935,
Expand Down Expand Up @@ -1446,8 +1446,8 @@
"ip_version": 4,
"octet_delta_count": 495,
"packet_delta_count": 3,
"post_nadt_estination_ipv4_address": "255.255.255.255",
"post_nast_ource_ipv4_address": "10.10.8.1",
"post_nat_destination_ipv4_address": "255.255.255.255",
"post_nat_source_ipv4_address": "10.10.8.1",
"protocol_identifier": 17,
"source_ipv4_address": "10.10.8.1",
"source_transport_port": 51931,
Expand Down Expand Up @@ -1510,8 +1510,8 @@
"ip_version": 4,
"octet_delta_count": 330,
"packet_delta_count": 2,
"post_nadt_estination_ipv4_address": "255.255.255.255",
"post_nast_ource_ipv4_address": "10.10.8.1",
"post_nat_destination_ipv4_address": "255.255.255.255",
"post_nat_source_ipv4_address": "10.10.8.1",
"protocol_identifier": 17,
"source_ipv4_address": "10.10.8.1",
"source_transport_port": 51931,
Expand Down Expand Up @@ -1574,8 +1574,8 @@
"ip_version": 4,
"octet_delta_count": 495,
"packet_delta_count": 3,
"post_nadt_estination_ipv4_address": "255.255.255.255",
"post_nast_ource_ipv4_address": "10.20.0.1",
"post_nat_destination_ipv4_address": "255.255.255.255",
"post_nat_source_ipv4_address": "10.20.0.1",
"protocol_identifier": 17,
"source_ipv4_address": "10.20.0.1",
"source_transport_port": 43454,
Expand Down Expand Up @@ -1638,8 +1638,8 @@
"ip_version": 4,
"octet_delta_count": 330,
"packet_delta_count": 2,
"post_nadt_estination_ipv4_address": "255.255.255.255",
"post_nast_ource_ipv4_address": "10.20.0.1",
"post_nat_destination_ipv4_address": "255.255.255.255",
"post_nat_source_ipv4_address": "10.20.0.1",
"protocol_identifier": 17,
"source_ipv4_address": "10.20.0.1",
"source_transport_port": 43454,
Expand Down Expand Up @@ -1702,8 +1702,8 @@
"ip_version": 4,
"octet_delta_count": 495,
"packet_delta_count": 3,
"post_nadt_estination_ipv4_address": "255.255.255.255",
"post_nast_ource_ipv4_address": "10.10.10.1",
"post_nat_destination_ipv4_address": "255.255.255.255",
"post_nat_source_ipv4_address": "10.10.10.1",
"protocol_identifier": 17,
"source_ipv4_address": "10.10.10.1",
"source_transport_port": 52837,
Expand Down Expand Up @@ -1766,8 +1766,8 @@
"ip_version": 4,
"octet_delta_count": 330,
"packet_delta_count": 2,
"post_nadt_estination_ipv4_address": "255.255.255.255",
"post_nast_ource_ipv4_address": "10.10.10.1",
"post_nat_destination_ipv4_address": "255.255.255.255",
"post_nat_source_ipv4_address": "10.10.10.1",
"protocol_identifier": 17,
"source_ipv4_address": "10.10.10.1",
"source_transport_port": 52837,
Expand Down
Loading

0 comments on commit 42957dc

Please sign in to comment.