Update code to use config options from yaml

- update procs, protos, geoip and filters to use the config options from the yaml config file - remove ConfigMeta as it's not needed anymore - now all the options are pointers and if one is nil then the option is not defined (takes the default value) - update all the tests to make them work - fix elastic#5
tsg · Apr 9, 2015 · 3baeeec · 3baeeec
1 parent b76685b
commit 3baeeec
Show file tree

Hide file tree

Showing 14 changed files with 179 additions and 159 deletions.
diff --git a/config/config.go b/config/config.go
@@ -1,7 +1,6 @@
 package config
 
 import (
-	"github.com/BurntSushi/toml"
 	"github.com/elastic/infrabeat/common/droppriv"
 	"github.com/elastic/infrabeat/outputs"
 	"github.com/elastic/packetbeat/procs"
@@ -16,11 +15,11 @@ type Config struct {
 	Procs      procs.ProcsConfig
 	RunOptions droppriv.RunOptions
 	Logging    Logging
-	Passwords  Passwords
 	Thrift     Thrift
 	Http       Http
 	Mysql      Mysql
 	Pgsql      Pgsql
+	Redis      Redis
 	Geoip      outputs.Geoip
 	Udpjson    Udpjson
 	GoBeacon   GoBeacon
@@ -50,44 +49,55 @@ type Logging struct {
 	Selectors []string
 }
 
-type Passwords struct {
-	Hide_keywords       []string
-	Strip_authorization bool
-}
-
 type Protocol struct {
+	Protocol      string
 	Ports         []int
 	Send_request  bool
 	Send_response bool
 }
 
 type Http struct {
-	Send_all_headers bool
-	Send_headers     []string
-	Split_cookie     bool
-	Real_ip_header   string
-	Include_body_for []string
+	Send_all_headers    *bool
+	Send_headers        []string
+	Split_cookie        *bool
+	Real_ip_header      *string
+	Include_body_for    []string
+	Hide_keywords       []string
+	Strip_authorization *bool
+	Send_request        *bool
+	Send_response       *bool
 }
 
 type Mysql struct {
-	Max_row_length int
-	Max_rows       int
+	Max_row_length *int
+	Max_rows       *int
+	Send_request   *bool
+	Send_response  *bool
 }
 
 type Pgsql struct {
-	Max_row_length int
-	Max_rows       int
+	Max_row_length *int
+	Max_rows       *int
+	Send_request   *bool
+	Send_response  *bool
 }
 
 type Thrift struct {
-	String_max_size            int
-	Collection_max_size        int
-	Drop_after_n_struct_fields int
-	Transport_type             string
-	Protocol_type              string
-	Capture_reply              bool
-	Obfuscate_strings          bool
+	String_max_size            *int
+	Collection_max_size        *int
+	Drop_after_n_struct_fields *int
+	Transport_type             *string
+	Protocol_type              *string
+	Capture_reply              *bool
+	Obfuscate_strings          *bool
 	Idl_files                  []string
+	Send_request               *bool
+	Send_response              *bool
+}
+
+type Redis struct {
+	Send_request  *bool
+	Send_response *bool
 }
 
 type Udpjson struct {
@@ -103,6 +113,3 @@ type GoBeacon struct {
 
 // Config Singleton
 var ConfigSingleton Config
-
-// Config metadata singleton
-var ConfigMeta toml.MetaData
diff --git a/filters_runner.go b/filters_runner.go
@@ -77,7 +77,8 @@ func LoadConfiguredFilters(config map[string]interface{}) ([]filters.FilterPlugi
 				return nil, fmt.Errorf("No such filter type and no corresponding configuration: %s", filter)
 			}
 		} else {
-			plugin_config, ok := cfg.(map[string]interface{})
+			logp.Debug("filters", "%v", cfg)
+			plugin_config, ok := cfg.(map[interface{}]interface{})
 			if !ok {
 				return nil, fmt.Errorf("Invalid configuration for: %s", filter)
 			}

diff --git a/filters_runner_test.go b/filters_runner_test.go
@@ -31,10 +31,10 @@ func TestLoadConfiguredFilters(t *testing.T) {
 		io{
 			Input: map[string]interface{}{
 				"filters": []interface{}{"nop1", "nop2"},
-				"nop1": map[string]interface{}{
+				"nop1": map[interface{}]interface{}{
 					"type": "nop",
 				},
-				"nop2": map[string]interface{}{
+				"nop2": map[interface{}]interface{}{
 					"type": "nop",
 				},
 			},
@@ -53,7 +53,7 @@ func TestLoadConfiguredFilters(t *testing.T) {
 		io{
 			Input: map[string]interface{}{
 				"filters": []interface{}{"nop", "sample1"},
-				"sample1": map[string]interface{}{
+				"sample1": map[interface{}]interface{}{
 					"type": "nop",
 				},
 			},
@@ -95,7 +95,7 @@ func TestLoadConfiguredFiltersNegative(t *testing.T) {
 		io{
 			Input: map[string]interface{}{
 				"filters": []interface{}{"nop1", "nop2"},
-				"nop1": map[string]interface{}{
+				"nop1": map[interface{}]interface{}{
 					"type": "nop",
 				},
 			},
@@ -104,7 +104,7 @@ func TestLoadConfiguredFiltersNegative(t *testing.T) {
 		io{
 			Input: map[string]interface{}{
 				"filters": []interface{}{"nop1", "nop"},
-				"nop1": map[string]interface{}{
+				"nop1": map[interface{}]interface{}{
 					"hype": "nop",
 				},
 			},
@@ -113,7 +113,7 @@ func TestLoadConfiguredFiltersNegative(t *testing.T) {
 		io{
 			Input: map[string]interface{}{
 				"filters": []interface{}{"nop1", "nop"},
-				"nop1": map[string]interface{}{
+				"nop1": map[interface{}]interface{}{
 					"type": 1,
 				},
 			},

diff --git a/main.go b/main.go
@@ -19,6 +19,7 @@ import (
 	"github.com/elastic/infrabeat/filters/nop"
 	"github.com/elastic/infrabeat/logp"
 	"github.com/elastic/infrabeat/outputs"
+	"gopkg.in/yaml.v2"
 
 	"github.com/elastic/packetbeat/config"
 	"github.com/elastic/packetbeat/procs"
@@ -30,8 +31,6 @@ import (
 	"github.com/elastic/packetbeat/protos/tcp"
 	"github.com/elastic/packetbeat/protos/thrift"
 	"github.com/elastic/packetbeat/sniffer"
-
-	"github.com/BurntSushi/toml"
 )
 
 const Version = "0.5.0"
@@ -72,7 +71,7 @@ func main() {
 	// Use our own FlagSet, because some libraries pollute the global one
 	var cmdLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
 
-	configfile := cmdLine.String("c", "packetbeat.conf", "Configuration file")
+	configfile := cmdLine.String("c", "packetbeat.yaml", "Configuration file")
 	file := cmdLine.String("I", "", "file")
 	loop := cmdLine.Int("l", 1, "Loop file. 0 - loop forever")
 	debugSelectorsStr := cmdLine.String("d", "", "Enable certain debug selectors")
@@ -108,8 +107,13 @@ func main() {
 
 	var err error
 
-	if config.ConfigMeta, err = toml.DecodeFile(*configfile, &config.ConfigSingleton); err != nil {
-		fmt.Printf("TOML config parsing failed on %s: %s. Exiting.\n", *configfile, err)
+	filecontent, err := ioutil.ReadFile(*configfile)
+	if err != nil {
+		fmt.Printf("Fail to read %s: %s. Exiting.\n", *configfile, err)
+		return
+	}
+	if err = yaml.Unmarshal(filecontent, &config.ConfigSingleton); err != nil {
+		fmt.Printf("YAML config parsing failed on %s: %s. Exiting.\n", *configfile, err)
 		return
 	}
 
@@ -136,6 +140,7 @@ func main() {
 		config.ConfigSingleton.Interfaces.Dumpfile = *dumpfile
 	}
 
+	logp.Debug("main", "Configuration %s", config.ConfigSingleton)
 	logp.Debug("main", "Initializing output plugins")
 	if err = outputs.Publisher.Init(*publishDisabled, config.ConfigSingleton.Output,
 		config.ConfigSingleton.Agent); err != nil {
@@ -149,7 +154,7 @@ func main() {
 		return
 	}
 
-	err = outputs.LoadGeoIPData(config.ConfigSingleton.Geoip, config.ConfigMeta)
+	err = outputs.LoadGeoIPData(config.ConfigSingleton.Geoip)
 	if err != nil {
 		logp.Critical(err.Error())
 		return
@@ -207,7 +212,7 @@ func main() {
 	}
 
 	// This needs to be after the sniffer Init but before the sniffer Run.
-	if err = droppriv.DropPrivileges(config.ConfigSingleton.RunOptions, config.ConfigMeta); err != nil {
+	if err = droppriv.DropPrivileges(config.ConfigSingleton.RunOptions); err != nil {
 		logp.Critical(err.Error())
 		return
 	}

diff --git a/packetbeat.yaml b/packetbeat.yaml
@@ -36,37 +36,38 @@ interfaces:
 
 # Configure the processes to be monitored and how to find them. The processes can
 # be found by searching their command line by a given string.
-procs: 
- - process: mysqld
- cmdline_grep: mysqld
+procs:
+  monitored:
+    - process: mysqld
+      cmdline_grep: mysqld
 
- - process: pgsql
- cmdline_grep: postgres
+    - process: pgsql
+      cmdline_grep: postgres
 
- - process: nginx
- cmdline_grep: nginx
+    - process: nginx
+      cmdline_grep: nginx
 
- - process: app
- cmdline_grep: gunicorn
+    - process: app
+      cmdline_grep: gunicorn
 
 # Configure which protocols to monitor and the ports where they are
 # running. You can disable a given protocol by commenting out its
 # configuration.
 protocols: 
- - protocol: http
- ports: [80, 8080, 8000, 5000, 8002]
+  http:
+    ports: [80, 8080, 8000, 5000, 8002]
 
- - protocol: mysql
- ports: [3306]
+  mysql:
+    ports: [3306]
 
- - protocol: pgsql
- ports: [5432]
+  pgsql:
+    ports: [5432]
 
- - protocol: redis
- ports: [6379]
+  redis:
+    ports: [6379]
 
- - protocol: thrift
- ports: [9090]
+  thrift:
+    ports: [9090]
 
 
 ############################# Filters ############################################
@@ -84,35 +85,35 @@ hide_keywords: ['pass', 'password', 'passw']
 # You can enable one or multiple outputs by setting enabled option to true.
 output:
 
- # Elasticsearch as output
- # Options:
- # host, port: where Elasticsearch is listening on
- # save_topology: specify if the topology is saved in Elasticsearch
- - type: elasticsearch
-   enabled: true
-   host: localhost
-   port: 9200
-   save_topology: true
-
- # Redis as output
- # Options:
- # host, port: where Redis is listening on
- # save_topology: specify if the topology is saved in Redis
- - type: redis
-   enabled: false
-   host: localhost
-   port: 6379
-   save_topology: true
-
- # File as output
- # Options
- # path: where to save the files
- # filename: name of the files
- # rotate_every_kb: maximum size of the files in path
- # number of files: maximum number of files in path
- - type: file
-   enabled: false
-   path: "/tmp/packetbeat"
-   filename: packetbeat
-   rotate_every_kb: 1000
-   number_of_files: 7
+  # Elasticsearch as output
+  # Options:
+  # host, port: where Elasticsearch is listening on
+  # save_topology: specify if the topology is saved in Elasticsearch
+  elasticsearch:
+    enabled: true
+    host: localhost
+    port: 9200
+    save_topology: true
+
+  # Redis as output
+  # Options:
+  # host, port: where Redis is listening on
+  # save_topology: specify if the topology is saved in Redis
+  redis:
+    enabled: false
+    host: localhost
+    port: 6379
+    save_topology: true
+
+  # File as output
+  # Options
+  # path: where to save the files
+  # filename: name of the files
+  # rotate_every_kb: maximum size of the files in path
+  # number of files: maximum number of files in path
+  file:
+    enabled: false
+    path: "/tmp/packetbeat"
+    filename: packetbeat
+    rotate_every_kb: 1000
+    number_of_files: 7
diff --git a/procs/procs.go b/procs/procs.go
@@ -62,11 +62,12 @@ type ProcessesWatcher struct {
 type ProcsConfig struct {
 	Dont_read_from_proc bool
 	Max_proc_read_freq  int
-	Monitored           map[string]ProcConfig
+	Monitored           []ProcConfig
 	Refresh_pids_freq   int
 }
 
 type ProcConfig struct {
+	Process      string
 	Cmdline_grep string
 }
 
@@ -109,14 +110,14 @@ func (proc *ProcessesWatcher) Init(config ProcsConfig) error {
 	}
 
 	if proc.ReadFromProc {
-		for pstr, procConfig := range config.Monitored {
+		for _, procConfig := range config.Monitored {
 
 			grepper := procConfig.Cmdline_grep
 			if len(grepper) == 0 {
-				grepper = pstr
+				grepper = procConfig.Process
 			}
 
-			p, err := NewProcess(proc, pstr, grepper, time.Tick(proc.RefreshPidsFreq))
+			p, err := NewProcess(proc, procConfig.Process, grepper, time.Tick(proc.RefreshPidsFreq))
 			if err != nil {
 				logp.Err("NewProcess: %s", err)
 			} else {