Merge remote-tracking branch 'origin/master' into handle-streams-input

CHANGELOG.next.asciidoc

@@ -130,7 +130,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - kubernetes.container.cpu.limit.cores and kubernetes.container.cpu.requests.cores are now floats. {issue}11975[11975]
 - Change types of numeric metrics from Kubelet summary api to double so as to cover big numbers. {pull}23335[23335]
 - Add container.image.name and containe.name ECS fields for state_container. {pull}23802[23802]
-- Add support for the MemoryPressure, DiskPressure, OutOfDisk and PIDPressure status conditions in state_node. {pull}[23905]
+- Add support for Consul 1.9. {pull}24123[24123]
+- Add support for the MemoryPressure, DiskPressure, OutOfDisk and PIDPressure status conditions in state_node. {pull}23905[23905]
 - Store `cloudfoundry.container.cpu.pct` in decimal form and as `scaled_float`. {pull}24219[24219]
 
 *Packetbeat*
@@ -879,6 +880,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add mime type detection for http responses. {pull}22976[22976]
 - Bundle synthetics deps with heartbeat docker image. {pull}23274[23274]
 - Handle datastreams for fleet. {pull}24223[24223]
+- Add --sandbox option for browser monitor. {pull}24172[24172]
 
 *Journalbeat*
 
@@ -1006,6 +1008,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Enrich events of `state_service` metricset with kubernetes services' metadata. {pull}23730[23730]
 - Add support for Darwin/arm M1. {pull}24019[24019]
 - Check fields are documented in aws metricsets. {pull}23887[23887]
+- Add support for defining metrics_filters for prometheus module in hints. {pull}24264[24264]
 
 *Packetbeat*
 

metricbeat/autodiscover/builder/hints/metrics.go

@@ -38,16 +38,17 @@ func init() {
 }
 
 const (
-	module      = "module"
-	namespace   = "namespace"
-	hosts       = "hosts"
-	metricsets  = "metricsets"
-	period      = "period"
-	timeout     = "timeout"
-	ssl         = "ssl"
-	metricspath = "metrics_path"
-	username    = "username"
-	password    = "password"
+	module         = "module"
+	namespace      = "namespace"
+	hosts          = "hosts"
+	metricsets     = "metricsets"
+	period         = "period"
+	timeout        = "timeout"
+	ssl            = "ssl"
+	metricsfilters = "metrics_filters"
+	metricspath    = "metrics_path"
+	username       = "username"
+	password       = "password"
 
 	defaultTimeout = "3s"
 	defaultPeriod  = "1m"
@@ -140,6 +141,10 @@ func (m *metricHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*c
 			"processors": procs,
 		}
 
+		if mod == "prometheus" {
+			moduleConfig[metricsfilters] = m.getMetricsFilters(hint)
+		}
+
 		if ns != "" {
 			moduleConfig["namespace"] = ns
 		}
@@ -300,6 +305,14 @@ func (m *metricHints) getSSLConfig(hints common.MapStr) common.MapStr {
 	return builder.GetHintMapStr(hints, m.Key, ssl)
 }
 
+func (m *metricHints) getMetricsFilters(hints common.MapStr) common.MapStr {
+	mf := common.MapStr{}
+	for k := range builder.GetHintMapStr(hints, m.Key, metricsfilters) {
+		mf[k] = builder.GetHintAsList(hints, m.Key, metricsfilters+"."+k)
+	}
+	return mf
+}
+
 func (m *metricHints) getModuleConfigs(hints common.MapStr) []common.MapStr {
 	return builder.GetHintAsConfigs(hints, m.Key)
 }

metricbeat/autodiscover/builder/hints/metrics_test.go

@@ -548,13 +548,47 @@ func TestGenerateHints(t *testing.T) {
 				},
 			},
 		},
+		{
+			message: "exclude/exclude in metrics filters are parsed as a list",
+			event: bus.Event{
+				"host": "1.2.3.4",
+				"hints": common.MapStr{
+					"metrics": common.MapStr{
+						"module":    "prometheus",
+						"namespace": "test",
+						"hosts":     "${data.host}:9090",
+						"metrics_filters": common.MapStr{
+							"exclude": "foo, bar",
+							"include": "xxx, yyy",
+						},
+					},
+				},
+			},
+			len: 1,
+			result: []common.MapStr{
+				{
+					"module":     "prometheus",
+					"namespace":  "test",
+					"metricsets": []string{"collector"},
+					"timeout":    "3s",
+					"period":     "1m",
+					"enabled":    true,
+					"hosts":      []interface{}{"1.2.3.4:9090"},
+					"metrics_filters": map[string]interface{}{
+						"exclude": []interface{}{"foo", "bar"},
+						"include": []interface{}{"xxx", "yyy"},
+					},
+				},
+			},
+		},
 	}
 	for _, test := range tests {
 		mockRegister := mb.NewRegister()
 		mockRegister.MustAddMetricSet("mockmodule", "one", NewMockMetricSet, mb.DefaultMetricSet())
 		mockRegister.MustAddMetricSet("mockmodule", "two", NewMockMetricSet, mb.DefaultMetricSet())
 		mockRegister.MustAddMetricSet("mockmoduledefaults", "default", NewMockMetricSet, mb.DefaultMetricSet())
 		mockRegister.MustAddMetricSet("mockmoduledefaults", "other", NewMockMetricSet)
+		mockRegister.MustAddMetricSet("prometheus", "collector", NewMockMetricSet)
 
 		m := metricHints{
 			Key:      defaultConfig().Key,
@@ -679,8 +713,16 @@ func (ms *MockMetricSet) Fetch(report mb.Reporter) {
 
 }
 
+type MockPrometheus struct {
+	*MockMetricSet
+}
+
+func NewMockPrometheus(base mb.BaseMetricSet) (mb.MetricSet, error) {
+	return &MockPrometheus{}, nil
+}
+
 // create a keystore with an existing key
-/// `PASSWORD` with the value of `secret` variable.
+// `PASSWORD` with the value of `secret` variable.
 func createAnExistingKeystore(path string, secret string) keystore.Keystore {
 	keyStore, err := keystore.NewFileKeystore(path)
 	// Fail fast in the test suite

metricbeat/docs/autodiscover-hints.asciidoc

@@ -56,6 +56,18 @@ can make use of Kuberentes Secrets as described in <<kubernetes-secrets>>.
 
 SSL parameters, as seen in <<configuration-ssl>>.
 
+[float]
+===== `co.elastic.metrics/metrics_filters.*`
+
+Metrics filters (for prometheus module only).
+
+["source","yaml",subs="attributes"]
+-------------------------------------------------------------------------------------
+co.elastic.metrics/module: prometheus
+co.elastic.metrics/metrics_filters.include: node_filesystem_*
+co.elastic.metrics/metrics_filters.exclude: node_filesystem_device_foo,node_filesystem_device_bar
+-------------------------------------------------------------------------------------
+
 [float]
 ===== `co.elastic.metrics/raw`
 When an entire module configuration needs to be completely set the `raw` hint can be used. You can provide a

metricbeat/docs/modules/consul.asciidoc

@@ -12,7 +12,7 @@ This is the https://www.consul.io[Hashicorp's Consul] Metricbeat module. It is s
 [float]
 === Compatibility
 
-The module is being tested with https://github.com/hashicorp/docker-consul/blob/9bd2aa7ecf2414b8712e055f2374699148e8941c/0.X/Dockerfile[1.4.2] version
+The module is being tested with 1.4.2 and 1.9.3 versions of Consul.
 
 [float]
 === Dashboard

metricbeat/module/consul/_meta/docs.asciidoc

@@ -3,7 +3,7 @@ This is the https://www.consul.io[Hashicorp's Consul] Metricbeat module. It is s
 [float]
 === Compatibility
 
-The module is being tested with https://github.com/hashicorp/docker-consul/blob/9bd2aa7ecf2414b8712e055f2374699148e8941c/0.X/Dockerfile[1.4.2] version
+The module is being tested with 1.4.2 and 1.9.3 versions of Consul.
 
 [float]
 === Dashboard

metricbeat/module/consul/_meta/supported-versions.yml

@@ -0,0 +1,3 @@
+variants:
+  - CONSUL_VERSION: 1.4.2
+  - CONSUL_VERSION: 1.9.3

metricbeat/module/consul/docker-compose.yml

@@ -2,10 +2,10 @@ version: '2.3'
 
 services:
   consul:
-    image: docker.elastic.co/integrations-ci/beats-consul:${CONSUL_VERSION:-1.4.2}-1
+    image: docker.elastic.co/integrations-ci/beats-consul:${CONSUL_VERSION:-1.9.3}-1
     build:
       context: ./_meta
       args:
-        CONSUL_VERSION: ${CONSUL_VERSION:-1.4.2}
+        CONSUL_VERSION: ${CONSUL_VERSION:-1.9.3}
     ports:
       - 8500

metricbeat/module/consul/test_consul.py

@@ -20,6 +20,7 @@
 ]
 
 
+@metricbeat.parameterized_with_supported_versions
 class ConsulAgentTest(metricbeat.BaseTest):
 
     COMPOSE_SERVICES = ['consul']

x-pack/heartbeat/monitors/browser/browser.go

@@ -52,16 +52,21 @@ func create(name string, cfg *common.Config) (p plugin.Plugin, err error) {
 		return plugin.Plugin{}, err
 	}
 
+	extraArgs := []string{}
+	if ss.suiteCfg.Sandbox {
+		extraArgs = append(extraArgs, "--sandbox")
+	}
+
 	var j jobs.Job
 	if src, ok := ss.InlineSource(); ok {
-		j = synthexec.InlineJourneyJob(context.TODO(), src, ss.Params())
+		j = synthexec.InlineJourneyJob(context.TODO(), src, ss.Params(), extraArgs...)
 	} else {
 		j = func(event *beat.Event) ([]jobs.Job, error) {
 			err := ss.Fetch()
 			if err != nil {
 				return nil, fmt.Errorf("could not fetch for suite job: %w", err)
 			}
-			sj, err := synthexec.SuiteJob(context.TODO(), ss.Workdir(), ss.Params())
+			sj, err := synthexec.SuiteJob(context.TODO(), ss.Workdir(), ss.Params(), extraArgs...)
 			if err != nil {
 				return nil, err
 			}

x-pack/heartbeat/monitors/browser/config.go

@@ -11,6 +11,12 @@ import (
 	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser/source"
 )
 
+func DefaultConfig() *Config {
+	return &Config{
+		Sandbox: false,
+	}
+}
+
 type Config struct {
 	Schedule  string                 `config:"schedule"`
 	Params    map[string]interface{} `config:"params"`
@@ -19,7 +25,8 @@ type Config struct {
 	// Name is optional for lightweight checks but required for browsers
 	Name string `config:"name"`
 	// Id is optional for lightweight checks but required for browsers
-	Id string `config:"id"`
+	Id      string `config:"id"`
+	Sandbox bool   `config:"sandbox"`
 }
 
 var ErrNameRequired = fmt.Errorf("config 'name' must be specified for this monitor")

x-pack/heartbeat/monitors/browser/suite_runner.go

@@ -23,7 +23,7 @@ type SyntheticSuite struct {
 func NewSuite(rawCfg *common.Config) (*SyntheticSuite, error) {
 	ss := &SyntheticSuite{
 		rawCfg:   rawCfg,
-		suiteCfg: &Config{},
+		suiteCfg: DefaultConfig(),
 	}
 	err := rawCfg.Unpack(ss.suiteCfg)
 	if err != nil {

x-pack/heartbeat/monitors/browser/synthexec/synthexec.go

@@ -27,10 +27,10 @@ import (
 const debugSelector = "synthexec"
 
 // SuiteJob will run a single journey by name from the given suite.
-func SuiteJob(ctx context.Context, suitePath string, params common.MapStr) (jobs.Job, error) {
+func SuiteJob(ctx context.Context, suitePath string, params common.MapStr, extraArgs ...string) (jobs.Job, error) {
 	// Run the command in the given suitePath, use '.' as the first arg since the command runs
 	// in the correct dir
-	newCmd, err := suiteCommandFactory(suitePath, ".", "--screenshots")
+	newCmd, err := suiteCommandFactory(suitePath, append(extraArgs, ".", "--screenshots")...)
 	if err != nil {
 		return nil, err
 	}
@@ -55,9 +55,9 @@ func suiteCommandFactory(suitePath string, args ...string) (func() *exec.Cmd, er
 }
 
 // InlineJourneyJob returns a job that runs the given source as a single journey.
-func InlineJourneyJob(ctx context.Context, script string, params common.MapStr) jobs.Job {
+func InlineJourneyJob(ctx context.Context, script string, params common.MapStr, extraArgs ...string) jobs.Job {
 	newCmd := func() *exec.Cmd {
-		return exec.Command("elastic-synthetics", "--inline", "--screenshots")
+		return exec.Command("elastic-synthetics", append(extraArgs, "--inline", "--screenshots")...)
 	}
 
 	return startCmdJob(ctx, newCmd, &script, params)

x-pack/metricbeat/Jenkinsfile.yml

@@ -75,10 +75,11 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7"
-    windows-7-32:
-        mage: "mage build unitTest"
-        platforms:             ## override default labels in this specific stage.
-            - "windows-7-32-bit"
+    # windows-7-32 builder has been disabled due to https://github.com/elastic/beats/issues/24337
+    #windows-7-32:
+    #    mage: "mage build unitTest"
+    #    platforms:             ## override default labels in this specific stage.
+    #        - "windows-7-32-bit"
     packaging-linux:
         packaging-linux: "mage package"
         e2e: