Add filtering option for prometheus collector (elastic#16420)

CHANGELOG.next.asciidoc

@@ -169,6 +169,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add Load Balancing metricset to GCP {pull}15559[15559]
 - Add support for Dropwizard metrics 4.1. {pull}16332[16332]
 - Improve the `haproxy` module to support metrics exposed via HTTPS. {issue}14579[14579] {pull}16333[16333]
+- Add filtering option for prometheus collector. {pull}16420[16420]
 - Add metricsets based on Ceph Manager Daemon to the `ceph` module. {issue}7723[7723] {pull}16254[16254]
 - Release `statsd` module as GA. {pull}16447[16447] {issue}14280[14280]
 

metricbeat/docs/modules/prometheus.asciidoc

@@ -32,6 +32,9 @@ metricbeat.modules:
   period: 10s
   hosts: ["localhost:9090"]
   metrics_path: /metrics
+  #metrics_filters:
+  #  include: []
+  #  exclude: []
   #username: "user"
   #password: "secret"
 

metricbeat/metricbeat.reference.yml

@@ -699,6 +699,9 @@ metricbeat.modules:
   period: 10s
   hosts: ["localhost:9090"]
   metrics_path: /metrics
+  #metrics_filters:
+  #  include: []
+  #  exclude: []
   #username: "user"
   #password: "secret"
 

metricbeat/module/prometheus/_meta/config.yml

@@ -2,6 +2,9 @@
   period: 10s
   hosts: ["localhost:9090"]
   metrics_path: /metrics
+  #metrics_filters:
+  #  include: []
+  #  exclude: []
   #username: "user"
   #password: "secret"
 

metricbeat/module/prometheus/collector/_meta/docs.asciidoc

@@ -40,4 +40,23 @@ metricbeat.modules:
   metrics_path: '/federate'
   query:
     'match[]': '{__name__!=""}'
--------------------------------------------------------------------------------------
+-------------------------------------------------------------------------------------
+
+[float]
+=== Filtering metrics
+
+In order to filter out/in metrics one can make use of `metrics_filters.include` `metrics_filters.exclude` settings:
+
+[source,yaml]
+-------------------------------------------------------------------------------------
+- module: prometheus
+  period: 10s
+  hosts: ["localhost:9092"]
+  metrics_path: /metrics
+  metrics_filters:
+    include: ["node_filesystem_*"]
+    exclude: ["node_filesystem_device_*", "node_filesystem_readonly"]
+-------------------------------------------------------------------------------------
+
+The configuration above will include only metrics that match `node_filesystem_*` pattern and do not match `node_filesystem_device_*`
+and are not `node_filesystem_readonly` metric.

metricbeat/module/prometheus/collector/collector.go

@@ -18,7 +18,10 @@
 package collector
 
 import (
+	"regexp"
+
 	"github.com/pkg/errors"
+	dto "github.com/prometheus/client_model/go"
 
 	"github.com/elastic/beats/libbeat/common"
 	p "github.com/elastic/beats/metricbeat/helper/prometheus"
@@ -49,20 +52,36 @@ func init() {
 // MetricSet for fetching prometheus data
 type MetricSet struct {
 	mb.BaseMetricSet
-	prometheus p.Prometheus
+	prometheus     p.Prometheus
+	includeMetrics []*regexp.Regexp
+	excludeMetrics []*regexp.Regexp
 }
 
 // New creates a new metricset
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
+	config := defaultConfig
+	if err := base.Module().UnpackConfig(&config); err != nil {
+		return nil, err
+	}
 	prometheus, err := p.NewPrometheusClient(base)
 	if err != nil {
 		return nil, err
 	}
 
-	return &MetricSet{
+	ms := &MetricSet{
 		BaseMetricSet: base,
 		prometheus:    prometheus,
-	}, nil
+	}
+	ms.excludeMetrics, err = compilePatternList(config.MetricsFilters.ExcludeMetrics)
+	if err != nil {
+		return nil, errors.Wrapf(err, "unable to compile exclude patterns")
+	}
+	ms.includeMetrics, err = compilePatternList(config.MetricsFilters.IncludeMetrics)
+	if err != nil {
+		return nil, errors.Wrapf(err, "unable to compile include patterns")
+	}
+
+	return ms, nil
 }
 
 // Fetch fetches data and reports it
@@ -81,6 +100,9 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error {
 	}
 
 	for _, family := range families {
+		if m.skipFamily(family) {
+			continue
+		}
 		promEvents := getPromEventsFromMetricFamily(family)
 
 		for _, promEvent := range promEvents {
@@ -140,3 +162,58 @@ func (m *MetricSet) addUpEvent(eventList map[string]common.MapStr, up int) {
 	}
 
 }
+
+func (m *MetricSet) skipFamily(family *dto.MetricFamily) bool {
+	// example:
+	//	include_metrics:
+	//		- node_*
+	//	exclude_metrics:
+	//		- node_disk_*
+	//
+	// This would mean that we want to keep only the metrics that start with node_ prefix but
+	// are not related to disk so we exclude node_disk_* metrics from them.
+
+	if family == nil {
+		return true
+	}
+
+	// if include_metrics are defined, check if this metric should be included
+	if len(m.includeMetrics) > 0 {
+		if !matchMetricFamily(*family.Name, m.includeMetrics) {
+			return true
+		}
+	}
+	// now exclude the metric if it matches any of the given patterns
+	if len(m.excludeMetrics) > 0 {
+		if matchMetricFamily(*family.Name, m.excludeMetrics) {
+			return true
+		}
+	}
+	return false
+}
+
+func compilePatternList(patterns *[]string) ([]*regexp.Regexp, error) {
+	var compiledPatterns []*regexp.Regexp
+	compiledPatterns = []*regexp.Regexp{}
+	if patterns != nil {
+		for _, pattern := range *patterns {
+			r, err := regexp.Compile(pattern)
+			if err != nil {
+				return nil, errors.Wrapf(err, "compiling pattern '%s'", pattern)
+			}
+			compiledPatterns = append(compiledPatterns, r)
+		}
+		return compiledPatterns, nil
+	}
+	return []*regexp.Regexp{}, nil
+}
+
+func matchMetricFamily(family string, matchMetrics []*regexp.Regexp) bool {
+	for _, checkMetric := range matchMetrics {
+		matched := checkMetric.MatchString(family)
+		if matched {
+			return true
+		}
+	}
+	return false
+}

metricbeat/module/prometheus/collector/collector_test.go

@@ -22,6 +22,8 @@ package collector
 import (
 	"testing"
 
+	"github.com/elastic/beats/metricbeat/mb"
+
 	"github.com/golang/protobuf/proto"
 	dto "github.com/prometheus/client_model/go"
 	"github.com/stretchr/testify/assert"
@@ -200,6 +202,164 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) {
 	}
 }
 
+func TestSkipMetricFamily(t *testing.T) {
+	testFamilies := []*dto.MetricFamily{
+		{
+			Name: proto.String("http_request_duration_microseconds_a_a_in"),
+			Help: proto.String("foo"),
+			Type: dto.MetricType_COUNTER.Enum(),
+			Metric: []*dto.Metric{
+				{
+					Label: []*dto.LabelPair{
+						{
+							Name:  proto.String("handler"),
+							Value: proto.String("query"),
+						},
+					},
+					Counter: &dto.Counter{
+						Value: proto.Float64(10),
+					},
+				},
+			},
+		},
+		{
+			Name: proto.String("http_request_duration_microseconds_a_b_in"),
+			Help: proto.String("foo"),
+			Type: dto.MetricType_COUNTER.Enum(),
+			Metric: []*dto.Metric{
+				{
+					Label: []*dto.LabelPair{
+						{
+							Name:  proto.String("handler"),
+							Value: proto.String("query"),
+						},
+					},
+					Counter: &dto.Counter{
+						Value: proto.Float64(10),
+					},
+				},
+			},
+		},
+		{
+			Name: proto.String("http_request_duration_microseconds_b_in"),
+			Help: proto.String("foo"),
+			Type: dto.MetricType_GAUGE.Enum(),
+			Metric: []*dto.Metric{
+				{
+					Gauge: &dto.Gauge{
+						Value: proto.Float64(10),
+					},
+				},
+			},
+		},
+		{
+			Name: proto.String("http_request_duration_microseconds_c_in"),
+			Help: proto.String("foo"),
+			Type: dto.MetricType_SUMMARY.Enum(),
+			Metric: []*dto.Metric{
+				{
+					Summary: &dto.Summary{
+						SampleCount: proto.Uint64(10),
+						SampleSum:   proto.Float64(10),
+						Quantile: []*dto.Quantile{
+							{
+								Quantile: proto.Float64(0.99),
+								Value:    proto.Float64(10),
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			Name: proto.String("http_request_duration_microseconds_d_in"),
+			Help: proto.String("foo"),
+			Type: dto.MetricType_HISTOGRAM.Enum(),
+			Metric: []*dto.Metric{
+				{
+					Histogram: &dto.Histogram{
+						SampleCount: proto.Uint64(10),
+						SampleSum:   proto.Float64(10),
+						Bucket: []*dto.Bucket{
+							{
+								UpperBound:      proto.Float64(0.99),
+								CumulativeCount: proto.Uint64(10),
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			Name: proto.String("http_request_duration_microseconds_e_in"),
+			Help: proto.String("foo"),
+			Type: dto.MetricType_UNTYPED.Enum(),
+			Metric: []*dto.Metric{
+				{
+					Label: []*dto.LabelPair{
+						{
+							Name:  proto.String("handler"),
+							Value: proto.String("query"),
+						},
+					},
+					Untyped: &dto.Untyped{
+						Value: proto.Float64(10),
+					},
+				},
+			},
+		},
+	}
+
+	ms := &MetricSet{
+		BaseMetricSet: mb.BaseMetricSet{},
+	}
+
+	// test with no filters
+	ms.includeMetrics, _ = compilePatternList(&[]string{})
+	ms.excludeMetrics, _ = compilePatternList(&[]string{})
+	metricsToKeep := 0
+	for _, testFamily := range testFamilies {
+		if !ms.skipFamily(testFamily) {
+			metricsToKeep += 1
+		}
+	}
+	assert.Equal(t, metricsToKeep, len(testFamilies))
+
+	// test with only one include filter
+	ms.includeMetrics, _ = compilePatternList(&[]string{"http_request_duration_microseconds_a_*"})
+	ms.excludeMetrics, _ = compilePatternList(&[]string{})
+	metricsToKeep = 0
+	for _, testFamily := range testFamilies {
+		if !ms.skipFamily(testFamily) {
+			metricsToKeep += 1
+		}
+	}
+	assert.Equal(t, metricsToKeep, 2)
+
+	// test with only one exclude filter
+	ms.includeMetrics, _ = compilePatternList(&[]string{""})
+	ms.excludeMetrics, _ = compilePatternList(&[]string{"http_request_duration_microseconds_a_*"})
+	metricsToKeep = 0
+	for _, testFamily := range testFamilies {
+		if !ms.skipFamily(testFamily) {
+			metricsToKeep += 1
+		}
+	}
+	assert.Equal(t, len(testFamilies)-2, metricsToKeep)
+
+	// test with ine include and one exclude
+	ms.includeMetrics, _ = compilePatternList(&[]string{"http_request_duration_microseconds_a_*"})
+	ms.excludeMetrics, _ = compilePatternList(&[]string{"http_request_duration_microseconds_a_b_*"})
+	metricsToKeep = 0
+	for _, testFamily := range testFamilies {
+		if !ms.skipFamily(testFamily) {
+			metricsToKeep += 1
+		}
+	}
+	assert.Equal(t, 1, metricsToKeep)
+
+}
+
 func TestData(t *testing.T) {
 	mbtest.TestDataFiles(t, "prometheus", "collector")
 }

metricbeat/module/prometheus/collector/config.go

@@ -0,0 +1,38 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package collector
+
+type metricsetConfig struct {
+	MetricsFilters MetricFilters `config:"metrics_filters" yaml:"metrics_filters,omitempty"`
+}
+
+type MetricFilters struct {
+	IncludeMetrics *[]string `config:"include" yaml:"include,omitempty"`
+	ExcludeMetrics *[]string `config:"exclude" yaml:"exclude,omitempty"`
+}
+
+var defaultConfig = metricsetConfig{
+	MetricsFilters: MetricFilters{
+		IncludeMetrics: nil,
+		ExcludeMetrics: nil},
+}
+
+func (c *metricsetConfig) Validate() error {
+	// validate configuration here
+	return nil
+}