From db85050ebe97ddafca07d494bbdf3974f7adb3b8 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Mon, 24 Feb 2020 18:20:09 +0200 Subject: [PATCH] Add filtering option for prometheus collector (#16420) --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/modules/prometheus.asciidoc | 3 + metricbeat/metricbeat.reference.yml | 3 + metricbeat/module/prometheus/_meta/config.yml | 3 + .../prometheus/collector/_meta/docs.asciidoc | 21 ++- .../module/prometheus/collector/collector.go | 83 ++++++++- .../prometheus/collector/collector_test.go | 160 ++++++++++++++++++ .../module/prometheus/collector/config.go | 38 +++++ metricbeat/modules.d/prometheus.yml.disabled | 3 + x-pack/metricbeat/metricbeat.reference.yml | 3 + 10 files changed, 314 insertions(+), 4 deletions(-) create mode 100644 metricbeat/module/prometheus/collector/config.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 76f8d0eb448..0c551133719 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -169,6 +169,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add Load Balancing metricset to GCP {pull}15559[15559] - Add support for Dropwizard metrics 4.1. {pull}16332[16332] - Improve the `haproxy` module to support metrics exposed via HTTPS. {issue}14579[14579] {pull}16333[16333] +- Add filtering option for prometheus collector. {pull}16420[16420] - Add metricsets based on Ceph Manager Daemon to the `ceph` module. {issue}7723[7723] {pull}16254[16254] - Release `statsd` module as GA. {pull}16447[16447] {issue}14280[14280] diff --git a/metricbeat/docs/modules/prometheus.asciidoc b/metricbeat/docs/modules/prometheus.asciidoc index f5c57b82477..d56da8a7cf9 100644 --- a/metricbeat/docs/modules/prometheus.asciidoc +++ b/metricbeat/docs/modules/prometheus.asciidoc @@ -32,6 +32,9 @@ metricbeat.modules: period: 10s hosts: ["localhost:9090"] metrics_path: /metrics + #metrics_filters: + # include: [] + # exclude: [] #username: "user" #password: "secret" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 49c51354413..84fdee94a3f 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -699,6 +699,9 @@ metricbeat.modules: period: 10s hosts: ["localhost:9090"] metrics_path: /metrics + #metrics_filters: + # include: [] + # exclude: [] #username: "user" #password: "secret" diff --git a/metricbeat/module/prometheus/_meta/config.yml b/metricbeat/module/prometheus/_meta/config.yml index 69f3219bbd2..61832ea6e97 100644 --- a/metricbeat/module/prometheus/_meta/config.yml +++ b/metricbeat/module/prometheus/_meta/config.yml @@ -2,6 +2,9 @@ period: 10s hosts: ["localhost:9090"] metrics_path: /metrics + #metrics_filters: + # include: [] + # exclude: [] #username: "user" #password: "secret" diff --git a/metricbeat/module/prometheus/collector/_meta/docs.asciidoc b/metricbeat/module/prometheus/collector/_meta/docs.asciidoc index a60e3580c15..0c93d9aac69 100644 --- a/metricbeat/module/prometheus/collector/_meta/docs.asciidoc +++ b/metricbeat/module/prometheus/collector/_meta/docs.asciidoc @@ -40,4 +40,23 @@ metricbeat.modules: metrics_path: '/federate' query: 'match[]': '{__name__!=""}' -------------------------------------------------------------------------------------- \ No newline at end of file +------------------------------------------------------------------------------------- + +[float] +=== Filtering metrics + +In order to filter out/in metrics one can make use of `metrics_filters.include` `metrics_filters.exclude` settings: + +[source,yaml] +------------------------------------------------------------------------------------- +- module: prometheus + period: 10s + hosts: ["localhost:9092"] + metrics_path: /metrics + metrics_filters: + include: ["node_filesystem_*"] + exclude: ["node_filesystem_device_*", "node_filesystem_readonly"] +------------------------------------------------------------------------------------- + +The configuration above will include only metrics that match `node_filesystem_*` pattern and do not match `node_filesystem_device_*` +and are not `node_filesystem_readonly` metric. diff --git a/metricbeat/module/prometheus/collector/collector.go b/metricbeat/module/prometheus/collector/collector.go index a189ed1c408..41a67076e52 100644 --- a/metricbeat/module/prometheus/collector/collector.go +++ b/metricbeat/module/prometheus/collector/collector.go @@ -18,7 +18,10 @@ package collector import ( + "regexp" + "github.com/pkg/errors" + dto "github.com/prometheus/client_model/go" "github.com/elastic/beats/libbeat/common" p "github.com/elastic/beats/metricbeat/helper/prometheus" @@ -49,20 +52,36 @@ func init() { // MetricSet for fetching prometheus data type MetricSet struct { mb.BaseMetricSet - prometheus p.Prometheus + prometheus p.Prometheus + includeMetrics []*regexp.Regexp + excludeMetrics []*regexp.Regexp } // New creates a new metricset func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + config := defaultConfig + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } prometheus, err := p.NewPrometheusClient(base) if err != nil { return nil, err } - return &MetricSet{ + ms := &MetricSet{ BaseMetricSet: base, prometheus: prometheus, - }, nil + } + ms.excludeMetrics, err = compilePatternList(config.MetricsFilters.ExcludeMetrics) + if err != nil { + return nil, errors.Wrapf(err, "unable to compile exclude patterns") + } + ms.includeMetrics, err = compilePatternList(config.MetricsFilters.IncludeMetrics) + if err != nil { + return nil, errors.Wrapf(err, "unable to compile include patterns") + } + + return ms, nil } // Fetch fetches data and reports it @@ -81,6 +100,9 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { } for _, family := range families { + if m.skipFamily(family) { + continue + } promEvents := getPromEventsFromMetricFamily(family) for _, promEvent := range promEvents { @@ -140,3 +162,58 @@ func (m *MetricSet) addUpEvent(eventList map[string]common.MapStr, up int) { } } + +func (m *MetricSet) skipFamily(family *dto.MetricFamily) bool { + // example: + // include_metrics: + // - node_* + // exclude_metrics: + // - node_disk_* + // + // This would mean that we want to keep only the metrics that start with node_ prefix but + // are not related to disk so we exclude node_disk_* metrics from them. + + if family == nil { + return true + } + + // if include_metrics are defined, check if this metric should be included + if len(m.includeMetrics) > 0 { + if !matchMetricFamily(*family.Name, m.includeMetrics) { + return true + } + } + // now exclude the metric if it matches any of the given patterns + if len(m.excludeMetrics) > 0 { + if matchMetricFamily(*family.Name, m.excludeMetrics) { + return true + } + } + return false +} + +func compilePatternList(patterns *[]string) ([]*regexp.Regexp, error) { + var compiledPatterns []*regexp.Regexp + compiledPatterns = []*regexp.Regexp{} + if patterns != nil { + for _, pattern := range *patterns { + r, err := regexp.Compile(pattern) + if err != nil { + return nil, errors.Wrapf(err, "compiling pattern '%s'", pattern) + } + compiledPatterns = append(compiledPatterns, r) + } + return compiledPatterns, nil + } + return []*regexp.Regexp{}, nil +} + +func matchMetricFamily(family string, matchMetrics []*regexp.Regexp) bool { + for _, checkMetric := range matchMetrics { + matched := checkMetric.MatchString(family) + if matched { + return true + } + } + return false +} diff --git a/metricbeat/module/prometheus/collector/collector_test.go b/metricbeat/module/prometheus/collector/collector_test.go index d8a96b92517..55bb2a9f109 100644 --- a/metricbeat/module/prometheus/collector/collector_test.go +++ b/metricbeat/module/prometheus/collector/collector_test.go @@ -22,6 +22,8 @@ package collector import ( "testing" + "github.com/elastic/beats/metricbeat/mb" + "github.com/golang/protobuf/proto" dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/assert" @@ -200,6 +202,164 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { } } +func TestSkipMetricFamily(t *testing.T) { + testFamilies := []*dto.MetricFamily{ + { + Name: proto.String("http_request_duration_microseconds_a_a_in"), + Help: proto.String("foo"), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("handler"), + Value: proto.String("query"), + }, + }, + Counter: &dto.Counter{ + Value: proto.Float64(10), + }, + }, + }, + }, + { + Name: proto.String("http_request_duration_microseconds_a_b_in"), + Help: proto.String("foo"), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("handler"), + Value: proto.String("query"), + }, + }, + Counter: &dto.Counter{ + Value: proto.Float64(10), + }, + }, + }, + }, + { + Name: proto.String("http_request_duration_microseconds_b_in"), + Help: proto.String("foo"), + Type: dto.MetricType_GAUGE.Enum(), + Metric: []*dto.Metric{ + { + Gauge: &dto.Gauge{ + Value: proto.Float64(10), + }, + }, + }, + }, + { + Name: proto.String("http_request_duration_microseconds_c_in"), + Help: proto.String("foo"), + Type: dto.MetricType_SUMMARY.Enum(), + Metric: []*dto.Metric{ + { + Summary: &dto.Summary{ + SampleCount: proto.Uint64(10), + SampleSum: proto.Float64(10), + Quantile: []*dto.Quantile{ + { + Quantile: proto.Float64(0.99), + Value: proto.Float64(10), + }, + }, + }, + }, + }, + }, + { + Name: proto.String("http_request_duration_microseconds_d_in"), + Help: proto.String("foo"), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(10), + SampleSum: proto.Float64(10), + Bucket: []*dto.Bucket{ + { + UpperBound: proto.Float64(0.99), + CumulativeCount: proto.Uint64(10), + }, + }, + }, + }, + }, + }, + { + Name: proto.String("http_request_duration_microseconds_e_in"), + Help: proto.String("foo"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("handler"), + Value: proto.String("query"), + }, + }, + Untyped: &dto.Untyped{ + Value: proto.Float64(10), + }, + }, + }, + }, + } + + ms := &MetricSet{ + BaseMetricSet: mb.BaseMetricSet{}, + } + + // test with no filters + ms.includeMetrics, _ = compilePatternList(&[]string{}) + ms.excludeMetrics, _ = compilePatternList(&[]string{}) + metricsToKeep := 0 + for _, testFamily := range testFamilies { + if !ms.skipFamily(testFamily) { + metricsToKeep += 1 + } + } + assert.Equal(t, metricsToKeep, len(testFamilies)) + + // test with only one include filter + ms.includeMetrics, _ = compilePatternList(&[]string{"http_request_duration_microseconds_a_*"}) + ms.excludeMetrics, _ = compilePatternList(&[]string{}) + metricsToKeep = 0 + for _, testFamily := range testFamilies { + if !ms.skipFamily(testFamily) { + metricsToKeep += 1 + } + } + assert.Equal(t, metricsToKeep, 2) + + // test with only one exclude filter + ms.includeMetrics, _ = compilePatternList(&[]string{""}) + ms.excludeMetrics, _ = compilePatternList(&[]string{"http_request_duration_microseconds_a_*"}) + metricsToKeep = 0 + for _, testFamily := range testFamilies { + if !ms.skipFamily(testFamily) { + metricsToKeep += 1 + } + } + assert.Equal(t, len(testFamilies)-2, metricsToKeep) + + // test with ine include and one exclude + ms.includeMetrics, _ = compilePatternList(&[]string{"http_request_duration_microseconds_a_*"}) + ms.excludeMetrics, _ = compilePatternList(&[]string{"http_request_duration_microseconds_a_b_*"}) + metricsToKeep = 0 + for _, testFamily := range testFamilies { + if !ms.skipFamily(testFamily) { + metricsToKeep += 1 + } + } + assert.Equal(t, 1, metricsToKeep) + +} + func TestData(t *testing.T) { mbtest.TestDataFiles(t, "prometheus", "collector") } diff --git a/metricbeat/module/prometheus/collector/config.go b/metricbeat/module/prometheus/collector/config.go new file mode 100644 index 00000000000..1a2c5688177 --- /dev/null +++ b/metricbeat/module/prometheus/collector/config.go @@ -0,0 +1,38 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package collector + +type metricsetConfig struct { + MetricsFilters MetricFilters `config:"metrics_filters" yaml:"metrics_filters,omitempty"` +} + +type MetricFilters struct { + IncludeMetrics *[]string `config:"include" yaml:"include,omitempty"` + ExcludeMetrics *[]string `config:"exclude" yaml:"exclude,omitempty"` +} + +var defaultConfig = metricsetConfig{ + MetricsFilters: MetricFilters{ + IncludeMetrics: nil, + ExcludeMetrics: nil}, +} + +func (c *metricsetConfig) Validate() error { + // validate configuration here + return nil +} diff --git a/metricbeat/modules.d/prometheus.yml.disabled b/metricbeat/modules.d/prometheus.yml.disabled index 21bc4c78800..cc5ec9397d9 100644 --- a/metricbeat/modules.d/prometheus.yml.disabled +++ b/metricbeat/modules.d/prometheus.yml.disabled @@ -5,6 +5,9 @@ period: 10s hosts: ["localhost:9090"] metrics_path: /metrics + #metrics_filters: + # include: [] + # exclude: [] #username: "user" #password: "secret" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index ac0583fe0fe..26e24fcdb8c 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -898,6 +898,9 @@ metricbeat.modules: period: 10s hosts: ["localhost:9090"] metrics_path: /metrics + #metrics_filters: + # include: [] + # exclude: [] #username: "user" #password: "secret"