From 5df9c1d2a0203bd04d424cc5fc0d39999ca05e73 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 3 Aug 2020 12:25:26 -0700 Subject: [PATCH] Support "cluster" scope in Metricbeat elasticsearch module (#18547) * Adding configuration for hosts_mode * Only perform master check in HostsModeNode * Only ask the node if it's the master node if we're in HostsModeNode * Unpack host_mode string into enum * Adding some specific TODOs in node_stats code * Updating x-pack/metricbeat reference config * Set correct service URI * Get master node ID * Adding CHANGELOG entry * Rename hosts_mode => scope * Removing stale TODO comment * Adding docs * Refactoring common code into helper method * Do not set service URI up front * Updating documentation per review * Remove comments from doc examples * Adding configuration for hosts_mode * Set correct service URI * Adding CHANGELOG entry * Rename hosts_mode => scope * Do not set service URI up front * Update metricbeat/docs/modules/elasticsearch.asciidoc Co-authored-by: DeDe Morton * Update metricbeat/module/elasticsearch/_meta/docs.asciidoc Co-authored-by: DeDe Morton * Update reference config * Cleaning up CHANGELOG * Updating generated files Co-authored-by: DeDe Morton --- CHANGELOG.next.asciidoc | 1 + .../docs/modules/elasticsearch.asciidoc | 16 ++++-- metricbeat/metricbeat.reference.yml | 5 +- .../elasticsearch/_meta/config.reference.yml | 5 +- .../module/elasticsearch/_meta/docs.asciidoc | 11 +++- metricbeat/module/elasticsearch/ccr/ccr.go | 9 ++-- .../cluster_stats/cluster_stats.go | 11 ++-- .../module/elasticsearch/elasticsearch.go | 22 ++++++++ .../module/elasticsearch/enrich/enrich.go | 9 ++-- .../module/elasticsearch/index/index.go | 10 ++-- .../index_recovery/index_recovery.go | 11 ++-- .../index_summary/index_summary.go | 9 ++-- metricbeat/module/elasticsearch/metricset.go | 54 ++++++++++++++++++- .../module/elasticsearch/ml_job/ml_job.go | 12 ++--- .../elasticsearch/node_stats/data_xpack.go | 17 +++--- .../elasticsearch/node_stats/node_stats.go | 40 +++++++++++++- .../node_stats/node_stats_test.go | 50 +++++++++++++++++ .../pending_tasks/pending_tasks.go | 11 ++-- .../module/elasticsearch/shard/shard.go | 11 ++-- x-pack/metricbeat/metricbeat.reference.yml | 5 +- 20 files changed, 222 insertions(+), 97 deletions(-) create mode 100644 metricbeat/module/elasticsearch/node_stats/node_stats_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c12c2a9574f..81e6a650ee7 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -610,6 +610,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added cache and connection_errors metrics to status metricset of MySQL module {issue}16955[16955] {pull}19844[19844] - Update MySQL dashboard with connection errors and cache metrics {pull}19913[19913] {issue}16955[16955] - Add cloud.instance.name into aws ec2 metricset. {pull}20077[20077] +- Add `scope` setting for elasticsearch module, allowing it to monitor an Elasticsearch cluster behind a load-balancing proxy. {issue}18539[18539] {pull}18547[18547] *Packetbeat* diff --git a/metricbeat/docs/modules/elasticsearch.asciidoc b/metricbeat/docs/modules/elasticsearch.asciidoc index 6dc13a8dd5c..18f9004a99a 100644 --- a/metricbeat/docs/modules/elasticsearch.asciidoc +++ b/metricbeat/docs/modules/elasticsearch.asciidoc @@ -13,7 +13,16 @@ The `elasticsearch` module collects metrics about {es}. The `elasticsearch` module works with {es} 6.7.0 and later. [float] -=== Usage for Stack Monitoring +=== Module-specific configuration notes + +Like other {beatname_uc} modules, the `elasticsearch` module accepts a `hosts` configuration setting. +This setting can contain a list of entries. The related `scope` setting determines how each entry in +the `hosts` list is interpreted by the module. + +* If `scope` is set to `node` (default), each entry in the `hosts` list indicates a distinct node in an + {es} cluster. +* If `scope` is set to `cluster`, each entry in the `hosts` list indicates a single endpoint for a distinct + {es} cluster (for example, a load-balancing proxy fronting the cluster). The `elasticsearch` module can be used to collect metrics shown in our {stack} {monitor-features} UI in {kib}. To enable this usage, set `xpack.enabled: true` and remove any `metricsets` @@ -45,12 +54,9 @@ metricbeat.modules: #password: "changeme" #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - # Set to false to fetch all entries #index_recovery.active_only: true - - # Set to true to send data collected by module to X-Pack - # Monitoring instead of metricbeat-* indices. #xpack.enabled: false + #scope: node ---- This module supports TLS connections when using `ssl` config field, as described in <>. diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 92aed4ead8f..28cf3e5c477 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -273,12 +273,9 @@ metricbeat.modules: #password: "changeme" #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - # Set to false to fetch all entries #index_recovery.active_only: true - - # Set to true to send data collected by module to X-Pack - # Monitoring instead of metricbeat-* indices. #xpack.enabled: false + #scope: node #------------------------------ Envoyproxy Module ------------------------------ - module: envoyproxy diff --git a/metricbeat/module/elasticsearch/_meta/config.reference.yml b/metricbeat/module/elasticsearch/_meta/config.reference.yml index cc446dfc6e7..61599fa2ae2 100644 --- a/metricbeat/module/elasticsearch/_meta/config.reference.yml +++ b/metricbeat/module/elasticsearch/_meta/config.reference.yml @@ -13,9 +13,6 @@ #password: "changeme" #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - # Set to false to fetch all entries #index_recovery.active_only: true - - # Set to true to send data collected by module to X-Pack - # Monitoring instead of metricbeat-* indices. #xpack.enabled: false + #scope: node diff --git a/metricbeat/module/elasticsearch/_meta/docs.asciidoc b/metricbeat/module/elasticsearch/_meta/docs.asciidoc index 0a259e61e3c..b2b1a585a69 100644 --- a/metricbeat/module/elasticsearch/_meta/docs.asciidoc +++ b/metricbeat/module/elasticsearch/_meta/docs.asciidoc @@ -6,7 +6,16 @@ The `elasticsearch` module collects metrics about {es}. The `elasticsearch` module works with {es} 6.7.0 and later. [float] -=== Usage for Stack Monitoring +=== Module-specific configuration notes + +Like other {beatname_uc} modules, the `elasticsearch` module accepts a `hosts` configuration setting. +This setting can contain a list of entries. The related `scope` setting determines how each entry in +the `hosts` list is interpreted by the module. + +* If `scope` is set to `node` (default), each entry in the `hosts` list indicates a distinct node in an + {es} cluster. +* If `scope` is set to `cluster`, each entry in the `hosts` list indicates a single endpoint for a distinct + {es} cluster (for example, a load-balancing proxy fronting the cluster). The `elasticsearch` module can be used to collect metrics shown in our {stack} {monitor-features} UI in {kib}. To enable this usage, set `xpack.enabled: true` and remove any `metricsets` diff --git a/metricbeat/module/elasticsearch/ccr/ccr.go b/metricbeat/module/elasticsearch/ccr/ccr.go index 591f3d12e22..74f7a232281 100644 --- a/metricbeat/module/elasticsearch/ccr/ccr.go +++ b/metricbeat/module/elasticsearch/ccr/ccr.go @@ -56,14 +56,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch gathers stats for each follower shard from the _ccr/stats API func (m *MetricSet) Fetch(r mb.ReporterV2) error { - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.GetServiceURI()) + shouldSkip, err := m.ShouldSkipFetch() if err != nil { - return errors.Wrap(err, "error determining if connected Elasticsearch node is master") + return err } - - // Not master, no event sent - if !isMaster { - m.Logger().Debug("trying to fetch ccr stats from a non-master node") + if shouldSkip { return nil } diff --git a/metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go b/metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go index 425ef0abacf..cd076cac83d 100644 --- a/metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go +++ b/metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go @@ -18,8 +18,6 @@ package cluster_stats import ( - "github.com/pkg/errors" - "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" ) @@ -51,14 +49,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right format func (m *MetricSet) Fetch(r mb.ReporterV2) error { - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.HostData().SanitizedURI+clusterStatsPath) + shouldSkip, err := m.ShouldSkipFetch() if err != nil { - return errors.Wrap(err, "error determining if connected Elasticsearch node is master") + return err } - - // Not master, no event sent - if !isMaster { - m.Logger().Debug("trying to fetch cluster stats from a non-master node") + if shouldSkip { return nil } diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go index 48050b224cc..52ee233d88c 100644 --- a/metricbeat/module/elasticsearch/elasticsearch.go +++ b/metricbeat/module/elasticsearch/elasticsearch.go @@ -436,6 +436,28 @@ func IsMLockAllEnabled(http *helper.HTTP, resetURI, nodeID string) (bool, error) return false, fmt.Errorf("could not determine if mlockall is enabled on node ID = %v", nodeID) } +// GetMasterNodeID returns the ID of the Elasticsearch cluster's master node +func GetMasterNodeID(http *helper.HTTP, resetURI string) (string, error) { + content, err := fetchPath(http, resetURI, "_nodes/_master", "filter_path=nodes.*.name") + if err != nil { + return "", err + } + + var response struct { + Nodes map[string]interface{} `json:"nodes"` + } + + if err := json.Unmarshal(content, &response); err != nil { + return "", err + } + + for nodeID, _ := range response.Nodes { + return nodeID, nil + } + + return "", errors.New("could not determine master node ID") +} + // PassThruField copies the field at the given path from the given source data object into // the same path in the given target data object. func PassThruField(fieldPath string, sourceData, targetData common.MapStr) error { diff --git a/metricbeat/module/elasticsearch/enrich/enrich.go b/metricbeat/module/elasticsearch/enrich/enrich.go index 6b60394a23e..c533657502a 100644 --- a/metricbeat/module/elasticsearch/enrich/enrich.go +++ b/metricbeat/module/elasticsearch/enrich/enrich.go @@ -55,14 +55,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch gathers stats for each enrich coordinator node func (m *MetricSet) Fetch(r mb.ReporterV2) error { - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.GetServiceURI()) + shouldSkip, err := m.ShouldSkipFetch() if err != nil { - return errors.Wrap(err, "error determining if connected Elasticsearch node is master") + return err } - - // Not master, no event sent - if !isMaster { - m.Logger().Debug("trying to fetch enrich stats from a non-master node") + if shouldSkip { return nil } diff --git a/metricbeat/module/elasticsearch/index/index.go b/metricbeat/module/elasticsearch/index/index.go index 372f9a2dc82..221e78ccfea 100644 --- a/metricbeat/module/elasticsearch/index/index.go +++ b/metricbeat/module/elasticsearch/index/index.go @@ -59,15 +59,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch gathers stats for each index from the _stats API func (m *MetricSet) Fetch(r mb.ReporterV2) error { - - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.HostData().SanitizedURI+statsPath) + shouldSkip, err := m.ShouldSkipFetch() if err != nil { - return errors.Wrap(err, "error determining if connected Elasticsearch node is master") + return err } - - // Not master, no event sent - if !isMaster { - m.Logger().Debug("trying to fetch index stats from a non-master node") + if shouldSkip { return nil } diff --git a/metricbeat/module/elasticsearch/index_recovery/index_recovery.go b/metricbeat/module/elasticsearch/index_recovery/index_recovery.go index 68d1ee295d3..e30463e3848 100644 --- a/metricbeat/module/elasticsearch/index_recovery/index_recovery.go +++ b/metricbeat/module/elasticsearch/index_recovery/index_recovery.go @@ -18,8 +18,6 @@ package index_recovery import ( - "github.com/pkg/errors" - "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" ) @@ -67,14 +65,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch gathers stats for each index from the _stats API func (m *MetricSet) Fetch(r mb.ReporterV2) error { - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.GetServiceURI()) + shouldSkip, err := m.ShouldSkipFetch() if err != nil { - return errors.Wrap(err, "error determining if connected Elasticsearch node is master") + return err } - - // Not master, no event sent - if !isMaster { - m.Logger().Debug("trying to fetch index recovery stats from a non-master node") + if shouldSkip { return nil } diff --git a/metricbeat/module/elasticsearch/index_summary/index_summary.go b/metricbeat/module/elasticsearch/index_summary/index_summary.go index 569e23492cb..dc3dbdd7207 100644 --- a/metricbeat/module/elasticsearch/index_summary/index_summary.go +++ b/metricbeat/module/elasticsearch/index_summary/index_summary.go @@ -62,14 +62,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch gathers stats for each index from the _stats API func (m *MetricSet) Fetch(r mb.ReporterV2) error { - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.HostData().SanitizedURI+statsPath) + shouldSkip, err := m.ShouldSkipFetch() if err != nil { - return errors.Wrap(err, "error determining if connected Elasticsearch node is master") + return err } - - // Not master, no event sent - if !isMaster { - m.Logger().Debug("trying to fetch index summary stats from a non-master node") + if shouldSkip { return nil } diff --git a/metricbeat/module/elasticsearch/metricset.go b/metricbeat/module/elasticsearch/metricset.go index c1daae4f577..22b4b2c6c49 100644 --- a/metricbeat/module/elasticsearch/metricset.go +++ b/metricbeat/module/elasticsearch/metricset.go @@ -18,6 +18,10 @@ package elasticsearch import ( + "fmt" + + "github.com/pkg/errors" + "github.com/elastic/beats/v7/metricbeat/helper" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" @@ -36,6 +40,31 @@ var ( }.Build() ) +type Scope int + +const ( + // Indicates that each item in the hosts list points to a distinct Elasticsearch node in a + // cluster. + ScopeNode Scope = iota + + // Indicates that each item in the hosts lists points to a endpoint for a distinct Elasticsearch + // cluster (e.g. a load-balancing proxy) fronting the cluster. + ScopeCluster +) + +func (h *Scope) Unpack(str string) error { + switch str { + case "node": + *h = ScopeNode + case "cluster": + *h = ScopeCluster + default: + return fmt.Errorf("invalid scope: %v", str) + } + + return nil +} + // MetricSet can be used to build other metric sets that query RabbitMQ // management plugin type MetricSet struct { @@ -43,6 +72,7 @@ type MetricSet struct { servicePath string *helper.HTTP XPack bool + Scope Scope } // NewMetricSet creates an metric set that can be used to build other metric @@ -54,9 +84,11 @@ func NewMetricSet(base mb.BaseMetricSet, servicePath string) (*MetricSet, error) } config := struct { - XPack bool `config:"xpack.enabled"` + XPack bool `config:"xpack.enabled"` + Scope Scope `config:"scope"` }{ XPack: false, + Scope: ScopeNode, } if err := base.Module().UnpackConfig(&config); err != nil { return nil, err @@ -67,6 +99,7 @@ func NewMetricSet(base mb.BaseMetricSet, servicePath string) (*MetricSet, error) servicePath, http, config.XPack, + config.Scope, } ms.SetServiceURI(servicePath) @@ -84,3 +117,22 @@ func (m *MetricSet) SetServiceURI(servicePath string) { m.servicePath = servicePath m.HTTP.SetURI(m.GetServiceURI()) } + +func (m *MetricSet) ShouldSkipFetch() (bool, error) { + // If we're talking to a set of ES nodes directly, only collect stats from the master node so + // we don't collect the same stats from every node and end up duplicating them. + if m.Scope == ScopeNode { + isMaster, err := IsMaster(m.HTTP, m.GetServiceURI()) + if err != nil { + return false, errors.Wrap(err, "error determining if connected Elasticsearch node is master") + } + + // Not master, no event sent + if !isMaster { + m.Logger().Debugf("trying to fetch %v stats from a non-master node", m.Name()) + return true, nil + } + } + + return false, nil +} diff --git a/metricbeat/module/elasticsearch/ml_job/ml_job.go b/metricbeat/module/elasticsearch/ml_job/ml_job.go index 5f4dbe9f002..d5d58b2d2c6 100644 --- a/metricbeat/module/elasticsearch/ml_job/ml_job.go +++ b/metricbeat/module/elasticsearch/ml_job/ml_job.go @@ -18,8 +18,6 @@ package ml_job import ( - "github.com/pkg/errors" - "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" ) @@ -54,15 +52,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right format func (m *MetricSet) Fetch(r mb.ReporterV2) error { - - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.GetServiceURI()) + shouldSkip, err := m.ShouldSkipFetch() if err != nil { - return errors.Wrap(err, "error determining if connected Elasticsearch node is master") + return err } - - // Not master, no event sent - if !isMaster { - m.Logger().Debug("trying to fetch machine learning job stats from a non-master node") + if shouldSkip { return nil } diff --git a/metricbeat/module/elasticsearch/node_stats/data_xpack.go b/metricbeat/module/elasticsearch/node_stats/data_xpack.go index 53340103176..e4efeb3b8e8 100644 --- a/metricbeat/module/elasticsearch/node_stats/data_xpack.go +++ b/metricbeat/module/elasticsearch/node_stats/data_xpack.go @@ -187,18 +187,14 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, return errors.Wrap(err, "failure parsing Elasticsearch Node Stats API response") } - // Normally the nodeStruct should only contain one node. But if _local is removed - // from the path and Metricbeat is not installed on the same machine as the node - // it will provid the data for multiple nodes. This will mean the detection of the - // master node will not be accurate anymore as often in these cases a proxy is in front - // of ES and it's not know if the request will be routed to the same node as before. + masterNodeID, err := elasticsearch.GetMasterNodeID(m.HTTP, m.HTTP.GetURI()) + if err != nil { + return err + } + var errs multierror.Errors for nodeID, node := range nodesStruct.Nodes { - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.HTTP.GetURI()) - if err != nil { - errs = append(errs, errors.Wrap(err, "error determining if connected Elasticsearch node is master")) - continue - } + isMaster := nodeID == masterNodeID event := mb.Event{} @@ -207,6 +203,7 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, errs = append(errs, errors.Wrap(err, "failure to apply node schema")) continue } + nodeData["node_master"] = isMaster nodeData["node_id"] = nodeID diff --git a/metricbeat/module/elasticsearch/node_stats/node_stats.go b/metricbeat/module/elasticsearch/node_stats/node_stats.go index 7498948fd76..5f856e2eeac 100644 --- a/metricbeat/module/elasticsearch/node_stats/node_stats.go +++ b/metricbeat/module/elasticsearch/node_stats/node_stats.go @@ -18,6 +18,8 @@ package node_stats import ( + "net/url" + "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" ) @@ -33,7 +35,8 @@ func init() { } const ( - nodeStatsPath = "/_nodes/_local/stats" + nodeLocalStatsPath = "/_nodes/_local/stats" + nodesAllStatsPath = "/_nodes/_all/stats" ) // MetricSet type defines all fields of the MetricSet @@ -44,7 +47,7 @@ type MetricSet struct { // New create a new instance of the MetricSet func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Get the stats from the local node - ms, err := elasticsearch.NewMetricSet(base, nodeStatsPath) + ms, err := elasticsearch.NewMetricSet(base, "") // servicePath will be set in Fetch() if err != nil { return nil, err } @@ -53,6 +56,14 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right format func (m *MetricSet) Fetch(r mb.ReporterV2) error { + if err := m.updateServiceURI(); err != nil { + if m.XPack { + m.Logger().Error(err) + return nil + } + return err + } + content, err := m.HTTP.FetchContent() if err != nil { return err @@ -78,3 +89,28 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return nil } + +func (m *MetricSet) updateServiceURI() error { + u, err := getServiceURI(m.GetURI(), m.Scope) + if err != nil { + return err + } + + m.HTTP.SetURI(u) + return nil + +} + +func getServiceURI(currURI string, scope elasticsearch.Scope) (string, error) { + u, err := url.Parse(currURI) + if err != nil { + return "", err + } + + u.Path = nodeLocalStatsPath + if scope == elasticsearch.ScopeCluster { + u.Path = nodesAllStatsPath + } + + return u.String(), nil +} diff --git a/metricbeat/module/elasticsearch/node_stats/node_stats_test.go b/metricbeat/module/elasticsearch/node_stats/node_stats_test.go new file mode 100644 index 00000000000..120493947b5 --- /dev/null +++ b/metricbeat/module/elasticsearch/node_stats/node_stats_test.go @@ -0,0 +1,50 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package node_stats + +import ( + "testing" + + "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" + + "github.com/stretchr/testify/require" +) + +func TestGetServiceURI(t *testing.T) { + tests := map[string]struct { + scope elasticsearch.Scope + expectedURI string + }{ + "scope_node": { + scope: elasticsearch.ScopeNode, + expectedURI: "/_nodes/_local/stats", + }, + "scope_cluster": { + scope: elasticsearch.ScopeCluster, + expectedURI: "/_nodes/_all/stats", + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + newURI, err := getServiceURI("/foo/bar", test.scope) + require.NoError(t, err) + require.Equal(t, test.expectedURI, newURI) + }) + } +} diff --git a/metricbeat/module/elasticsearch/pending_tasks/pending_tasks.go b/metricbeat/module/elasticsearch/pending_tasks/pending_tasks.go index 01f11e76318..a25ffb2b9a8 100644 --- a/metricbeat/module/elasticsearch/pending_tasks/pending_tasks.go +++ b/metricbeat/module/elasticsearch/pending_tasks/pending_tasks.go @@ -18,8 +18,6 @@ package pending_tasks import ( - "github.com/pkg/errors" - "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" ) @@ -59,14 +57,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right format func (m *MetricSet) Fetch(r mb.ReporterV2) error { - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.GetServiceURI()) + shouldSkip, err := m.ShouldSkipFetch() if err != nil { - return errors.Wrap(err, "error determining if connected Elasticsearch node is master") + return err } - - // Not master, no event sent - if !isMaster { - m.Logger().Debug("trying to fetch pending tasks from a non-master node") + if shouldSkip { return nil } diff --git a/metricbeat/module/elasticsearch/shard/shard.go b/metricbeat/module/elasticsearch/shard/shard.go index fa46777dffd..4367810a8ca 100644 --- a/metricbeat/module/elasticsearch/shard/shard.go +++ b/metricbeat/module/elasticsearch/shard/shard.go @@ -18,8 +18,6 @@ package shard import ( - "github.com/pkg/errors" - "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" ) @@ -53,14 +51,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right format func (m *MetricSet) Fetch(r mb.ReporterV2) error { - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.HostData().SanitizedURI+statePath) + shouldSkip, err := m.ShouldSkipFetch() if err != nil { - return errors.Wrap(err, "error determining if connected Elasticsearch node is master") + return err } - - // Not master, no event sent - if !isMaster { - m.Logger().Debug("trying to fetch shard stats from a non-master node") + if shouldSkip { return nil } diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 141a1cc79b2..457a2c2bfe7 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -482,12 +482,9 @@ metricbeat.modules: #password: "changeme" #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - # Set to false to fetch all entries #index_recovery.active_only: true - - # Set to true to send data collected by module to X-Pack - # Monitoring instead of metricbeat-* indices. #xpack.enabled: false + #scope: node #------------------------------ Envoyproxy Module ------------------------------ - module: envoyproxy