From 965a80e43bc9832792d7ea9e302775cc03795209 Mon Sep 17 00:00:00 2001 From: Mario Castro Date: Wed, 18 Mar 2020 20:55:18 +0100 Subject: [PATCH] Cherry-pick #16455 to 7.x: [Metricbeat] Update Couchdb to v2 (#17078) --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 4 +- metricbeat/docs/modules/couchdb.asciidoc | 9 +- .../docs/modules/couchdb/server.asciidoc | 2 +- metricbeat/docs/modules_list.asciidoc | 2 +- metricbeat/metricbeat.reference.yml | 2 +- metricbeat/module/couchdb/_meta/docs.asciidoc | 5 +- metricbeat/module/couchdb/_meta/fields.yml | 4 +- .../couchdb/_meta/test/serverstats.json | 45 --- metricbeat/module/couchdb/fields.go | 2 +- .../module/couchdb/server/_meta/data.json | 21 +- .../module/couchdb/server/_meta/fields.yml | 6 +- metricbeat/module/couchdb/server/server.go | 105 +++++- .../couchdb/server/server_integration_test.go | 7 + .../module/couchdb/server/server_test.go | 104 ------ .../module/couchdb/server/{data.go => v1.go} | 143 ++++---- metricbeat/module/couchdb/server/v2.go | 330 ++++++++++++++++++ x-pack/metricbeat/metricbeat.reference.yml | 2 +- 18 files changed, 544 insertions(+), 250 deletions(-) delete mode 100644 metricbeat/module/couchdb/_meta/test/serverstats.json delete mode 100644 metricbeat/module/couchdb/server/server_test.go rename metricbeat/module/couchdb/server/{data.go => v1.go} (78%) create mode 100644 metricbeat/module/couchdb/server/v2.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5f7f9d79f44..2786f60e3a9 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -311,6 +311,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add Prometheus remote write endpoint {pull}16609[16609] - Release STAN module as GA. {pull}16980[16980] - Release ActiveMQ module as GA. {issue}17047[17047] {pull}17049[17049] +- Add support for CouchDB v2 {issue}16352[16352] {pull}16455[16455] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 13b3e851253..3bc3ba6d64f 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -5800,7 +5800,7 @@ type: long -- [[exported-fields-couchdb]] -== couchdb fields +== CouchDB fields couchdb module @@ -5809,7 +5809,7 @@ couchdb module [float] === couchdb - +Couchdb metrics [float] diff --git a/metricbeat/docs/modules/couchdb.asciidoc b/metricbeat/docs/modules/couchdb.asciidoc index 5867e5a02a5..aaf88063cf5 100644 --- a/metricbeat/docs/modules/couchdb.asciidoc +++ b/metricbeat/docs/modules/couchdb.asciidoc @@ -3,7 +3,7 @@ This file is generated! See scripts/mage/docs_collector.go //// [[metricbeat-module-couchdb]] -== couchdb module +== CouchDB module This is the couchdb module. @@ -12,7 +12,10 @@ The default metricset is `server`. [float] === Compatibility -The Couchdb module is tested with Couchdb 1.7. +The Couchdb module is tested in CI with Couchdb 1.7 and 2.3. Because of the differences between v1 and v2 for exposing metrics, the path to request metrics for each version is different: + +* v1.* uses `[host]:5984/_stats` so the hosts of your config should just be `[host]:5984` +* v2.* exposes metrics in various places. Local in `[host]:5986/_stats` and cluster wide in `[host]:5984/_node/[node-name]/_stats` or `[host]:5984/_node/_local/_stats`. Recommended config is `[host]:5986` to use the local path (double check that you are using port `5986`) or to use the full path on `5984` `[host]:5984/_node/[node name or _local]/_stats` [float] @@ -26,7 +29,7 @@ image::./images/metricbeat-couchdb-overview.png[] [float] === Example configuration -The couchdb module supports the standard configuration options that are described +The CouchDB module supports the standard configuration options that are described in <>. Here is an example configuration: [source,yaml] diff --git a/metricbeat/docs/modules/couchdb/server.asciidoc b/metricbeat/docs/modules/couchdb/server.asciidoc index edceac1e0e0..4e4e534d1b9 100644 --- a/metricbeat/docs/modules/couchdb/server.asciidoc +++ b/metricbeat/docs/modules/couchdb/server.asciidoc @@ -3,7 +3,7 @@ This file is generated! See scripts/mage/docs_collector.go //// [[metricbeat-metricset-couchdb-server]] -=== couchdb server metricset +=== CouchDB server metricset include::../../../module/couchdb/server/_meta/docs.asciidoc[] diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 7a0e2fa7d3f..ec5681aa596 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -70,7 +70,7 @@ This file is generated! See scripts/mage/docs_collector.go .3+| .3+| |<> |<> |<> -|<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | +|<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .9+| .9+| |<> diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 9879528f5bf..0e3ced59e9f 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -199,7 +199,7 @@ metricbeat.modules: hosts: ["localhost:8091"] enabled: true -#------------------------------- Couchdb Module ------------------------------- +#------------------------------- CouchDB Module ------------------------------- - module: couchdb metricsets: ["server"] period: 10s diff --git a/metricbeat/module/couchdb/_meta/docs.asciidoc b/metricbeat/module/couchdb/_meta/docs.asciidoc index 5cf0aa819a4..6d1ec2f49e1 100644 --- a/metricbeat/module/couchdb/_meta/docs.asciidoc +++ b/metricbeat/module/couchdb/_meta/docs.asciidoc @@ -5,7 +5,10 @@ The default metricset is `server`. [float] === Compatibility -The Couchdb module is tested with Couchdb 1.7. +The Couchdb module is tested in CI with Couchdb 1.7 and 2.3. Because of the differences between v1 and v2 for exposing metrics, the path to request metrics for each version is different: + +* v1.* uses `[host]:5984/_stats` so the hosts of your config should just be `[host]:5984` +* v2.* exposes metrics in various places. Local in `[host]:5986/_stats` and cluster wide in `[host]:5984/_node/[node-name]/_stats` or `[host]:5984/_node/_local/_stats`. Recommended config is `[host]:5986` to use the local path (double check that you are using port `5986`) or to use the full path on `5984` `[host]:5984/_node/[node name or _local]/_stats` [float] diff --git a/metricbeat/module/couchdb/_meta/fields.yml b/metricbeat/module/couchdb/_meta/fields.yml index 89770386186..7c6d7aa2de1 100644 --- a/metricbeat/module/couchdb/_meta/fields.yml +++ b/metricbeat/module/couchdb/_meta/fields.yml @@ -1,10 +1,10 @@ - key: couchdb - title: "couchdb" + title: "CouchDB" description: > couchdb module release: ga fields: - name: couchdb type: group - description: > + description: Couchdb metrics fields: diff --git a/metricbeat/module/couchdb/_meta/test/serverstats.json b/metricbeat/module/couchdb/_meta/test/serverstats.json deleted file mode 100644 index 2b4bd27d25b..00000000000 --- a/metricbeat/module/couchdb/_meta/test/serverstats.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "couchdb": { - "server": { - "couchdb": { - "open_os_files": 0, - "database_writes": 0, - "open_databases": 0, - "auth_cache_misses": 0, - "request_time": 142, - "database_reads": 0, - "auth_cache_hits": 0 - }, - "httpd": { - "requests": 3, - "viewReads": 0, - "bulk_requests": 0, - "clients_requesting_changes": 0, - "temporary_view_reads": 0 - }, - "httpd_request_methods": { - "GET": 3, - "PUT": 0, - "COPY": 0, - "HEAD": 0, - "POST": 0, - "DELETE": 0 - }, - "httpd_status_codes": { - "401": 0, - "405": 0, - "304": 0, - "400": 0, - "404": 0, - "409": 0, - "201": 0, - "202": 0, - "403": 0, - "500": 0, - "200": 3, - "301": 0, - "412": 0 - } - } - } -} diff --git a/metricbeat/module/couchdb/fields.go b/metricbeat/module/couchdb/fields.go index fb4182e2dbd..a6fc61d4264 100644 --- a/metricbeat/module/couchdb/fields.go +++ b/metricbeat/module/couchdb/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCouchdb returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/couchdb. func AssetCouchdb() string { - return "eJy8mMlu2zwQx+9+ioHvAeTtEB8+IHGc5EOzGI2DoieBJscWEYmjklQM9+kLypZ3RWECiUdJnP9Ps3CZC3jD1RA4ZTwSsxaAlTbGIbQ3T9otAIGGa5laSWoI/7UAoPgeEhJZjC0AjTEyg0NYsBbAXGIszDD/9AIUS3Bfwg27St3HmrJ08+SMyqGhfWMG9Tvq7eNz9kptrseIlGVSGRg5rpvrjUkwllmz9+XhjxXjmGufLbI2FQdvyvAqEN24n04nOZM0VnJz9P4cxj7Ku8RlqJGJ44k7ppjU4szLCqx8PGXJDDXQPNeBQ51SplkWv4Ua/2RobB1YOyqnBCdKpWA8lqisKdikWoQ8YmqB9VJuZGFOGjgpK1VGmYET7VJui0lKmulVWGu4d8RbQa+4fyrkp9M92fJ6+TjmB3VaRDtM0EZ04rjv1e3GNpy3XVW8o+fJ71rjmDM6FY8iuR9f3dQP5VQ8oCbPL9P6oZyKB9TN+GE8HdePtdbxALsbN+Csu7GPryavTcTvtQrpcGVwu25mQk7iZAf4/naeGcgNf31vb3eDoF2/17pBAM8/QKNJSZnP7EftbtBpBKwDI43MovCk6zZC14UrzjH1xes14rxe0IFHekcBE9QJU6hsvPLk7DfC2YcnsvBIQs6lpyv7jRRIPwjgmgn4udns/QibCHY/6MCrYpmNSMu/3k7sNYLYg1vSMykEKk++WvLwFHCdiLeUKV8HDhpx4AAe82NmjnkVx7T0jvRlI6CX7u49jyX3rJVOE+t2v9OFiUZOSkg3DW6ZjD0dOWhk3RkEAfyvLGrFYnhZNy/GWpOuYj1qyBz/yJfPNoW9L59oBLNsxgyGSy1tzbduKxM0wLaasGQG1lduUR1iSlGFxdR6QZ0UnEqVorllPuSMRxgm0tRN59RQWclZXiy5LhzplqIWV28XixooH1AtbJRTbm/iUhkpcNvzW0obUeaOFzySv3BWzbzN0QbaLEWKEs8SVDZPUScLc03JXu56JUUka+66nU2JA9WPy4pMOJdxRd5+u0/kJLYzSO/awBEzOUfrXwAAAP//0DVtvQ==" + return "eJy8mEtv6jgUx/d8iiP2lcJrURaVWkrb0fSBplSjWUXGPhCriZ2xnSLup79yIJAUaK5b2Vkm9vn/ch5+nAt4x80YqCxowhYdAMNNimPoTuyb25tuB4ChpornhksxhqsOAFTjIZOsSLEDoDBFonEMK9IBWHJMmR6XQy9AkAzrEvYxm9wOVrLId28aKpPKPhrFqd4NqZutm9aoPlDtX5+yfqRwVfsAMJHCEC407H57ZxK0IUbXRjZ/s3o+c9XZEmNy1vhyDq8F0T4P8/msZOLaHNzyFUYd5YPjOlZI2OeJB6ZUitWJjy1Y5fNcZAtUIJelDjR1zjItivQ9Vvh/gdr4wDpQWSU4UjoLRlOOwuiKjYtVTBMiVuiXcicLS6mASmG4KGSh4Uj7LLfBLJeKqE3sNdwH4r2gU9yDhLwsl69D3ijTKthxhiaRR377WdnubMNp2221O3mZ/effV1bFoUYepte3/qGsigPU7OV17h/KqjhA3U4fp/Opf6ytjgPY/TSAs+6nLr6avYWI31sbUnNlsJtuoWMq2dEG8PPdvNBQGv7+1t7tR1HXv9f6UQQvf4NCnUuh/2Q76vajXhCwHkwUEoPMka4fhK4P15Ri7oo3COK8QdSDJ/mBDGaoMiJQmHTjyDkMwjmEZ2ngSTK+5I6uHAYpkGEUwQ1h8M9us3cjDBHsYdSDN0EKk0jFfzk7cRAEcQB3Ui04Yygc+bzk4THgNhHvZCFcHTgK4sARPJXHzBLzOk3l2jnSl0FAL+3Ve5ly6lgrvRDr9rDXh5lCKgXjdhrcEZ46OnIUZN0ZRRH8JQwqQVJ43fYupkpJ1cZ6ujvTpPzG0abqD337QMOIIQuiMV4rbjzfuQ3PUAPZa8KaaNheuFl7hGWOIq6m+gW1UnAsdRbNrvIxJTTBOOPaN51VQ2E4JWWtlLrwSfcsanXztrHwQPmIYmWSknJ/EedCc4b7jt+am0QW9nRBE/4vLtqZ9zkaoMlSpaikRYbClClqZWGpZFbLXaekSLjnBszJlGiofl1WUsdLnnrOW6uwnyHVoQecEF1idH4HAAD//3gQco4=" } diff --git a/metricbeat/module/couchdb/server/_meta/data.json b/metricbeat/module/couchdb/server/_meta/data.json index 29c13a18b32..b6402ad7c47 100644 --- a/metricbeat/module/couchdb/server/_meta/data.json +++ b/metricbeat/module/couchdb/server/_meta/data.json @@ -1,9 +1,5 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "couchdb": { "server": { "couchdb": { @@ -13,25 +9,25 @@ "database_writes": 0, "open_databases": 0, "open_os_files": 0, - "request_time": 96 + "request_time": 12.667 }, "httpd": { "bulk_requests": 0, "clients_requesting_changes": 0, - "requests": 159, + "requests": 4, "temporary_view_reads": 0, "view_reads": 0 }, "httpd_request_methods": { "COPY": 0, "DELETE": 0, - "GET": 159, + "GET": 4, "HEAD": 0, "POST": 0, "PUT": 0 }, "httpd_status_codes": { - "200": 159, + "200": 4, "201": 0, "202": 0, "301": 0, @@ -53,10 +49,13 @@ "module": "couchdb" }, "metricset": { - "name": "server" + "name": "server", + "period": 10000 }, "service": { - "address": "127.0.0.1:5984", - "type": "couchdb" + "address": "172.19.0.2:5984", + "id": "7021f7f554d9dd612adf13a6987d1358", + "type": "couchdb", + "version": "1.7.2" } } \ No newline at end of file diff --git a/metricbeat/module/couchdb/server/_meta/fields.yml b/metricbeat/module/couchdb/server/_meta/fields.yml index 2c3ff6ae1e3..308eee5be43 100644 --- a/metricbeat/module/couchdb/server/_meta/fields.yml +++ b/metricbeat/module/couchdb/server/_meta/fields.yml @@ -30,7 +30,7 @@ Number of temporary view reads - name: requests - type: long + type: long description: > Number of HTTP requests @@ -139,7 +139,7 @@ description: > Number of HTTP 500 Internal Server Error responses - - name: couchdb + - name: couchdb type: group description: > couchdb statistics @@ -175,6 +175,6 @@ Number of authentication cache hits - name: open_os_files - type: long + type: long description: > Number of file descriptors CouchDB has open diff --git a/metricbeat/module/couchdb/server/server.go b/metricbeat/module/couchdb/server/server.go index 7207fbf08ea..5c6123cc95a 100644 --- a/metricbeat/module/couchdb/server/server.go +++ b/metricbeat/module/couchdb/server/server.go @@ -18,11 +18,17 @@ package server import ( - "github.com/pkg/errors" + "encoding/json" + "net/http" + "time" + + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/helper" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" + + "github.com/pkg/errors" ) const ( @@ -37,6 +43,10 @@ var ( }.Build() ) +type VersionStrategy interface { + MapEvent(info *CommonInfo, byt []byte) (mb.Event, error) +} + func init() { mb.Registry.MustAddMetricSet("couchdb", "server", New, mb.WithHostParser(hostParser), @@ -47,7 +57,9 @@ func init() { // MetricSet type defines all fields of the MetricSet type MetricSet struct { mb.BaseMetricSet - http *helper.HTTP + http *helper.HTTP + fetcher VersionStrategy + info *CommonInfo } // New creates a new instance of the MetricSet. @@ -61,10 +73,17 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { if err != nil { return nil, err } - return &MetricSet{ - base, - http, - }, nil + + m := &MetricSet{ + BaseMetricSet: base, + http: http, + fetcher: nil, + } + if err = m.retrieveFetcher(); err != nil { + m.Logger().Warnf("error trying to get CouchDB version: '%s'. Retrying on next fetch...", err.Error()) + } + + return m, nil } // Fetch methods implements the data gathering and data conversion to the right @@ -76,8 +95,78 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { return errors.Wrap(err, "error in http fetch") } - event, _ := eventMapping(content) - reporter.Event(mb.Event{MetricSetFields: event}) + if err = m.retrieveFetcher(); err != nil { + return errors.Wrapf(err, "error trying to get CouchDB version. Retrying on next fetch...") + } + event, err := m.fetcher.MapEvent(m.info, content) + if err != nil { + return errors.Wrap(err, "error trying to get couchdb data") + } + reporter.Event(event) return nil } + +func (m *MetricSet) retrieveFetcher() (err error) { + if m.fetcher != nil { + return nil + } + + m.info, err = m.getInfoFromCouchdbHost(m.Host()) + if err != nil { + return errors.Wrap(err, "cannot start CouchDB metricbeat module") + } + + version, err := common.NewVersion(m.info.Version) + if err != nil { + return errors.Wrap(err, "could not capture couchdb version") + } + + m.Logger().Debugf("found couchdb version %d", version.Major) + + switch version.Major { + case 1: + m.fetcher = &V1{} + case 2: + m.fetcher = &V2{} + default: + m.fetcher = nil + } + + return +} + +// CommonInfo defines the data you receive when you make a GET request to the root path of a Couchdb server +type CommonInfo struct { + Version string `json:"version"` + UUID string `json:"uuid"` +} + +// Extract basic information from "/" path in Couchdb host +func (m *MetricSet) getInfoFromCouchdbHost(h string) (*CommonInfo, error) { + c := http.DefaultClient + c.Timeout = 30 * time.Second + + hpb := parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: "/", + } + + hostdata, err := hpb.Build()(m.Module(), h) + if err != nil { + return nil, errors.Wrap(err, "error using host parser") + } + + res, err := c.Get(hostdata.URI) + if err != nil { + return nil, errors.Wrap(err, "error trying to do GET request to couchdb") + } + defer res.Body.Close() + + var info CommonInfo + if err = json.NewDecoder(res.Body).Decode(&info); err != nil { + return nil, errors.Wrap(err, "error trying to parse couchdb info") + } + + return &info, nil +} diff --git a/metricbeat/module/couchdb/server/server_integration_test.go b/metricbeat/module/couchdb/server/server_integration_test.go index 22c7a841a34..c3fc3976e96 100644 --- a/metricbeat/module/couchdb/server/server_integration_test.go +++ b/metricbeat/module/couchdb/server/server_integration_test.go @@ -20,6 +20,7 @@ package server import ( + "os" "testing" "github.com/stretchr/testify/assert" @@ -51,6 +52,12 @@ func TestData(t *testing.T) { } func getConfig(host string) map[string]interface{} { + h := os.Getenv("COUCHDB_HOST") + + if h != "" { + host = h + } + return map[string]interface{}{ "module": "couchdb", "metricsets": []string{"server"}, diff --git a/metricbeat/module/couchdb/server/server_test.go b/metricbeat/module/couchdb/server/server_test.go deleted file mode 100644 index d76b2f2c904..00000000000 --- a/metricbeat/module/couchdb/server/server_test.go +++ /dev/null @@ -1,104 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package server - -import ( - "io/ioutil" - "net/http" - "net/http/httptest" - "path/filepath" - "strings" - "testing" - "time" - - mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" - - "github.com/stretchr/testify/assert" -) - -func TestFetchEventContent(t *testing.T) { - absPath, err := filepath.Abs("../_meta/test/") - assert.NoError(t, err) - - response, err := ioutil.ReadFile(absPath + "/serverstats.json") - assert.NoError(t, err) - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - w.Header().Set("Content-Type", "application/json;") - w.Write([]byte(response)) - })) - defer server.Close() - - config := map[string]interface{}{ - "module": "couchdb", - "metricsets": []string{"server"}, - "hosts": []string{server.URL}, - } - f := mbtest.NewReportingMetricSetV2Error(t, config) - events, errs := mbtest.ReportingFetchV2Error(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) - t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0]) -} - -func TestFetchTimeout(t *testing.T) { - absPath, err := filepath.Abs("../_meta/test/") - assert.NoError(t, err) - - response, err := ioutil.ReadFile(absPath + "/serverstats.json") - assert.NoError(t, err) - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - w.Header().Set("Content-Type", "application/json;") - w.Write([]byte(response)) - <-r.Context().Done() - })) - defer server.Close() - - config := map[string]interface{}{ - "module": "couchdb", - "metricsets": []string{"server"}, - "hosts": []string{server.URL}, - "timeout": "50ms", - } - - f := mbtest.NewReportingMetricSetV2Error(t, config) - - start := time.Now() - events, errs := mbtest.ReportingFetchV2Error(f) - if len(errs) == 0 { - t.Fatalf("Expected an error, had %d. %v\n", len(errs), errs) - } - assert.Empty(t, events) - elapsed := time.Since(start) - var found bool - for _, err := range errs { - if strings.Contains(err.Error(), "request canceled (Client.Timeout exceeded") { - found = true - } - } - if !found { - assert.Failf(t, "", "expected an error containing 'request canceled (Client.Timeout exceeded'. Got %v", errs) - } - - assert.True(t, elapsed < 5*time.Second, "elapsed time: %s", elapsed.String()) -} diff --git a/metricbeat/module/couchdb/server/data.go b/metricbeat/module/couchdb/server/v1.go similarity index 78% rename from metricbeat/module/couchdb/server/data.go rename to metricbeat/module/couchdb/server/v1.go index 9b6c3c25cd7..88d9a05924b 100644 --- a/metricbeat/module/couchdb/server/data.go +++ b/metricbeat/module/couchdb/server/v1.go @@ -21,75 +21,18 @@ import ( "encoding/json" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/logp" -) - -// Server type defines all fields of the Server Metricset -type Server struct { - Httpd Httpd `json:"httpd"` - HttpdRequestMethods HttpdRequestMethods `json:"httpd_request_methods"` - HttpdStatusCodes HttpdStatusCodes `json:"httpd_status_codes"` - Couchdb Couchdb `json:"couchdb"` -} - -// Httpd type defines httpd fields of the Server Metricset -type Httpd struct { - ViewReads General `json:"view_reads"` - BulkRequests General `json:"bulk_requests"` - ClientsRequestingChanges General `json:"clients_requesting_changes"` - TemporaryViewReads General `json:"temporary_view_reads"` - Requests General `json:"requests"` -} - -// HttpdRequestMethods type defines httpd requests methods fields of the Server Metricset -type HttpdRequestMethods struct { - Copy General `json:"COPY"` - Head General `json:"HEAD"` - Post General `json:"POST"` - Delete General `json:"DELETE"` - Get General `json:"GET"` - Put General `json:"PUT"` -} - -// HttpdStatusCodes type defines httpd status codes fields of the Server Metricset -type HttpdStatusCodes struct { - Num200 General `json:"200"` - Num201 General `json:"201"` - Num202 General `json:"202"` - Num301 General `json:"301"` - Num304 General `json:"304"` - Num400 General `json:"400"` - Num401 General `json:"401"` - Num403 General `json:"403"` - Num404 General `json:"404"` - Num405 General `json:"405"` - Num409 General `json:"409"` - Num412 General `json:"412"` - Num500 General `json:"500"` -} + "github.com/elastic/beats/v7/metricbeat/mb" -// Couchdb type defines couchdb fields of the Server Metricset -type Couchdb struct { - OpenOsFiles General `json:"open_os_files"` - OpenDatabases General `json:"open_databases"` - AuthCacheHits General `json:"auth_cache_hits"` - RequestTime General `json:"request_time"` - DatabaseReads General `json:"database_reads"` - DatabaseWrites General `json:"database_writes"` - AuthCacheMisses General `json:"auth_cache_misses"` -} + "github.com/pkg/errors" +) -// General type defines common fields of the Server Metricset -type General struct { - Current float64 `json:"current"` -} +type V1 struct{} -func eventMapping(content []byte) (common.MapStr, error) { - var data Server - err := json.Unmarshal(content, &data) +func (v *V1) MapEvent(info *CommonInfo, in []byte) (mb.Event, error) { + var data ServerV1 + err := json.Unmarshal(in, &data) if err != nil { - logp.Err("Error: %+v", err) - return nil, err + return mb.Event{}, errors.Wrap(err, "error parsing v1 server JSON") } event := common.MapStr{ @@ -133,5 +76,73 @@ func eventMapping(content []byte) (common.MapStr, error) { "open_os_files": data.Couchdb.OpenOsFiles.Current, }, } - return event, nil + + ecs := common.MapStr{} + ecs.Put("service.id", info.UUID) + ecs.Put("service.version", info.Version) + + return mb.Event{ + RootFields: ecs, + MetricSetFields: event, + }, nil +} + +// Server type defines all fields of the Server Metricset +type ServerV1 struct { + Httpd HttpdV1 `json:"httpd"` + HttpdRequestMethods HttpdRequestMethodsV1 `json:"httpd_request_methods"` + HttpdStatusCodes HttpdStatusCodesV1 `json:"httpd_status_codes"` + Couchdb CouchdbV1 `json:"couchdb"` +} + +// HttpdV1 type defines httpd fields of the Server Metricset +type HttpdV1 struct { + ViewReads General `json:"view_reads"` + BulkRequests General `json:"bulk_requests"` + ClientsRequestingChanges General `json:"clients_requesting_changes"` + TemporaryViewReads General `json:"temporary_view_reads"` + Requests General `json:"requests"` +} + +// HttpdRequestMethodsV1 type defines httpd requests methods fields of the Server Metricset +type HttpdRequestMethodsV1 struct { + Copy General `json:"COPY"` + Head General `json:"HEAD"` + Post General `json:"POST"` + Delete General `json:"DELETE"` + Get General `json:"GET"` + Put General `json:"PUT"` +} + +// HttpdStatusCodesV1 type defines httpd status codes fields of the Server Metricset +type HttpdStatusCodesV1 struct { + Num200 General `json:"200"` + Num201 General `json:"201"` + Num202 General `json:"202"` + Num301 General `json:"301"` + Num304 General `json:"304"` + Num400 General `json:"400"` + Num401 General `json:"401"` + Num403 General `json:"403"` + Num404 General `json:"404"` + Num405 General `json:"405"` + Num409 General `json:"409"` + Num412 General `json:"412"` + Num500 General `json:"500"` +} + +// CouchdbV1 type defines couchdb fields of the Server Metricset +type CouchdbV1 struct { + OpenOsFiles General `json:"open_os_files"` + OpenDatabases General `json:"open_databases"` + AuthCacheHits General `json:"auth_cache_hits"` + RequestTime General `json:"request_time"` + DatabaseReads General `json:"database_reads"` + DatabaseWrites General `json:"database_writes"` + AuthCacheMisses General `json:"auth_cache_misses"` +} + +// General type defines common fields of the Server Metricset +type General struct { + Current float64 `json:"current"` } diff --git a/metricbeat/module/couchdb/server/v2.go b/metricbeat/module/couchdb/server/v2.go new file mode 100644 index 00000000000..8a575a01954 --- /dev/null +++ b/metricbeat/module/couchdb/server/v2.go @@ -0,0 +1,330 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package server + +import ( + "encoding/json" + + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/metricbeat/mb" +) + +type V2 struct{} + +func (v *V2) MapEvent(info *CommonInfo, in []byte) (mb.Event, error) { + var data ServerV2 + err := json.Unmarshal(in, &data) + if err != nil { + return mb.Event{}, errors.Wrap(err, "error parsing v2 server JSON") + } + + event := common.MapStr{ + "httpd": common.MapStr{ + "view_reads": data.Couchdb.Httpd.ViewReads.Value, + "bulk_requests": data.Couchdb.Httpd.BulkRequests.Value, + "clients_requesting_changes": data.Couchdb.Httpd.ClientsRequestingChanges.Value, + "temporary_view_reads": data.Couchdb.Httpd.TemporaryViewReads.Value, + "requests": data.Couchdb.Httpd.Requests.Value, + }, + "httpd_request_methods": common.MapStr{ + "COPY": data.Couchdb.HttpdRequestMethods.COPY.Value, + "HEAD": data.Couchdb.HttpdRequestMethods.HEAD.Value, + "POST": data.Couchdb.HttpdRequestMethods.POST.Value, + "DELETE": data.Couchdb.HttpdRequestMethods.DELETE.Value, + "GET": data.Couchdb.HttpdRequestMethods.GET.Value, + "PUT": data.Couchdb.HttpdRequestMethods.PUT.Value, + }, + "httpd_status_codes": common.MapStr{ + "200": data.Couchdb.HttpdStatusCodes.Num200.Value, + "201": data.Couchdb.HttpdStatusCodes.Num201.Value, + "202": data.Couchdb.HttpdStatusCodes.Num202.Value, + "301": data.Couchdb.HttpdStatusCodes.Num301.Value, + "304": data.Couchdb.HttpdStatusCodes.Num304.Value, + "400": data.Couchdb.HttpdStatusCodes.Num400.Value, + "401": data.Couchdb.HttpdStatusCodes.Num401.Value, + "403": data.Couchdb.HttpdStatusCodes.Num403.Value, + "404": data.Couchdb.HttpdStatusCodes.Num404.Value, + "405": data.Couchdb.HttpdStatusCodes.Num405.Value, + "409": data.Couchdb.HttpdStatusCodes.Num409.Value, + "412": data.Couchdb.HttpdStatusCodes.Num412.Value, + "500": data.Couchdb.HttpdStatusCodes.Num500.Value, + }, + "couchdb": common.MapStr{ + "database_writes": data.Couchdb.DatabaseWrites.Value, + "open_databases": data.Couchdb.OpenDatabases.Value, + "auth_cache_misses": data.Couchdb.AuthCacheMisses.Value, + "request_time": data.Couchdb.RequestTime.Value.ArithmeticMean, + "database_reads": data.Couchdb.DatabaseReads.Value, + "auth_cache_hits": data.Couchdb.AuthCacheMisses.Value, + "open_os_files": data.Couchdb.OpenOsFiles.Value, + }, + } + + ecs := common.MapStr{} + ecs.Put("service.id", info.UUID) + ecs.Put("service.version", info.Version) + + return mb.Event{ + RootFields: ecs, + MetricSetFields: event, + }, nil +} + +type ServerV2 struct { + GlobalChanges struct { + DbWrites ValueTypeDesc `json:"db_writes"` + EventDocConflict ValueTypeDesc `json:"event_doc_conflict"` + ListenerPendingUpdates ValueTypeDesc `json:"listener_pending_updates"` + Rpcs ValueTypeDesc `json:"rpcs"` + ServerPendingUpdates ValueTypeDesc `json:"server_pending_updates"` + } `json:"global_changes"` + Mem3 struct { + ShardCache struct { + Eviction ValueTypeDesc `json:"eviction"` + Hit ValueTypeDesc `json:"hit"` + Miss ValueTypeDesc `json:"miss"` + } `json:"shard_cache"` + } `json:"mem3"` + CouchLog struct { + Level struct { + Alert ValueTypeDesc `json:"alert"` + Critical ValueTypeDesc `json:"critical"` + Debug ValueTypeDesc `json:"debug"` + Emergency ValueTypeDesc `json:"emergency"` + Error ValueTypeDesc `json:"error"` + Info ValueTypeDesc `json:"info"` + Notice ValueTypeDesc `json:"notice"` + Warning ValueTypeDesc `json:"warning"` + } `json:"level"` + } `json:"couch_log"` + DdocCache struct { + Hit ValueTypeDesc `json:"hit"` + Miss ValueTypeDesc `json:"miss"` + Recovery ValueTypeDesc `json:"recovery"` + } `json:"ddoc_cache"` + Fabric struct { + Worker struct { + Timeouts ValueTypeDesc `json:"timeouts"` + } `json:"worker"` + OpenShard struct { + Timeouts ValueTypeDesc `json:"timeouts"` + } `json:"open_shard"` + ReadRepairs struct { + Success ValueTypeDesc `json:"success"` + Failure ValueTypeDesc `json:"failure"` + } `json:"read_repairs"` + DocUpdate struct { + Errors ValueTypeDesc `json:"errors"` + MismatchedErrors ValueTypeDesc `json:"mismatched_errors"` + WriteQuorumErrors ValueTypeDesc `json:"write_quorum_errors"` + } `json:"doc_update"` + } `json:"fabric"` + Couchdb struct { + Mrview struct { + MapDoc ValueTypeDesc `json:"map_doc"` + Emits ValueTypeDesc `json:"emits"` + } `json:"mrview"` + AuthCacheHits ValueTypeDesc `json:"auth_cache_hits"` + AuthCacheMisses ValueTypeDesc `json:"auth_cache_misses"` + CollectResultsTime struct { + Value AggValue `json:"value"` + Type string `json:"type"` + Desc string `json:"desc"` + } `json:"collect_results_time"` + DatabaseWrites ValueTypeDesc `json:"database_writes"` + DatabaseReads ValueTypeDesc `json:"database_reads"` + DatabasePurges ValueTypeDesc `json:"database_purges"` + DbOpenTime struct { + Value AggValue `json:"value"` + Type string `json:"type"` + Desc string `json:"desc"` + } `json:"db_open_time"` + DocumentInserts ValueTypeDesc `json:"document_inserts"` + DocumentWrites ValueTypeDesc `json:"document_writes"` + DocumentPurges struct { + Total ValueTypeDesc `json:"total"` + Success ValueTypeDesc `json:"success"` + Failure ValueTypeDesc `json:"failure"` + } `json:"document_purges"` + LocalDocumentWrites ValueTypeDesc `json:"local_document_writes"` + Httpd struct { + BulkDocs struct { + Value AggValue `json:"value"` + Type string `json:"type"` + Desc string `json:"desc"` + } `json:"bulk_docs"` + BulkRequests ValueTypeDesc `json:"bulk_requests"` + Requests ValueTypeDesc `json:"requests"` + TemporaryViewReads ValueTypeDesc `json:"temporary_view_reads"` + ViewReads ValueTypeDesc `json:"view_reads"` + ClientsRequestingChanges ValueTypeDesc `json:"clients_requesting_changes"` + PurgeRequests ValueTypeDesc `json:"purge_requests"` + AbortedRequests ValueTypeDesc `json:"aborted_requests"` + } `json:"httpd"` + HttpdRequestMethods struct { + COPY ValueTypeDesc `json:"COPY"` + DELETE ValueTypeDesc `json:"DELETE"` + GET ValueTypeDesc `json:"GET"` + HEAD ValueTypeDesc `json:"HEAD"` + OPTIONS ValueTypeDesc `json:"OPTIONS"` + POST ValueTypeDesc `json:"POST"` + PUT ValueTypeDesc `json:"PUT"` + } `json:"httpd_request_methods"` + HttpdStatusCodes struct { + Num200 ValueTypeDesc `json:"200"` + Num201 ValueTypeDesc `json:"201"` + Num202 ValueTypeDesc `json:"202"` + Num204 ValueTypeDesc `json:"204"` + Num206 ValueTypeDesc `json:"206"` + Num301 ValueTypeDesc `json:"301"` + Num302 ValueTypeDesc `json:"302"` + Num304 ValueTypeDesc `json:"304"` + Num400 ValueTypeDesc `json:"400"` + Num401 ValueTypeDesc `json:"401"` + Num403 ValueTypeDesc `json:"403"` + Num404 ValueTypeDesc `json:"404"` + Num405 ValueTypeDesc `json:"405"` + Num406 ValueTypeDesc `json:"406"` + Num409 ValueTypeDesc `json:"409"` + Num412 ValueTypeDesc `json:"412"` + Num413 ValueTypeDesc `json:"413"` + Num414 ValueTypeDesc `json:"414"` + Num415 ValueTypeDesc `json:"415"` + Num416 ValueTypeDesc `json:"416"` + Num417 ValueTypeDesc `json:"417"` + Num500 ValueTypeDesc `json:"500"` + Num501 ValueTypeDesc `json:"501"` + Num503 ValueTypeDesc `json:"503"` + } `json:"httpd_status_codes"` + OpenDatabases ValueTypeDesc `json:"open_databases"` + OpenOsFiles ValueTypeDesc `json:"open_os_files"` + RequestTime struct { + Value AggValue `json:"value"` + Type string `json:"type"` + Desc string `json:"desc"` + } `json:"request_time"` + CouchServer struct { + LruSkip ValueTypeDesc `json:"lru_skip"` + } `json:"couch_server"` + QueryServer struct { + VduRejects ValueTypeDesc `json:"vdu_rejects"` + VduProcessTime struct { + Value AggValue `json:"value"` + Type string `json:"type"` + Desc string `json:"desc"` + } `json:"vdu_process_time"` + } `json:"query_server"` + Dbinfo struct { + Value AggValue `json:"value"` + Type string `json:"type"` + Desc string `json:"desc"` + } `json:"dbinfo"` + } `json:"couchdb"` + Rexi struct { + Buffered ValueTypeDesc `json:"buffered"` + Down ValueTypeDesc `json:"down"` + Dropped ValueTypeDesc `json:"dropped"` + Streams struct { + Timeout struct { + InitStream ValueTypeDesc `json:"init_stream"` + Stream ValueTypeDesc `json:"stream"` + WaitForAck ValueTypeDesc `json:"wait_for_ack"` + } `json:"timeout"` + } `json:"streams"` + } `json:"rexi"` + Pread struct { + ExceedEOF ValueTypeDesc `json:"exceed_eof"` + ExceedLimit ValueTypeDesc `json:"exceed_limit"` + } `json:"pread"` + CouchReplicator struct { + ChangesReadFailures ValueTypeDesc `json:"changes_read_failures"` + ChangesReaderDeaths ValueTypeDesc `json:"changes_reader_deaths"` + ChangesManagerDeaths ValueTypeDesc `json:"changes_manager_deaths"` + ChangesQueueDeaths ValueTypeDesc `json:"changes_queue_deaths"` + Checkpoints struct { + Success ValueTypeDesc `json:"success"` + Failure ValueTypeDesc `json:"failure"` + } `json:"checkpoints"` + FailedStarts ValueTypeDesc `json:"failed_starts"` + Requests ValueTypeDesc `json:"requests"` + Responses struct { + Failure ValueTypeDesc `json:"failure"` + Success ValueTypeDesc `json:"success"` + } `json:"responses"` + StreamResponses struct { + Failure ValueTypeDesc `json:"failure"` + Success ValueTypeDesc `json:"success"` + } `json:"stream_responses"` + WorkerDeaths ValueTypeDesc `json:"worker_deaths"` + WorkersStarted ValueTypeDesc `json:"workers_started"` + ClusterIsStable ValueTypeDesc `json:"cluster_is_stable"` + DbScans ValueTypeDesc `json:"db_scans"` + Docs struct { + DbsCreated ValueTypeDesc `json:"dbs_created"` + DbsDeleted ValueTypeDesc `json:"dbs_deleted"` + DbsFound ValueTypeDesc `json:"dbs_found"` + DbChanges ValueTypeDesc `json:"db_changes"` + FailedStateUpdates ValueTypeDesc `json:"failed_state_updates"` + CompletedStateUpdates ValueTypeDesc `json:"completed_state_updates"` + } `json:"docs"` + Jobs struct { + Adds ValueTypeDesc `json:"adds"` + DuplicateAdds ValueTypeDesc `json:"duplicate_adds"` + Removes ValueTypeDesc `json:"removes"` + Starts ValueTypeDesc `json:"starts"` + Stops ValueTypeDesc `json:"stops"` + Crashes ValueTypeDesc `json:"crashes"` + Running ValueTypeDesc `json:"running"` + Pending ValueTypeDesc `json:"pending"` + Crashed ValueTypeDesc `json:"crashed"` + Total ValueTypeDesc `json:"total"` + } `json:"jobs"` + Connection struct { + Acquires ValueTypeDesc `json:"acquires"` + Creates ValueTypeDesc `json:"creates"` + Releases ValueTypeDesc `json:"releases"` + OwnerCrashes ValueTypeDesc `json:"owner_crashes"` + WorkerCrashes ValueTypeDesc `json:"worker_crashes"` + Closes ValueTypeDesc `json:"closes"` + } `json:"connection"` + } `json:"couch_replicator"` +} + +type ValueTypeDesc struct { + Value float64 `json:"value"` + Type string `json:"type"` + Desc string `json:"desc"` +} + +type AggValue struct { + Min float64 `json:"min"` + Max float64 `json:"max"` + ArithmeticMean float64 `json:"arithmetic_mean"` + GeometricMean float64 `json:"geometric_mean"` + HarmonicMean float64 `json:"harmonic_mean"` + Median float64 `json:"median"` + Variance float64 `json:"variance"` + StandardDeviation float64 `json:"standard_deviation"` + Skewness float64 `json:"skewness"` + Kurtosis float64 `json:"kurtosis"` + Percentile [][]float64 `json:"percentile"` + Histogram [][]float64 `json:"histogram"` + N float64 `json:"n"` +} diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 6234e6bb067..b0a869e6150 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -385,7 +385,7 @@ metricbeat.modules: hosts: ["localhost:8091"] enabled: true -#------------------------------- Couchdb Module ------------------------------- +#------------------------------- CouchDB Module ------------------------------- - module: couchdb metricsets: ["server"] period: 10s