From 3895a23faa77e3e172db346f46a6500f2dc3f78a Mon Sep 17 00:00:00 2001 From: Mario Castro Date: Mon, 28 Jan 2019 15:12:06 +0100 Subject: [PATCH] server Metricset for Zookeeper Metricbeat module (#10341) --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 124 ++++++++++ metricbeat/docs/modules/zookeeper.asciidoc | 8 +- .../docs/modules/zookeeper/server.asciidoc | 21 ++ metricbeat/docs/modules_list.asciidoc | 3 +- metricbeat/include/list.go | 1 + metricbeat/metricbeat.reference.yml | 2 +- .../zookeeper/_meta/config.reference.yml | 2 +- metricbeat/module/zookeeper/_meta/config.yml | 1 + .../module/zookeeper/_meta/docs.asciidoc | 2 +- metricbeat/module/zookeeper/fields.go | 2 +- .../module/zookeeper/server/_meta/data.json | 39 +++ .../zookeeper/server/_meta/docs.asciidoc | 15 ++ .../module/zookeeper/server/_meta/fields.yml | 47 ++++ metricbeat/module/zookeeper/server/data.go | 234 ++++++++++++++++++ .../zookeeper/server/data_integration_test.go | 54 ++++ metricbeat/module/zookeeper/server/server.go | 100 ++++++++ .../server/server_integration_test.go | 67 +++++ .../module/zookeeper/server/server_test.go | 70 ++++++ metricbeat/modules.d/zookeeper.yml.disabled | 1 + metricbeat/tests/system/test_zookeeper.py | 28 +++ x-pack/metricbeat/metricbeat.reference.yml | 2 +- .../performance/data_integration_test.go | 1 + 23 files changed, 817 insertions(+), 8 deletions(-) create mode 100644 metricbeat/docs/modules/zookeeper/server.asciidoc create mode 100644 metricbeat/module/zookeeper/server/_meta/data.json create mode 100644 metricbeat/module/zookeeper/server/_meta/docs.asciidoc create mode 100644 metricbeat/module/zookeeper/server/_meta/fields.yml create mode 100644 metricbeat/module/zookeeper/server/data.go create mode 100644 metricbeat/module/zookeeper/server/data_integration_test.go create mode 100644 metricbeat/module/zookeeper/server/server.go create mode 100644 metricbeat/module/zookeeper/server/server_integration_test.go create mode 100644 metricbeat/module/zookeeper/server/server_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b1151e645fc..ebb406f8358 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -228,6 +228,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Release use of xpack.enabled: true flag in Elasticsearch and Kibana modules as GA. {pull}10222[10222] - Add support for MySQL 8.0 and tests also for Percona and MariaDB. {pull}10261[10261] - Rename 'db' Metricset to 'transaction_log' in MSSQL Metricbeat module {pull}10109[10109] +- Added 'server' Metricset to Zookeeper Metricbeat module {issue}8938[8938] {pull}10341[10341] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 5c651924e16..aae6b1282c6 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -25689,3 +25689,127 @@ Number of znodes reported by the local ZooKeeper process. -- +[float] +== server fields + +server contains the metrics reported by the four-letter `srvr` command. + + +*`zookeeper.server.connections`*:: ++ +-- +type: long + +Connections established by the server + +-- + + +*`zookeeper.server.latency.avg`*:: ++ +-- +type: long + +Average latency of the server + +-- + +*`zookeeper.server.latency.max`*:: ++ +-- +type: long + +Max latency reached by the server + +-- + +*`zookeeper.server.latency.min`*:: ++ +-- +type: long + +Minimum latency that has been reached by the server + +-- + +*`zookeeper.server.mode`*:: ++ +-- +type: keyword + +Server mode + +-- + +*`zookeeper.server.node_count`*:: ++ +-- +type: long + +Total number of nodes + +-- + +*`zookeeper.server.outstanding`*:: ++ +-- +type: long + +Outstanding + +-- + +*`zookeeper.server.received`*:: ++ +-- +type: long + +Received requests to the server + +-- + +*`zookeeper.server.sent`*:: ++ +-- +type: long + +Requests sent by the server + +-- + +*`zookeeper.server.version_date`*:: ++ +-- +type: date + +Date of the Zookeeper release in use + +-- + +*`zookeeper.server.zxid`*:: ++ +-- +type: keyword + +Original value of the Zookeeper transaction ID + +-- + +*`zookeeper.server.count`*:: ++ +-- +type: long + +Total transactions of the leader in epoch + +-- + +*`zookeeper.server.epoch`*:: ++ +-- +type: long + +Epoch value of the Zookeeper transaction ID + +-- + diff --git a/metricbeat/docs/modules/zookeeper.asciidoc b/metricbeat/docs/modules/zookeeper.asciidoc index 30a7fa3cf48..32eb71cfc56 100644 --- a/metricbeat/docs/modules/zookeeper.asciidoc +++ b/metricbeat/docs/modules/zookeeper.asciidoc @@ -6,7 +6,7 @@ This file is generated! See scripts/docs_collector.py == ZooKeeper module The ZooKeeper module fetches statistics from the ZooKeeper service. The default -metricset is `mntr`. +metricset is `mntr` and `server`. [float] === Compatibility @@ -26,7 +26,7 @@ in <>. Here is an example configuration: metricbeat.modules: - module: zookeeper enabled: true - metricsets: ["mntr"] + metricsets: ["mntr", "server"] period: 10s hosts: ["localhost:2181"] ---- @@ -38,5 +38,9 @@ The following metricsets are available: * <> +* <> + include::zookeeper/mntr.asciidoc[] +include::zookeeper/server.asciidoc[] + diff --git a/metricbeat/docs/modules/zookeeper/server.asciidoc b/metricbeat/docs/modules/zookeeper/server.asciidoc new file mode 100644 index 00000000000..18097850acf --- /dev/null +++ b/metricbeat/docs/modules/zookeeper/server.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-zookeeper-server]] +=== ZooKeeper server metricset + +include::../../../module/zookeeper/server/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/zookeeper/server/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index fd98531677d..c3908400b0e 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -161,7 +161,8 @@ This file is generated! See scripts/docs_collector.py .2+| .2+| |<> beta[] |<> |<> |image:./images/icon-no.png[No prebuilt dashboards] | -.1+| .1+| |<> +.2+| .2+| |<> +|<> |================================ -- diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 20b43a8fb59..be46965fc00 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -179,4 +179,5 @@ import ( _ "github.com/elastic/beats/metricbeat/module/windows/service" _ "github.com/elastic/beats/metricbeat/module/zookeeper" _ "github.com/elastic/beats/metricbeat/module/zookeeper/mntr" + _ "github.com/elastic/beats/metricbeat/module/zookeeper/server" ) diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index b78ba735bee..94314930f7f 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -695,7 +695,7 @@ metricbeat.modules: #------------------------------ ZooKeeper Module ----------------------------- - module: zookeeper enabled: true - metricsets: ["mntr"] + metricsets: ["mntr", "server"] period: 10s hosts: ["localhost:2181"] diff --git a/metricbeat/module/zookeeper/_meta/config.reference.yml b/metricbeat/module/zookeeper/_meta/config.reference.yml index 04742813c55..51b3e0b83bd 100644 --- a/metricbeat/module/zookeeper/_meta/config.reference.yml +++ b/metricbeat/module/zookeeper/_meta/config.reference.yml @@ -1,5 +1,5 @@ - module: zookeeper enabled: true - metricsets: ["mntr"] + metricsets: ["mntr", "server"] period: 10s hosts: ["localhost:2181"] diff --git a/metricbeat/module/zookeeper/_meta/config.yml b/metricbeat/module/zookeeper/_meta/config.yml index 17fb7135973..d6701a8b80f 100644 --- a/metricbeat/module/zookeeper/_meta/config.yml +++ b/metricbeat/module/zookeeper/_meta/config.yml @@ -1,5 +1,6 @@ - module: zookeeper #metricsets: # - mntr + # - server period: 10s hosts: ["localhost:2181"] diff --git a/metricbeat/module/zookeeper/_meta/docs.asciidoc b/metricbeat/module/zookeeper/_meta/docs.asciidoc index 34b3f950010..67f71737cad 100644 --- a/metricbeat/module/zookeeper/_meta/docs.asciidoc +++ b/metricbeat/module/zookeeper/_meta/docs.asciidoc @@ -1,5 +1,5 @@ The ZooKeeper module fetches statistics from the ZooKeeper service. The default -metricset is `mntr`. +metricset is `mntr` and `server`. [float] === Compatibility diff --git a/metricbeat/module/zookeeper/fields.go b/metricbeat/module/zookeeper/fields.go index 8292d50c21d..a122b325be7 100644 --- a/metricbeat/module/zookeeper/fields.go +++ b/metricbeat/module/zookeeper/fields.go @@ -32,5 +32,5 @@ func init() { // AssetZookeeper returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/zookeeper. func AssetZookeeper() string { - return "eJy0lsGyozYQRff+iq7ZP3+AF6nKOpVZZJkN0xYXo7JQE6mxzfv6lMBgcOCVJ4O1ciGr7+mrllofdEZ7oE+RM1Aj7IjUqsOBvv0t8kf37duOKEc0wdZqxR/otx0R0ThPFTRYE8mIczCKnI4taQkqpAkfDqrpT+KtSrD+REaqin0e9zuiWErQzIgv7OlABbuIHVGAA0cc6MQ7osLC5fHQqX6Q5wpz4jS0rdPfgzT1/csCcho/xpU/yIhXtj52sEMWAbWEexJjjuPyKXsaU7YpX+X1sWgJ7wvEDjMFeIHw2eZ+3SzUHXk/+Tj3dxjPuUzzKSVq+jWbHPI6o71KyJ/mvshuXj9D7P2iMtd1kJutWJHlrJxF+7mM4cSffo7h90dsSmFJiglXUltmcqzwpt3z5Vnwf5NcEPiEITIdoVfAE3xEdXToTIpkPVXWORthZFKFczrUJSoEdjEz0njdCPF7Ux0RkkWjAH16ybGCUYhzckWIm+uPkSkmj+7HwDQhwGvn1DJRxbessA7ZoCRhU4f+5Jutmor8g9Q60EMtEnfkORUSOuhHtdVBDOKKl0PBVXzbmHUouJcqa8SwfisM638ewzdVxs5ekBqHh0nBt6+ySWxSmeyUlqzEYaw411JHswwrNfxbq+77erUl7eF4vFhp0mhU9rn1pyzgnwZRt7d2IkKDSG+rB/Lk9hED5qPPGddERVjmrtmcoXEfYGAveG5Gv8788M9DrxLOgyQNkl+DRbxhx9ehktwKEPrdja0322/tPTp10dNOGg6hTTs+P0RjYxsv82XaiHBByKKybvj4+Esc0l0zPxgD0wpJ6w3y7H1trReYdLfxoXct4YkpNduZIWQjOXC+diguCNHK8l3NzvJzDjVr2VtuDfbLq19+1t2XE/ucjo11OUUN/Ynv01pmvrKa8k0XZBcbcXJxRyhJXwdODLtXr8nu4fMmyv5R9Z9n/irfvwEAAP//BzvbMw==" + return "eJy0mM2O4zYMx+95CmIve5o8QA4Fim4PRbG7wLSnvXgYmYmFyKIr0vmYpy9kx47t2JlMPnwa2CP+f6RIkcoLbOiwgHfmDVFBYQagVh0t4Msv5r+rd19mACmJCbZQy34Bv80AANrvkJMGawQMO0dGKYXlATQjWHEZXhypxn9ib5WD9WswnOfoU5nPACTjoIlhv7LrBazQCc0AAjlCoQWscQawsuRSWVSqL+Axpz5xfPRQxH8PXBbHNyPI8XlrV76BYa9ovVSwjReBCg5HJ1of2+Vd9vh02bp8udfTojG8C4gVZjRwBeEwzPW6nqkj8rzzsh/f5hn60vUnY9H4V+9j49eGDjsO6eDbBe/6+dPYno8qY1EE3tsclZIUFROx7+MYjv36cwy/n2xDNAu86nBFtXEmh0reHOa4HQreTLKlgGtqLMOSdEfkgbxQvnRUBUnAesitc1bIcCcL+3RUZJRTQCeJ4dLrgxB/lPmSQgxRKwDvnlOawFixc7yjIA/Xby2DxBgdy8CUIZDXKlLjRDnuk5V1lDRKHB4aoe+4t3mZgz+RWkdwUhPAijyFFYcK+pRtRWBDMhHLJuFy3D+YtUm4qzKrxbD+URjWfx7Dl3mCzm4pNg5PJhp/fJZ1bINyZ6c0QwUMbca5A1Q047BckH9q1v2Yzrao3ZTHlZnGpYqiT61fJ4H+K0n08aHtiEAjUofVE6Ux2ktqME99zrhSlMI4d4FmQyrzQIbslobN6H7mU/w86Y7DppGERvIymNATdnwaKspNAFG9u3Lw5vFbe7QOlfW4kwZDOMQd7xdR29jaw3ycVihsKSSiqA8cPl7ZUTxr+oXRME2QHLyhNHleW6sFOt2tHfR2GXlAiM22FxCwAo4wnSqKLQWxPH5Wo7M49KFAzeqQW0Pz8dVXj3XH5YA+hWVpXQqioa742q1x5h2qyZ50QFa2SToHt5AC13ng2KC79pisBp8nUdZD1dmY/yFfv2Q+d/v4Wi+64cIhYVtdVOpLxte7bhn3tvM/Oi2bRHHprGQn7rPAnE02o6rDwE150bV3fjO46MWZJ8P7AK+mPID+gHuX6nfct4qB0FyOXU/5bCL8pPJgDqyGgQwFlnHC/5il5eD05jbxT10CZybasfO+kv+XFV3nblBV+Ucz2E1KPyfWN/bvmpFej4s7gxt/tCs3zz6vjUa0cMX2H3tOkk5NCyMfeoLfYlc9Ftuv5pei5iSLE0Mp48nxvrfj8bwi8X4Gu7YeHWzRlSPyGtALVuca/PVt4uS8Ly07EtIA1JNFdJoKNtnEbw3DL9fq/hlXXunx/wEAAP//35m7ZA==" } diff --git a/metricbeat/module/zookeeper/server/_meta/data.json b/metricbeat/module/zookeeper/server/_meta/data.json new file mode 100644 index 00000000000..0cc6c7b0b66 --- /dev/null +++ b/metricbeat/module/zookeeper/server/_meta/data.json @@ -0,0 +1,39 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "event": { + "dataset": "zookeeper.server", + "duration": 115000, + "module": "zookeeper" + }, + "metricset": { + "name": "server" + }, + "service": { + "address": "localhost:2181", + "type": "zookeeper", + "version": "3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03" + }, + "zookeeper": { + "server": { + "connections": 1, + "count": 0, + "epoch": 0, + "latency": { + "avg": 0, + "max": 0, + "min": 0 + }, + "mode": "standalone", + "node_count": 4, + "outstanding": 0, + "received": 38, + "sent": 37, + "version_date": "06/29/2018 04:05 GMT", + "zxid": "0x0" + } + } +} \ No newline at end of file diff --git a/metricbeat/module/zookeeper/server/_meta/docs.asciidoc b/metricbeat/module/zookeeper/server/_meta/docs.asciidoc new file mode 100644 index 00000000000..fd30fbeebe2 --- /dev/null +++ b/metricbeat/module/zookeeper/server/_meta/docs.asciidoc @@ -0,0 +1,15 @@ +`server` Metricset fetches the data returned by the `srvr` admin keyword. + +* *connections*: Connections established by the server +* *latency.avg*: Average latency of the server +* *latency.max*: Max latency reached by the server +* *latency.min*: Minimum latency that has been reached by the server +* *mode*: Server mode +* *node_count*: Total number of nodes +* *outstanding*: Outstanding +* *received*: Received requests to the server +* *sent*: Requests sent by the server +* *version_date*: Date of the Zookeeper release in use +* *zxid*: Original value of the Zookeeper transaction ID +* *count*: Total transactions of the leader in epoch +* *epoch*: Epoch value of the Zookeeper transaction ID diff --git a/metricbeat/module/zookeeper/server/_meta/fields.yml b/metricbeat/module/zookeeper/server/_meta/fields.yml new file mode 100644 index 00000000000..2e691a459b7 --- /dev/null +++ b/metricbeat/module/zookeeper/server/_meta/fields.yml @@ -0,0 +1,47 @@ +- name: server + type: group + description: 'server contains the metrics reported by the four-letter `srvr` command.' + release: ga + fields: + - name: connections + type: long + description: Connections established by the server + - name: latency + type: group + fields: + - name: avg + type: long + description: Average latency of the server + - name: max + type: long + description: Max latency reached by the server + - name: min + type: long + description: Minimum latency that has been reached by the server + - name: mode + type: keyword + description: Server mode + - name: node_count + type: long + description: Total number of nodes + - name: outstanding + type: long + description: Outstanding + - name: received + type: long + description: Received requests to the server + - name: sent + type: long + description: Requests sent by the server + - name: version_date + type: date + description: Date of the Zookeeper release in use + - name: zxid + type: keyword + description: Original value of the Zookeeper transaction ID + - name: count + type: long + description: Total transactions of the leader in epoch + - name: epoch + type: long + description: Epoch value of the Zookeeper transaction ID diff --git a/metricbeat/module/zookeeper/server/data.go b/metricbeat/module/zookeeper/server/data.go new file mode 100644 index 00000000000..c768b87f1fb --- /dev/null +++ b/metricbeat/module/zookeeper/server/data.go @@ -0,0 +1,234 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package server + +import ( + "bufio" + "encoding/binary" + "io" + "regexp" + "strconv" + "strings" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" +) + +var latencyCapturer = regexp.MustCompile(`(\d+)/(\d+)/(\d+)`) +var ipCapturer = regexp.MustCompile(`\d+\.\d+\.\d+\.\d+`) +var thatNumberCapturer = regexp.MustCompile(`\[(\d+)\]`) +var portCapturer = regexp.MustCompile(`:(\d+)\[`) +var dataCapturer = regexp.MustCompile(`(\w+)=(\d+)`) +var fieldsCapturer = regexp.MustCompile(`^([a-zA-Z\s]+):\s(\d+)`) +var versionCapturer = regexp.MustCompile(`:\s(.*),`) +var dateCapturer = regexp.MustCompile(`built on (.*)`) + +func parseSrvr(i io.Reader) (common.MapStr, string, error) { + scanner := bufio.NewScanner(i) + + //Get version + ok := scanner.Scan() + + if !ok { + return nil, "", errors.New("no initial successful text scan, aborting") + } + + output := common.MapStr{} + + version := versionCapturer.FindStringSubmatch(scanner.Text())[1] + output.Put("version_date", dateCapturer.FindStringSubmatch(scanner.Text())[1]) + + for scanner.Scan() { + line := scanner.Text() + + if strings.Contains(line, "Zxid") { + xid, err := parseZxid(line) + if err != nil { + err = errors.Wrapf(err, "error parsing 'zxid' line '%s'", line) + logger.Debug(err.Error()) + continue + } + + output.Update(xid) + + continue + } + + if strings.Contains(line, "Latency") { + latency, err := parseLatencyLine(line) + if err != nil { + err = errors.Wrapf(err, "error parsing 'latency values' line '%s'", line) + logger.Debug(err.Error()) + continue + } + + output.Put("latency", latency) + + continue + } + + if strings.Contains(line, "Proposal sizes") { + proposalSizes, err := parseProposalSizes(line) + if err != nil { + err = errors.Wrapf(err, "error parsing 'proposal sizes' line '%s'", line) + logger.Debug(err.Error()) + continue + } + + output.Put("proposal_sizes", proposalSizes) + + continue + } + + if strings.Contains(line, "Mode") { + modeSplit := strings.Split(line, " ") + if len(modeSplit) < 1 { + logger.Debugf("no tokens after splitting line '%s'", line) + continue + } + + output.Put("mode", modeSplit[1]) + continue + } + + // If code reaches here, just easy to parse lines or blank lines like the following are left: + // Received: 46 + // + // Sent: 45 + // Connections: 1 + // Outstanding: 0 + results := fieldsCapturer.FindAllStringSubmatch(line, -1) + if len(results) == 0 { + //probably a blank line + continue + } + + for _, result := range results { + // When submatching, the method returns the original value and the captured values, as you can see in the + // regexp of fieldsCapturer, they are 2 (so no less than 3, counting original value) + if len(result) < 3 { + logger.Debug("less than 3 tokens (%v) when regexp submatching '%s'", result, line) + continue + } + + val, err := strconv.ParseInt(result[2], 10, 64) + if err != nil { + err = errors.Wrapf(err, "error trying to parse value '%s' as int", result[2]) + logger.Debug(err.Error()) + continue + } + + output.Put(strings.ToLower(strings.Replace(result[1], " ", "_", -1)), val) + } + } + + return output, version, nil +} + +func parseZxid(line string) (common.MapStr, error) { + output := common.MapStr{} + + zxidSplit := strings.Split(line, " ") + if len(zxidSplit) < 2 { + return nil, errors.Errorf("less than 2 tokens (%v) after splitting", zxidSplit) + } + + zxidString := zxidSplit[1] + if len(zxidString) < 3 { + return nil, errors.Errorf("less than 3 characters on '%s'", zxidString) + } + zxid, err := strconv.ParseInt(zxidString[2:], 16, 64) + if err != nil { + return nil, errors.Wrapf(err, "error trying to parse value '%s' to int", zxidString[2:]) + } + + bs := make([]byte, 8) + binary.BigEndian.PutUint64(bs, uint64(zxid)) + + epoch := bs[:4] + count := bs[4:] + + output.Put("zxid", zxidString) + output.Put("epoch", binary.BigEndian.Uint32(epoch)) + output.Put("count", binary.BigEndian.Uint32(count)) + + return output, nil +} + +func parseProposalSizes(line string) (common.MapStr, error) { + output := common.MapStr{} + + initialSplit := strings.Split(line, " ") + if len(initialSplit) < 4 { + return nil, errors.Errorf("less than 4 tokens (%v) after splitting", initialSplit) + } + + values := strings.Split(initialSplit[3], "/") + if len(values) < 3 { + return nil, errors.Errorf("less than 3 tokens (%v) after splitting", values) + } + last, err := strconv.ParseInt(values[0], 10, 64) + if err != nil { + return nil, errors.Wrapf(err, "error trying to parse 'last' value as int from '%s'", values[0]) + } + output.Put("last", last) + + min, err := strconv.ParseInt(values[1], 10, 64) + if err != nil { + return nil, errors.Wrapf(err, "error trying to parse 'min' value as int from '%s'", values[1]) + } + output.Put("min", min) + + max, err := strconv.ParseInt(values[2], 10, 64) + if err != nil { + return nil, errors.Wrapf(err, "error trying to parse 'max' value as int from '%s'", values[2]) + } + output.Put("max", max) + + return output, nil +} + +func parseLatencyLine(line string) (common.MapStr, error) { + output := common.MapStr{} + + values := latencyCapturer.FindStringSubmatch(line) + if len(values) < 4 { + return nil, errors.Errorf("less than 4 fields (%v) after splitting", values) + } + + min, err := strconv.ParseInt(values[1], 10, 64) + if err != nil { + return nil, errors.Wrapf(err, "error trying to parse 'min' value '%s' as int", values[1]) + } + output.Put("min", min) + + avg, err := strconv.ParseInt(values[2], 10, 64) + if err != nil { + return nil, errors.Wrapf(err, "error trying to parse 'avg' value '%s' as int", values[2]) + } + output.Put("avg", avg) + + max, err := strconv.ParseInt(values[3], 10, 64) + if err != nil { + return nil, errors.Wrapf(err, "error trying to parse 'max' value '%s' as int", values[3]) + } + output.Put("max", max) + + return output, nil +} diff --git a/metricbeat/module/zookeeper/server/data_integration_test.go b/metricbeat/module/zookeeper/server/data_integration_test.go new file mode 100644 index 00000000000..843db0a7dca --- /dev/null +++ b/metricbeat/module/zookeeper/server/data_integration_test.go @@ -0,0 +1,54 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package server + +import ( + "testing" + + "github.com/elastic/beats/metricbeat/module/zookeeper" + + _ "github.com/denisenkom/go-mssqldb" + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + t.Skip("Skipping `data.json` generation test") + + f := mbtest.NewReportingMetricSetV2(t, getDataConfig()) + events, errs := mbtest.ReportingFetchV2(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + + if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { + t.Fatal("write", err) + } +} + +func getDataConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "zookeeper", + "metricsets": []string{"server"}, + "hosts": []string{zookeeper.GetZookeeperEnvHost() + ":" + zookeeper.GetZookeeperEnvPort()}, + } +} diff --git a/metricbeat/module/zookeeper/server/server.go b/metricbeat/module/zookeeper/server/server.go new file mode 100644 index 00000000000..f73d6904fcd --- /dev/null +++ b/metricbeat/module/zookeeper/server/server.go @@ -0,0 +1,100 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* +Package server fetches metrics from ZooKeeper by using the srvr command + +See the srvr command documentation at +https://zookeeper.apache.org/doc/current/zookeeperAdmin.html + +ZooKeeper srvr Command Output + + $ echo srvr | nc localhost 2181 + Zookeeper version: 3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03, built on 06/29/2018 04:05 GMT +Latency min/avg/max: 1/2/3 +Received: 46 +Sent: 45 +Connections: 1 +Outstanding: 0 +Zxid: 0x700601132 +Mode: standalone +Node count: 4 +Proposal sizes last/min/max: -3/-999/-1 + + +*/ +package server + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + + "github.com/elastic/beats/libbeat/logp" + + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/elastic/beats/metricbeat/module/zookeeper" +) + +var logger = logp.NewLogger("zookeeper.server") + +func init() { + mb.Registry.MustAddMetricSet("zookeeper", "server", New, + mb.WithHostParser(parse.PassThruHostParser), + mb.DefaultMetricSet(), + ) +} + +// MetricSet for fetching ZooKeeper health metrics. +type MetricSet struct { + mb.BaseMetricSet +} + +// New creates new instance of MetricSet. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + return &MetricSet{ + BaseMetricSet: base, + }, nil +} + +// Fetch fetches metrics from ZooKeeper by making a tcp connection to the +// command port and sending the "srvr" command and parsing the output. +func (m *MetricSet) Fetch(reporter mb.ReporterV2) { + outputReader, err := zookeeper.RunCommand("srvr", m.Host(), m.Module().Config().Timeout) + if err != nil { + reporter.Error(errors.Wrap(err, "srvr command failed")) + return + } + + metricsetFields, version, err := parseSrvr(outputReader) + if err != nil { + reporter.Error(err) + return + } + + event := mb.Event{ + MetricSetFields: metricsetFields, + RootFields: common.MapStr{ + "service": common.MapStr{ + "version": version, + }, + }, + } + + reporter.Event(event) +} diff --git a/metricbeat/module/zookeeper/server/server_integration_test.go b/metricbeat/module/zookeeper/server/server_integration_test.go new file mode 100644 index 00000000000..7c5d4abecbc --- /dev/null +++ b/metricbeat/module/zookeeper/server/server_integration_test.go @@ -0,0 +1,67 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package server + +import ( + "testing" + + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/module/zookeeper" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestFetch(t *testing.T) { + logp.TestingSetup() + + compose.EnsureUp(t, "zookeeper") + + f := mbtest.NewReportingMetricSetV2(t, getConfig()) + events, errs := mbtest.ReportingFetchV2(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + + for _, event := range events { + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) + metricsetFields := event.MetricSetFields + + // Check values + assert.Equal(t, "06/29/2018 04:05 GMT", metricsetFields["version_date"]) + + received := metricsetFields["received"].(int64) + assert.True(t, received >= 0) + + nodeCount := metricsetFields["node_count"].(int64) + assert.True(t, nodeCount >= 1) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "zookeeper", + "metricsets": []string{"server"}, + "hosts": []string{zookeeper.GetZookeeperEnvHost() + ":" + zookeeper.GetZookeeperEnvPort()}, + } +} diff --git a/metricbeat/module/zookeeper/server/server_test.go b/metricbeat/module/zookeeper/server/server_test.go new file mode 100644 index 00000000000..eec8e5a2494 --- /dev/null +++ b/metricbeat/module/zookeeper/server/server_test.go @@ -0,0 +1,70 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package server + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/common" +) + +var srvrTestInput = `Zookeeper version: 3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03, built on 06/29/2018 04:05 GMT +Latency min/avg/max: 1/2/3 +Received: 46 +Sent: 45 +Connections: 1 +Outstanding: 0 +Zxid: 0x700601132 +Mode: standalone +Node count: 4 +Proposal sizes last/min/max: -3/-999/-1 +` + +func TestParser(t *testing.T) { + mapStr, versionID, err := parseSrvr(bytes.NewReader([]byte(srvrTestInput))) + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, "06/29/2018 04:05 GMT", mapStr["version_date"]) + assert.Equal(t, "3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03", versionID) + + latency := mapStr["latency"].(common.MapStr) + assert.Equal(t, int64(1), latency["min"]) + assert.Equal(t, int64(2), latency["avg"]) + assert.Equal(t, int64(3), latency["max"]) + + assert.Equal(t, int64(46), mapStr["received"]) + assert.Equal(t, int64(45), mapStr["sent"]) + assert.Equal(t, int64(1), mapStr["connections"]) + assert.Equal(t, int64(0), mapStr["outstanding"]) + assert.Equal(t, "standalone", mapStr["mode"]) + assert.Equal(t, int64(4), mapStr["node_count"]) + + proposalSizes := mapStr["proposal_sizes"].(common.MapStr) + assert.Equal(t, int64(-3), proposalSizes["last"]) + assert.Equal(t, int64(-999), proposalSizes["min"]) + assert.Equal(t, int64(-1), proposalSizes["max"]) + + assert.Equal(t, "0x700601132", mapStr["zxid"]) + assert.Equal(t, uint32(7), mapStr["epoch"]) + assert.Equal(t, uint32(0x601132), mapStr["count"]) +} diff --git a/metricbeat/modules.d/zookeeper.yml.disabled b/metricbeat/modules.d/zookeeper.yml.disabled index 253de00d3e2..7d44efb938e 100644 --- a/metricbeat/modules.d/zookeeper.yml.disabled +++ b/metricbeat/modules.d/zookeeper.yml.disabled @@ -4,5 +4,6 @@ - module: zookeeper #metricsets: # - mntr + # - server period: 10s hosts: ["localhost:2181"] diff --git a/metricbeat/tests/system/test_zookeeper.py b/metricbeat/tests/system/test_zookeeper.py index ee367201239..6e3baadea63 100644 --- a/metricbeat/tests/system/test_zookeeper.py +++ b/metricbeat/tests/system/test_zookeeper.py @@ -50,6 +50,34 @@ def test_output(self): self.assert_fields_are_documented(evt) + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_output(self): + """ + ZooKeeper server module outputs an event. + """ + self.render_config_template(modules=[{ + "name": "zookeeper", + "metricsets": ["server"], + "hosts": self.get_hosts(), + "period": "5s" + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + self.assert_no_logged_warnings() + + output = self.read_output_json() + self.assertEqual(len(output), 1) + evt = output[0] + + self.assertItemsEqual(self.de_dot(ZK_FIELDS), evt.keys()) + zk_srvr = evt["zookeeper"]["server"] + + assert zk_srvr["connections"] >= 0 + + self.assert_fields_are_documented(evt) + def get_hosts(self): return [os.getenv('ZOOKEEPER_HOST', 'localhost') + ':' + os.getenv('ZOOKEEPER_PORT', '2181')] diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 3178b56ac5d..9b9675b01fc 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -713,7 +713,7 @@ metricbeat.modules: #------------------------------ ZooKeeper Module ------------------------------ - module: zookeeper enabled: true - metricsets: ["mntr"] + metricsets: ["mntr", "server"] period: 10s hosts: ["localhost:2181"] diff --git a/x-pack/metricbeat/module/mssql/performance/data_integration_test.go b/x-pack/metricbeat/module/mssql/performance/data_integration_test.go index 9c688593783..116a7269825 100644 --- a/x-pack/metricbeat/module/mssql/performance/data_integration_test.go +++ b/x-pack/metricbeat/module/mssql/performance/data_integration_test.go @@ -18,6 +18,7 @@ import ( func TestData(t *testing.T) { t.Skip("Skipping `data.json` generation test") + _, config, err := getHostURI() if err != nil { t.Fatal("error getting config information", err.Error())