diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 214e3640732..e5e910bd06b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -700,6 +700,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add state_daemonset metricset for Kubernetes Metricbeat module {pull}20649[20649] - Add host inventory metrics to azure compute_vm metricset. {pull}20641[20641] - Add host inventory metrics to googlecloud compute metricset. {pull}20391[20391] +- Add host inventory metrics to system module. {pull}20415[20415] - Add billing data collection from Cost Explorer into aws billing metricset. {pull}20527[20527] {issue}20103[20103] - Migrate `compute_vm` metricset to a light one, map `cloud.instance.id` field. {pull}20889[20889] - Request prometheus endpoints to be gzipped by default {pull}20766[20766] diff --git a/metricbeat/module/system/cpu/_meta/data.json b/metricbeat/module/system/cpu/_meta/data.json index ff6b7a15491..4a5fd7c8ff7 100644 --- a/metricbeat/module/system/cpu/_meta/data.json +++ b/metricbeat/module/system/cpu/_meta/data.json @@ -1,36 +1,38 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "event": { "dataset": "system.cpu", "duration": 115000, "module": "system" }, + "host": { + "cpu": { + "pct": 0.0816 + } + }, "metricset": { - "name": "cpu" + "name": "cpu", + "period": 10000 }, "service": { "type": "system" }, "system": { "cpu": { - "cores": 4, + "cores": 12, "idle": { "norm": { - "pct": 0.7198 + "pct": 0.9184 }, - "pct": 2.8792, - "ticks": 81308898 + "pct": 11.0208, + "ticks": 1964402 }, "iowait": { "norm": { "pct": 0 }, "pct": 0, - "ticks": 499109 + "ticks": 5083 }, "irq": { "norm": { @@ -44,14 +46,14 @@ "pct": 0 }, "pct": 0, - "ticks": 172471 + "ticks": 9752 }, "softirq": { "norm": { - "pct": 0 + "pct": 0.0058 }, - "pct": 0, - "ticks": 578041 + "pct": 0.0699, + "ticks": 10386 }, "steal": { "norm": { @@ -62,23 +64,23 @@ }, "system": { "norm": { - "pct": 0.0591 + "pct": 0.005 }, - "pct": 0.2365, - "ticks": 25140781 + "pct": 0.06, + "ticks": 22274 }, "total": { "norm": { - "pct": 0.2802 + "pct": 0.0816 }, - "pct": 1.1208 + "pct": 0.9792 }, "user": { "norm": { - "pct": 0.2211 + "pct": 0.0708 }, - "pct": 0.8843, - "ticks": 75216920 + "pct": 0.8493, + "ticks": 123767 } } } diff --git a/metricbeat/module/system/cpu/cpu.go b/metricbeat/module/system/cpu/cpu.go index 8d017f0d373..7333df6dec7 100644 --- a/metricbeat/module/system/cpu/cpu.go +++ b/metricbeat/module/system/cpu/cpu.go @@ -70,7 +70,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { } event := common.MapStr{"cores": cpu.NumCores} - + hostFields := common.MapStr{} for _, metric := range m.config.Metrics { switch strings.ToLower(metric) { case percentages: @@ -95,6 +95,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { event.Put("softirq.norm.pct", normalizedPct.SoftIRQ) event.Put("steal.norm.pct", normalizedPct.Steal) event.Put("total.norm.pct", normalizedPct.Total) + hostFields.Put("host.cpu.pct", normalizedPct.Total) case ticks: ticks := sample.Ticks() event.Put("user.ticks", ticks.User) @@ -109,6 +110,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { } r.Event(mb.Event{ + RootFields: hostFields, MetricSetFields: event, }) diff --git a/metricbeat/module/system/diskio/_meta/data.json b/metricbeat/module/system/diskio/_meta/data.json index 291b1d23840..b9c8533b0c8 100644 --- a/metricbeat/module/system/diskio/_meta/data.json +++ b/metricbeat/module/system/diskio/_meta/data.json @@ -1,16 +1,13 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "event": { "dataset": "system.diskio", "duration": 115000, "module": "system" }, "metricset": { - "name": "diskio" + "name": "diskio", + "period": 10000 }, "service": { "type": "system" @@ -18,7 +15,7 @@ "system": { "diskio": { "io": { - "time": 656 + "time": 364 }, "iostat": { "await": 0, @@ -51,16 +48,16 @@ } } }, - "name": "nvme0n1p1", + "name": "loop1", "read": { - "bytes": 8028160, - "count": 3290, - "time": 130016 + "bytes": 5267456, + "count": 4124, + "time": 557 }, "write": { - "bytes": 5120, - "count": 3, - "time": 12 + "bytes": 0, + "count": 0, + "time": 0 } } } diff --git a/metricbeat/module/system/diskio/diskio.go b/metricbeat/module/system/diskio/diskio.go index 80c494409a3..9da3a3c2344 100644 --- a/metricbeat/module/system/diskio/diskio.go +++ b/metricbeat/module/system/diskio/diskio.go @@ -38,6 +38,13 @@ type MetricSet struct { mb.BaseMetricSet statistics *DiskIOStat includeDevices []string + prevCounters diskCounter +} + +// diskCounter stores previous disk counter values for calculating gauges in next collection +type diskCounter struct { + prevDiskReadBytes uint64 + prevDiskWriteBytes uint64 } // New is a mb.MetricSetFactory that returns a new MetricSet. @@ -54,6 +61,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { BaseMetricSet: base, statistics: NewDiskIOStat(), includeDevices: config.IncludeDevices, + prevCounters: diskCounter{}, }, nil } @@ -70,6 +78,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { // Store the last cpu counter when finished defer m.statistics.CloseSampling() + var diskReadBytes, diskWriteBytes uint64 for _, counters := range stats { event := common.MapStr{ "name": counters.Name, @@ -87,6 +96,11 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { "time": counters.IoTime, }, } + + // accumulate values from all interfaces + diskReadBytes += counters.ReadBytes + diskWriteBytes += counters.WriteBytes + var extraMetrics DiskIOMetric err := m.statistics.CalIOStatistics(&extraMetrics, counters) if err == nil { @@ -135,5 +149,23 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { } } + if m.prevCounters != (diskCounter{}) { + // convert network metrics from counters to gauges + r.Event(mb.Event{ + RootFields: common.MapStr{ + "host": common.MapStr{ + "disk": common.MapStr{ + "read.bytes": diskReadBytes - m.prevCounters.prevDiskReadBytes, + "write.bytes": diskWriteBytes - m.prevCounters.prevDiskWriteBytes, + }, + }, + }, + }) + } + + // update prevCounters + m.prevCounters.prevDiskReadBytes = diskReadBytes + m.prevCounters.prevDiskWriteBytes = diskWriteBytes + return nil } diff --git a/metricbeat/module/system/network/_meta/data.json b/metricbeat/module/system/network/_meta/data.json index 8cf8a5f69a6..81c174c93e8 100644 --- a/metricbeat/module/system/network/_meta/data.json +++ b/metricbeat/module/system/network/_meta/data.json @@ -1,16 +1,13 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "event": { "dataset": "system.network", "duration": 115000, "module": "system" }, "metricset": { - "name": "network" + "name": "network", + "period": 10000 }, "service": { "type": "system" @@ -18,17 +15,17 @@ "system": { "network": { "in": { - "bytes": 37904869172, - "dropped": 32, + "bytes": 0, + "dropped": 0, "errors": 0, - "packets": 32143403 + "packets": 0 }, - "name": "wlp4s0", + "name": "br-18285ad7f418", "out": { - "bytes": 6299331926, + "bytes": 0, "dropped": 0, "errors": 0, - "packets": 13362703 + "packets": 0 } } } diff --git a/metricbeat/module/system/network/network.go b/metricbeat/module/system/network/network.go index d9c04d83420..4796b5af841 100644 --- a/metricbeat/module/system/network/network.go +++ b/metricbeat/module/system/network/network.go @@ -43,7 +43,16 @@ func init() { // MetricSet for fetching system network IO metrics. type MetricSet struct { mb.BaseMetricSet - interfaces map[string]struct{} + interfaces map[string]struct{} + prevCounters networkCounter +} + +// networkCounter stores previous network counter values for calculating gauges in next collection +type networkCounter struct { + prevNetworkInBytes uint64 + prevNetworkInPackets uint64 + prevNetworkOutBytes uint64 + prevNetworkOutPackets uint64 } // New is a mb.MetricSetFactory that returns a new MetricSet. @@ -69,6 +78,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ BaseMetricSet: base, interfaces: interfaceSet, + prevCounters: networkCounter{}, }, nil } @@ -79,6 +89,8 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return errors.Wrap(err, "network io counters") } + var networkInBytes, networkOutBytes, networkInPackets, networkOutPackets uint64 + for _, counters := range stats { if m.interfaces != nil { // Select stats by interface name. @@ -91,11 +103,44 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { isOpen := r.Event(mb.Event{ MetricSetFields: ioCountersToMapStr(counters), }) + + // accumulate values from all interfaces + networkInBytes += counters.BytesRecv + networkOutBytes += counters.BytesSent + networkInPackets += counters.PacketsRecv + networkOutPackets += counters.PacketsSent + if !isOpen { return nil } } + if m.prevCounters != (networkCounter{}) { + // convert network metrics from counters to gauges + r.Event(mb.Event{ + RootFields: common.MapStr{ + "host": common.MapStr{ + "network": common.MapStr{ + "in": common.MapStr{ + "bytes": networkInBytes - m.prevCounters.prevNetworkInBytes, + "packets": networkInPackets - m.prevCounters.prevNetworkInPackets, + }, + "out": common.MapStr{ + "bytes": networkOutBytes - m.prevCounters.prevNetworkOutBytes, + "packets": networkOutPackets - m.prevCounters.prevNetworkOutPackets, + }, + }, + }, + }, + }) + } + + // update prevCounters + m.prevCounters.prevNetworkInBytes = networkInBytes + m.prevCounters.prevNetworkInPackets = networkInPackets + m.prevCounters.prevNetworkOutBytes = networkOutBytes + m.prevCounters.prevNetworkOutPackets = networkOutPackets + return nil } diff --git a/metricbeat/module/system/test_system.py b/metricbeat/module/system/test_system.py index ce8670781db..f689b99fb4c 100644 --- a/metricbeat/module/system/test_system.py +++ b/metricbeat/module/system/test_system.py @@ -45,6 +45,12 @@ SYSTEM_NETWORK_FIELDS = ["name", "out.bytes", "in.bytes", "out.packets", "in.packets", "in.error", "out.error", "in.dropped", "out.dropped"] +SYSTEM_CPU_HOST_FIELDS = ["pct"] + +SYSTEM_NETWORK_HOST_FIELDS = ["in.bytes", "out.bytes", "in.packets", "out.packets"] + +SYSTEM_DISK_HOST_FIELDS = ["read.bytes", "write.bytes"] + # cmdline is also part of the system process fields, but it may not be present # for some kernel level processes. fd is also part of the system process, but # is not available on all OSes and requires root to read for all processes. @@ -74,8 +80,12 @@ def test_cpu(self): evt = output[0] self.assert_fields_are_documented(evt) - cpu = evt["system"]["cpu"] - self.assertCountEqual(self.de_dot(SYSTEM_CPU_FIELDS), cpu.keys()) + if "system" in evt: + cpu = evt["system"]["cpu"] + self.assertCountEqual(self.de_dot(SYSTEM_CPU_FIELDS), cpu.keys()) + else: + host_cpu = evt["host"]["cpu"] + self.assertCountEqual(self.de_dot(SYSTEM_CPU_HOST_FIELDS), host_cpu.keys()) @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") def test_cpu_ticks_option(self): @@ -196,8 +206,12 @@ def test_diskio(self): for evt in output: self.assert_fields_are_documented(evt) if 'error' not in evt: - diskio = evt["system"]["diskio"] - self.assertCountEqual(self.de_dot(SYSTEM_DISKIO_FIELDS), diskio.keys()) + if "system" in evt: + diskio = evt["system"]["diskio"] + self.assertCountEqual(self.de_dot(SYSTEM_DISKIO_FIELDS), diskio.keys()) + elif "host" in evt: + host_disk = evt["host"]["disk"] + self.assertCountEqual(SYSTEM_DISK_HOST_FIELDS, host_disk.keys()) @unittest.skipUnless(re.match("(?i)linux", sys.platform), "os") def test_diskio_linux(self): @@ -219,8 +233,12 @@ def test_diskio_linux(self): for evt in output: self.assert_fields_are_documented(evt) - diskio = evt["system"]["diskio"] - self.assertCountEqual(self.de_dot(SYSTEM_DISKIO_FIELDS_LINUX), diskio.keys()) + if "system" in evt: + diskio = evt["system"]["diskio"] + self.assertCountEqual(self.de_dot(SYSTEM_DISKIO_FIELDS_LINUX), diskio.keys()) + else: + host_disk = evt["host"]["disk"] + self.assertCountEqual(SYSTEM_DISK_HOST_FIELDS, host_disk.keys()) @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") def test_filesystem(self): @@ -328,8 +346,12 @@ def test_network(self): for evt in output: self.assert_fields_are_documented(evt) - network = evt["system"]["network"] - self.assertCountEqual(self.de_dot(SYSTEM_NETWORK_FIELDS), network.keys()) + if "system" in evt: + network = evt["system"]["network"] + self.assertCountEqual(self.de_dot(SYSTEM_NETWORK_FIELDS), network.keys()) + else: + host_network = evt["host"]["network"] + self.assertCountEqual(self.de_dot(SYSTEM_NETWORK_HOST_FIELDS), host_network.keys()) @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd", sys.platform), "os") def test_process_summary(self):