Add host inventory metrics to system module (#20415)

* Add host inventory metrics to system module
* Update data.json files

CHANGELOG.next.asciidoc

@@ -700,6 +700,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add state_daemonset metricset for Kubernetes Metricbeat module {pull}20649[20649]
 - Add host inventory metrics to azure compute_vm metricset. {pull}20641[20641]
 - Add host inventory metrics to googlecloud compute metricset. {pull}20391[20391]
+- Add host inventory metrics to system module. {pull}20415[20415]
 - Add billing data collection from Cost Explorer into aws billing metricset. {pull}20527[20527] {issue}20103[20103]
 - Migrate `compute_vm` metricset to a light one, map `cloud.instance.id` field. {pull}20889[20889]
 - Request prometheus endpoints to be gzipped by default {pull}20766[20766]

metricbeat/module/system/cpu/_meta/data.json

@@ -1,36 +1,38 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
-    "agent": {
-        "hostname": "host.example.com",
-        "name": "host.example.com"
-    },
     "event": {
         "dataset": "system.cpu",
         "duration": 115000,
         "module": "system"
     },
+    "host": {
+        "cpu": {
+            "pct": 0.0816
+        }
+    },
     "metricset": {
-        "name": "cpu"
+        "name": "cpu",
+        "period": 10000
     },
     "service": {
         "type": "system"
     },
     "system": {
         "cpu": {
-            "cores": 4,
+            "cores": 12,
             "idle": {
                 "norm": {
-                    "pct": 0.7198
+                    "pct": 0.9184
                 },
-                "pct": 2.8792,
-                "ticks": 81308898
+                "pct": 11.0208,
+                "ticks": 1964402
             },
             "iowait": {
                 "norm": {
                     "pct": 0
                 },
                 "pct": 0,
-                "ticks": 499109
+                "ticks": 5083
             },
             "irq": {
                 "norm": {
@@ -44,14 +46,14 @@
                     "pct": 0
                 },
                 "pct": 0,
-                "ticks": 172471
+                "ticks": 9752
             },
             "softirq": {
                 "norm": {
-                    "pct": 0
+                    "pct": 0.0058
                 },
-                "pct": 0,
-                "ticks": 578041
+                "pct": 0.0699,
+                "ticks": 10386
             },
             "steal": {
                 "norm": {
@@ -62,23 +64,23 @@
             },
             "system": {
                 "norm": {
-                    "pct": 0.0591
+                    "pct": 0.005
                 },
-                "pct": 0.2365,
-                "ticks": 25140781
+                "pct": 0.06,
+                "ticks": 22274
             },
             "total": {
                 "norm": {
-                    "pct": 0.2802
+                    "pct": 0.0816
                 },
-                "pct": 1.1208
+                "pct": 0.9792
             },
             "user": {
                 "norm": {
-                    "pct": 0.2211
+                    "pct": 0.0708
                 },
-                "pct": 0.8843,
-                "ticks": 75216920
+                "pct": 0.8493,
+                "ticks": 123767
             }
         }
     }

metricbeat/module/system/cpu/cpu.go

@@ -70,7 +70,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error {
 	}
 
 	event := common.MapStr{"cores": cpu.NumCores}
-
+	hostFields := common.MapStr{}
 	for _, metric := range m.config.Metrics {
 		switch strings.ToLower(metric) {
 		case percentages:
@@ -95,6 +95,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error {
 			event.Put("softirq.norm.pct", normalizedPct.SoftIRQ)
 			event.Put("steal.norm.pct", normalizedPct.Steal)
 			event.Put("total.norm.pct", normalizedPct.Total)
+			hostFields.Put("host.cpu.pct", normalizedPct.Total)
 		case ticks:
 			ticks := sample.Ticks()
 			event.Put("user.ticks", ticks.User)
@@ -109,6 +110,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error {
 	}
 
 	r.Event(mb.Event{
+		RootFields:      hostFields,
 		MetricSetFields: event,
 	})
 

metricbeat/module/system/diskio/_meta/data.json

@@ -1,24 +1,21 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
-    "agent": {
-        "hostname": "host.example.com",
-        "name": "host.example.com"
-    },
     "event": {
         "dataset": "system.diskio",
         "duration": 115000,
         "module": "system"
     },
     "metricset": {
-        "name": "diskio"
+        "name": "diskio",
+        "period": 10000
     },
     "service": {
         "type": "system"
     },
     "system": {
         "diskio": {
             "io": {
-                "time": 656
+                "time": 364
             },
             "iostat": {
                 "await": 0,
@@ -51,16 +48,16 @@
                     }
                 }
             },
-            "name": "nvme0n1p1",
+            "name": "loop1",
             "read": {
-                "bytes": 8028160,
-                "count": 3290,
-                "time": 130016
+                "bytes": 5267456,
+                "count": 4124,
+                "time": 557
             },
             "write": {
-                "bytes": 5120,
-                "count": 3,
-                "time": 12
+                "bytes": 0,
+                "count": 0,
+                "time": 0
             }
         }
     }

metricbeat/module/system/diskio/diskio.go

@@ -38,6 +38,13 @@ type MetricSet struct {
 	mb.BaseMetricSet
 	statistics     *DiskIOStat
 	includeDevices []string
+	prevCounters   diskCounter
+}
+
+// diskCounter stores previous disk counter values for calculating gauges in next collection
+type diskCounter struct {
+	prevDiskReadBytes  uint64
+	prevDiskWriteBytes uint64
 }
 
 // New is a mb.MetricSetFactory that returns a new MetricSet.
@@ -54,6 +61,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 		BaseMetricSet:  base,
 		statistics:     NewDiskIOStat(),
 		includeDevices: config.IncludeDevices,
+		prevCounters:   diskCounter{},
 	}, nil
 }
 
@@ -70,6 +78,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error {
 	// Store the last cpu counter when finished
 	defer m.statistics.CloseSampling()
 
+	var diskReadBytes, diskWriteBytes uint64
 	for _, counters := range stats {
 		event := common.MapStr{
 			"name": counters.Name,
@@ -87,6 +96,11 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error {
 				"time": counters.IoTime,
 			},
 		}
+
+		// accumulate values from all interfaces
+		diskReadBytes += counters.ReadBytes
+		diskWriteBytes += counters.WriteBytes
+
 		var extraMetrics DiskIOMetric
 		err := m.statistics.CalIOStatistics(&extraMetrics, counters)
 		if err == nil {
@@ -135,5 +149,23 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error {
 		}
 	}
 
+	if m.prevCounters != (diskCounter{}) {
+		// convert network metrics from counters to gauges
+		r.Event(mb.Event{
+			RootFields: common.MapStr{
+				"host": common.MapStr{
+					"disk": common.MapStr{
+						"read.bytes":  diskReadBytes - m.prevCounters.prevDiskReadBytes,
+						"write.bytes": diskWriteBytes - m.prevCounters.prevDiskWriteBytes,
+					},
+				},
+			},
+		})
+	}
+
+	// update prevCounters
+	m.prevCounters.prevDiskReadBytes = diskReadBytes
+	m.prevCounters.prevDiskWriteBytes = diskWriteBytes
+
 	return nil
 }

metricbeat/module/system/network/_meta/data.json

@@ -1,34 +1,31 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
-    "agent": {
-        "hostname": "host.example.com",
-        "name": "host.example.com"
-    },
     "event": {
         "dataset": "system.network",
         "duration": 115000,
         "module": "system"
     },
     "metricset": {
-        "name": "network"
+        "name": "network",
+        "period": 10000
     },
     "service": {
         "type": "system"
     },
     "system": {
         "network": {
             "in": {
-                "bytes": 37904869172,
-                "dropped": 32,
+                "bytes": 0,
+                "dropped": 0,
                 "errors": 0,
-                "packets": 32143403
+                "packets": 0
             },
-            "name": "wlp4s0",
+            "name": "br-18285ad7f418",
             "out": {
-                "bytes": 6299331926,
+                "bytes": 0,
                 "dropped": 0,
                 "errors": 0,
-                "packets": 13362703
+                "packets": 0
             }
         }
     }

metricbeat/module/system/network/network.go

@@ -43,7 +43,16 @@ func init() {
 // MetricSet for fetching system network IO metrics.
 type MetricSet struct {
 	mb.BaseMetricSet
-	interfaces map[string]struct{}
+	interfaces   map[string]struct{}
+	prevCounters networkCounter
+}
+
+// networkCounter stores previous network counter values for calculating gauges in next collection
+type networkCounter struct {
+	prevNetworkInBytes    uint64
+	prevNetworkInPackets  uint64
+	prevNetworkOutBytes   uint64
+	prevNetworkOutPackets uint64
 }
 
 // New is a mb.MetricSetFactory that returns a new MetricSet.
@@ -69,6 +78,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	return &MetricSet{
 		BaseMetricSet: base,
 		interfaces:    interfaceSet,
+		prevCounters:  networkCounter{},
 	}, nil
 }
 
@@ -79,6 +89,8 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error {
 		return errors.Wrap(err, "network io counters")
 	}
 
+	var networkInBytes, networkOutBytes, networkInPackets, networkOutPackets uint64
+
 	for _, counters := range stats {
 		if m.interfaces != nil {
 			// Select stats by interface name.
@@ -91,11 +103,44 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error {
 		isOpen := r.Event(mb.Event{
 			MetricSetFields: ioCountersToMapStr(counters),
 		})
+
+		// accumulate values from all interfaces
+		networkInBytes += counters.BytesRecv
+		networkOutBytes += counters.BytesSent
+		networkInPackets += counters.PacketsRecv
+		networkOutPackets += counters.PacketsSent
+
 		if !isOpen {
 			return nil
 		}
 	}
 
+	if m.prevCounters != (networkCounter{}) {
+		// convert network metrics from counters to gauges
+		r.Event(mb.Event{
+			RootFields: common.MapStr{
+				"host": common.MapStr{
+					"network": common.MapStr{
+						"in": common.MapStr{
+							"bytes":   networkInBytes - m.prevCounters.prevNetworkInBytes,
+							"packets": networkInPackets - m.prevCounters.prevNetworkInPackets,
+						},
+						"out": common.MapStr{
+							"bytes":   networkOutBytes - m.prevCounters.prevNetworkOutBytes,
+							"packets": networkOutPackets - m.prevCounters.prevNetworkOutPackets,
+						},
+					},
+				},
+			},
+		})
+	}
+
+	// update prevCounters
+	m.prevCounters.prevNetworkInBytes = networkInBytes
+	m.prevCounters.prevNetworkInPackets = networkInPackets
+	m.prevCounters.prevNetworkOutBytes = networkOutBytes
+	m.prevCounters.prevNetworkOutPackets = networkOutPackets
+
 	return nil
 }