Enrich k8s state service events & add selector meta (elastic#23730)

ChrsMark · Feb 2, 2021 · 1f38fff · 1f38fff
1 parent e6bb5c9
commit 1f38fff
Show file tree

Hide file tree

Showing 23 changed files with 145 additions and 10 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -602,6 +602,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add kubernetes.volume.fs.used.pct field. {pull}23564[23564]
 - Add the `enable_krb5_fast` flag to the Kafka output to explicitly opt-in to FAST authentication. {pull}23629[23629]
 - Add deployment name in pod's meta. {pull}23610[23610]
+- Add `selector` information in kubernetes services' metadata. {pull}23730[23730]
 
 *Auditbeat*
 
@@ -963,6 +964,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Move IIS module to GA and map fields. {issue}22609[22609] {pull}23024[23024]
 - Apache: convert status.total_kbytes to status.total_bytes in fleet mode. {pull}23022[23022]
 - Release MSSQL as GA {pull}23146[23146]
+- Enrich events of `state_service` metricset with kubernetes services' metadata. {pull}23730[23730]
 
 *Packetbeat*
 

diff --git a/auditbeat/docs/fields.asciidoc b/auditbeat/docs/fields.asciidoc
@@ -11615,6 +11615,16 @@ type: object
 Kubernetes annotations map
 
 
+type: object
+
+--
+
+*`kubernetes.service.selectors.*`*::
++
+--
+Kubernetes Service selectors map
+
+
 type: object
 
 --

diff --git a/auditbeat/include/fields.go b/auditbeat/include/fields.go
diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -85616,6 +85616,16 @@ type: object
 Kubernetes annotations map
 
 
+type: object
+
+--
+
+*`kubernetes.service.selectors.*`*::
++
+--
+Kubernetes Service selectors map
+
+
 type: object
 
 --

diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go
diff --git a/heartbeat/docs/fields.asciidoc b/heartbeat/docs/fields.asciidoc
@@ -9113,6 +9113,16 @@ type: object
 Kubernetes annotations map
 
 
+type: object
+
+--
+
+*`kubernetes.service.selectors.*`*::
++
+--
+Kubernetes Service selectors map
+
+
 type: object
 
 --

diff --git a/heartbeat/include/fields.go b/heartbeat/include/fields.go
diff --git a/journalbeat/docs/fields.asciidoc b/journalbeat/docs/fields.asciidoc
@@ -9458,6 +9458,16 @@ type: object
 Kubernetes annotations map
 
 
+type: object
+
+--
+
+*`kubernetes.service.selectors.*`*::
++
+--
+Kubernetes Service selectors map
+
+
 type: object
 
 --

diff --git a/journalbeat/include/fields.go b/journalbeat/include/fields.go
diff --git a/libbeat/common/kubernetes/metadata/resource.go b/libbeat/common/kubernetes/metadata/resource.go
@@ -51,7 +51,7 @@ func (r *Resource) Generate(kind string, obj kubernetes.Resource, options ...Fie
 
 	labelMap := common.MapStr{}
 	if len(r.config.IncludeLabels) == 0 {
-		labelMap = generateMap(accessor.GetLabels(), r.config.LabelsDedot)
+		labelMap = GenerateMap(accessor.GetLabels(), r.config.LabelsDedot)
 	} else {
 		labelMap = generateMapSubset(accessor.GetLabels(), r.config.IncludeLabels, r.config.LabelsDedot)
 	}
@@ -126,7 +126,7 @@ func generateMapSubset(input map[string]string, keys []string, dedot bool) commo
 	return output
 }
 
-func generateMap(input map[string]string, dedot bool) common.MapStr {
+func GenerateMap(input map[string]string, dedot bool) common.MapStr {
 	output := common.MapStr{}
 	if input == nil {
 		return output

diff --git a/libbeat/common/kubernetes/metadata/service.go b/libbeat/common/kubernetes/metadata/service.go
@@ -22,6 +22,7 @@ import (
 
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/kubernetes"
+	"github.com/elastic/beats/v7/libbeat/common/safemapstr"
 )
 
 type service struct {
@@ -57,6 +58,15 @@ func (s *service) Generate(obj kubernetes.Resource, opts ...FieldOptions) common
 		}
 	}
 
+	selectors := svc.Spec.Selector
+	if len(selectors) == 0 {
+		return out
+	}
+	svcMap := GenerateMap(selectors, s.resource.config.LabelsDedot)
+	if len(svcMap) != 0 {
+		safemapstr.Put(out, "selectors", svcMap)
+	}
+
 	return out
 }
 

diff --git a/libbeat/common/kubernetes/metadata/service_test.go b/libbeat/common/kubernetes/metadata/service_test.go
@@ -59,6 +59,12 @@ func TestService_Generate(t *testing.T) {
 					Kind:       "Service",
 					APIVersion: "v1",
 				},
+				Spec: v1.ServiceSpec{
+					Selector: map[string]string{
+						"app":   "istiod",
+						"istio": "pilot",
+					},
+				},
 			},
 			output: common.MapStr{
 				"service": common.MapStr{
@@ -68,6 +74,10 @@ func TestService_Generate(t *testing.T) {
 				"labels": common.MapStr{
 					"foo": "bar",
 				},
+				"selectors": common.MapStr{
+					"app":   "istiod",
+					"istio": "pilot",
+				},
 				"namespace": "default",
 			},
 		},
@@ -96,6 +106,12 @@ func TestService_Generate(t *testing.T) {
 					Kind:       "Service",
 					APIVersion: "v1",
 				},
+				Spec: v1.ServiceSpec{
+					Selector: map[string]string{
+						"app":   "istiod",
+						"istio": "pilot",
+					},
+				},
 			},
 			output: common.MapStr{
 				"service": common.MapStr{
@@ -105,6 +121,10 @@ func TestService_Generate(t *testing.T) {
 				"labels": common.MapStr{
 					"foo": "bar",
 				},
+				"selectors": common.MapStr{
+					"app":   "istiod",
+					"istio": "pilot",
+				},
 				"namespace": "default",
 				"deployment": common.MapStr{
 					"name": "owner",

diff --git a/libbeat/processors/add_kubernetes_metadata/_meta/fields.yml b/libbeat/processors/add_kubernetes_metadata/_meta/fields.yml
@@ -47,6 +47,13 @@
           description: >
             Kubernetes annotations map
 
+        - name: service.selectors.*
+          type: object
+          object_type: keyword
+          object_type_mapping_type: "*"
+          description: >
+            Kubernetes Service selectors map
+
         - name: replicaset.name
           type: keyword
           description: >

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -27026,6 +27026,16 @@ type: object
 Kubernetes annotations map
 
 
+type: object
+
+--
+
+*`kubernetes.service.selectors.*`*::
++
+--
+Kubernetes Service selectors map
+
+
 type: object
 
 --

diff --git a/metricbeat/module/kubernetes/state_service/state_service.go b/metricbeat/module/kubernetes/state_service/state_service.go
@@ -18,8 +18,10 @@
 package state_service
 
 import (
+	"github.com/elastic/beats/v7/libbeat/common/kubernetes"
 	p "github.com/elastic/beats/v7/metricbeat/helper/prometheus"
 	"github.com/elastic/beats/v7/metricbeat/mb"
+	"github.com/elastic/beats/v7/metricbeat/module/kubernetes/util"
 )
 
 func init() {
@@ -37,6 +39,7 @@ type ServiceMetricSet struct {
 	mb.BaseMetricSet
 	prometheus p.Prometheus
 	mapping    *p.MetricsMapping
+	enricher   util.Enricher
 }
 
 // NewServiceMetricSet returns a prometheus based metricset for Services
@@ -75,19 +78,23 @@ func NewServiceMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) {
 				"hostname":         p.Label("ingress_hostname"),
 			},
 		},
+		enricher: util.NewResourceMetadataEnricher(base, &kubernetes.Service{}, false),
 	}, nil
 }
 
 // Fetch prometheus metrics and treats those prefixed by mb.ModuleDataKey as
 // module rooted fields at the event that gets reported
 func (m *ServiceMetricSet) Fetch(reporter mb.ReporterV2) {
+	m.enricher.Start()
 	events, err := m.prometheus.GetProcessedMetrics(m.mapping)
 	if err != nil {
 		m.Logger().Error(err)
 		reporter.Error(err)
 		return
 	}
 
+	m.enricher.Enrich(events)
+
 	for _, event := range events {
 		event[mb.NamespaceKey] = "service"
 		reported := reporter.Event(mb.TransformMapStrToEvent("kubernetes", event, nil))
@@ -98,3 +105,9 @@ func (m *ServiceMetricSet) Fetch(reporter mb.ReporterV2) {
 	}
 	return
 }
+
+// Close stops this metricset
+func (m *ServiceMetricSet) Close() error {
+	m.enricher.Stop()
+	return nil
+}
diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go
@@ -130,6 +130,7 @@ func NewResourceMetadataEnricher(
 
 	metaGen := metadata.NewResourceMetadataGenerator(cfg)
 	podMetaGen := metadata.NewPodMetadataGenerator(cfg, nil, watcher.Client(), nil, nil)
+	serviceMetaGen := metadata.NewServiceMetadataGenerator(cfg, nil, nil)
 	enricher := buildMetadataEnricher(watcher,
 		// update
 		func(m map[string]common.MapStr, r kubernetes.Resource) {
@@ -158,6 +159,8 @@ func NewResourceMetadataEnricher(
 
 			case *kubernetes.Deployment:
 				m[id] = metaGen.Generate("deployment", r)
+			case *kubernetes.Service:
+				m[id] = serviceMetaGen.Generate(r)
 			case *kubernetes.StatefulSet:
 				m[id] = metaGen.Generate("statefulset", r)
 			case *kubernetes.Namespace:

diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc
@@ -10919,6 +10919,16 @@ type: object
 Kubernetes annotations map
 
 
+type: object
+
+--
+
+*`kubernetes.service.selectors.*`*::
++
+--
+Kubernetes Service selectors map
+
+
 type: object
 
 --

diff --git a/packetbeat/include/fields.go b/packetbeat/include/fields.go
diff --git a/winlogbeat/docs/fields.asciidoc b/winlogbeat/docs/fields.asciidoc
@@ -8942,6 +8942,16 @@ type: object
 Kubernetes annotations map
 
 
+type: object
+
+--
+
+*`kubernetes.service.selectors.*`*::
++
+--
+Kubernetes Service selectors map
+
+
 type: object
 
 --

diff --git a/winlogbeat/include/fields.go b/winlogbeat/include/fields.go
diff --git a/x-pack/functionbeat/docs/fields.asciidoc b/x-pack/functionbeat/docs/fields.asciidoc
@@ -8765,6 +8765,16 @@ type: object
 Kubernetes annotations map
 
 
+type: object
+
+--
+
+*`kubernetes.service.selectors.*`*::
++
+--
+Kubernetes Service selectors map
+
+
 type: object
 
 --

diff --git a/x-pack/functionbeat/include/fields.go b/x-pack/functionbeat/include/fields.go
diff --git a/x-pack/heartbeat/include/fields.go b/x-pack/heartbeat/include/fields.go