Add pod-uid support for add_kubernetes_metadata (elastic#7072)

Added PodUidIndexer to add_kubernetes_metadata processor, containing the Pod.UID as the key and inside the metadata.
Extended Filebeat's LogPathMatcher with an additional configuration parameter (resource_type) which, when set to 'pod', uses a different logic to extract the MetadataIndex from the source field.

CHANGELOG.asciidoc

@@ -131,6 +131,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
   execve, execveat, fork, and vfork syscalls. A custom policy can be configured. {issue}5213[5213]
 - Update Sarama to v1.16.0, adding better support for kafka 0.11, 1.0, and 1.1 {pull}7025[7025]
 - Ship fields.yml as part of the binary {pull}4834[4834]
+- Add Indexer indexing by pod uid. Enable pod uid metadata gathering in add_kubernetes_metadata. Extended Matcher log_path matching to support volume mounts {pull}7072[7072]
 
 *Auditbeat*
 

auditbeat/docs/fields.asciidoc

@@ -3296,6 +3296,16 @@ type: keyword
 Kubernetes pod name
 
 
+--
+
+*`kubernetes.pod.uid`*::
++
+--
+type: keyword
+
+Kubernetes pod uid
+
+
 --
 
 *`kubernetes.namespace`*::

filebeat/docs/fields.asciidoc

@@ -1667,6 +1667,16 @@ type: keyword
 Kubernetes pod name
 
 
+--
+
+*`kubernetes.pod.uid`*::
++
+--
+type: keyword
+
+Kubernetes pod uid
+
+
 --
 
 *`kubernetes.namespace`*::

filebeat/processor/add_kubernetes_metadata/matchers.go

@@ -23,14 +23,17 @@ func init() {
 const LogPathMatcherName = "logs_path"
 
 type LogPathMatcher struct {
-	LogsPath string
+	LogsPath     string
+	ResourceType string
 }
 
 func newLogsPathMatcher(cfg common.Config) (add_kubernetes_metadata.Matcher, error) {
 	config := struct {
-		LogsPath string `config:"logs_path"`
+		LogsPath     string `config:"logs_path"`
+		ResourceType string `config:"resource_type"`
 	}{
-		LogsPath: "/var/lib/docker/containers/",
+		LogsPath:     "/var/lib/docker/containers/",
+		ResourceType: "container",
 	}
 
 	err := cfg.Unpack(&config)
@@ -42,15 +45,20 @@ func newLogsPathMatcher(cfg common.Config) (add_kubernetes_metadata.Matcher, err
 	if logPath[len(logPath)-1:] != "/" {
 		logPath = logPath + "/"
 	}
+	resourceType := config.ResourceType
 
 	logp.Debug("kubernetes", "logs_path matcher log path: %s", logPath)
+	logp.Debug("kubernetes", "logs_path matcher resource type: %s", resourceType)
 
-	return &LogPathMatcher{LogsPath: logPath}, nil
+	return &LogPathMatcher{LogsPath: logPath, ResourceType: resourceType}, nil
 }
 
 // Docker container ID is a 64-character-long hexadecimal string
 const containerIdLen = 64
 
+// Pod UID is on the 5th index of the path directories
+const podUIDPos = 5
+
 func (f *LogPathMatcher) MetadataIndex(event common.MapStr) string {
 	if value, ok := event["source"]; ok {
 		source := value.(string)
@@ -64,24 +72,40 @@ func (f *LogPathMatcher) MetadataIndex(event common.MapStr) string {
 		sourceLen := len(source)
 		logsPathLen := len(f.LogsPath)
 
-		// In case of the Kubernetes log path "/var/log/containers/",
-		// the container ID will be located right before the ".log" extension.
-		if strings.HasPrefix(f.LogsPath, "/var/log/containers/") && strings.HasSuffix(source, ".log") && sourceLen >= containerIdLen+4 {
-			containerIdEnd := sourceLen - 4
-			cid := source[containerIdEnd-containerIdLen : containerIdEnd]
-			logp.Debug("kubernetes", "Using container id: %s", cid)
-			return cid
-		}
-
-		// In any other case, we assume the container ID will follow right after the log path.
-		// However we need to check the length to prevent "slice bound out of range" runtime errors.
-		if sourceLen >= logsPathLen+containerIdLen {
-			cid := source[logsPathLen : logsPathLen+containerIdLen]
-			logp.Debug("kubernetes", "Using container id: %s", cid)
-			return cid
+		if f.ResourceType == "pod" {
+			// Specify a pod resource type when manually mounting log volumes and they end up under "/var/lib/kubelet/pods/"
+			// This will extract only the pod UID, which offers less granularity of metadata when compared to the container ID
+			if strings.HasPrefix(f.LogsPath, "/var/lib/kubelet/pods/") && strings.HasSuffix(source, ".log") {
+				pathDirs := strings.Split(source, "/")
+				if len(pathDirs) > podUIDPos {
+					podUID := strings.Split(source, "/")[podUIDPos]
+
+					logp.Debug("kubernetes", "Using pod uid: %s", podUID)
+					return podUID
+				}
+
+				logp.Debug("kubernetes", "Error extracting pod uid - source value contains matcher's logs_path, however it is too short to contain a Pod UID.")
+			}
+		} else {
+			// In case of the Kubernetes log path "/var/log/containers/",
+			// the container ID will be located right before the ".log" extension.
+			if strings.HasPrefix(f.LogsPath, "/var/log/containers/") && strings.HasSuffix(source, ".log") && sourceLen >= containerIdLen+4 {
+				containerIDEnd := sourceLen - 4
+				cid := source[containerIDEnd-containerIdLen : containerIDEnd]
+				logp.Debug("kubernetes", "Using container id: %s", cid)
+				return cid
+			}
+
+			// In any other case, we assume the container ID will follow right after the log path.
+			// However we need to check the length to prevent "slice bound out of range" runtime errors.
+			if sourceLen >= logsPathLen+containerIdLen {
+				cid := source[logsPathLen : logsPathLen+containerIdLen]
+				logp.Debug("kubernetes", "Using container id: %s", cid)
+				return cid
+			}
+
+			logp.Debug("kubernetes", "Error extracting container id - source value contains matcher's logs_path, however it is too short to contain a Docker container ID.")
 		}
-
-		logp.Debug("kubernetes", "Error extracting container id - source value contains matcher's logs_path, however it is too short to contain a Docker container ID.")
 	}
 
 	return ""

filebeat/processor/add_kubernetes_metadata/matchers_test.go

@@ -12,6 +12,9 @@ import (
 // A random container ID that we use for our tests
 const cid = "0069869de9adf97f574c62029aeba65d1ecd85a2a112e87fbc28afe4dec2b843"
 
+// A random pod UID that we use for our tests
+const puid = "005f3b90-4b9d-12f8-acf0-31020a840133"
+
 func TestLogsPathMatcher_InvalidSource1(t *testing.T) {
 	cfgLogsPath := "" // use the default matcher configuration
 	source := "/var/log/messages"
@@ -54,18 +57,42 @@ func TestLogsPathMatcher_AnotherLogDir(t *testing.T) {
 	executeTest(t, cfgLogsPath, source, expectedResult)
 }
 
+func TestLogsPathMatcher_VarLibKubeletPods(t *testing.T) {
+	cfgLogsPath := "/var/lib/kubelet/pods/"
+	cfgResourceType := "pod"
+	source := fmt.Sprintf("/var/lib/kubelet/pods/%s/volumes/kubernetes.io~empty-dir/applogs/server.log", puid)
+	expectedResult := puid
+	executeTestWithResourceType(t, cfgLogsPath, cfgResourceType, source, expectedResult)
+}
+
+func TestLogsPathMatcher_InvalidSource4(t *testing.T) {
+	cfgLogsPath := "/var/lib/kubelet/pods/"
+	cfgResourceType := "pod"
+	source := fmt.Sprintf("/invalid/dir/%s/volumes/kubernetes.io~empty-dir/applogs/server.log", puid)
+	expectedResult := ""
+	executeTestWithResourceType(t, cfgLogsPath, cfgResourceType, source, expectedResult)
+}
+
 func executeTest(t *testing.T, cfgLogsPath string, source string, expectedResult string) {
+	executeTestWithResourceType(t, cfgLogsPath, "", source, expectedResult)
+}
+
+func executeTestWithResourceType(t *testing.T, cfgLogsPath string, cfgResourceType string, source string, expectedResult string) {
 	var testConfig = common.NewConfig()
 	if cfgLogsPath != "" {
 		testConfig.SetString("logs_path", -1, cfgLogsPath)
 	}
 
+	if cfgResourceType != "" {
+		testConfig.SetString("resource_type", -1, cfgResourceType)
+	}
+
 	logMatcher, err := newLogsPathMatcher(*testConfig)
 	assert.Nil(t, err)
 
 	input := common.MapStr{
 		"source": source,
 	}
 	output := logMatcher.MetadataIndex(input)
-	assert.Equal(t, output, expectedResult)
+	assert.Equal(t, expectedResult, output)
 }

heartbeat/docs/fields.asciidoc

@@ -648,6 +648,16 @@ type: keyword
 Kubernetes pod name
 
 
+--
+
+*`kubernetes.pod.uid`*::
++
+--
+type: keyword
+
+Kubernetes pod uid
+
+
 --
 
 *`kubernetes.namespace`*::

heartbeat/include/fields.go

@@ -15,5 +15,5 @@ func init() {
 
 // Asset returns asset data
 func Asset() string {
-	return "eJzsW99z27gRfvdfsZOndkbWNEmT6fih09RJe5q73Hls91mGgJWIMwgwAGhZN/3jO/hFgiKpH5ZydWfqlwtJcL8F9sO3u6Du4hIecXMFCyT2AsByK/AK/h6uGBqqeWW5klfw1wsAgGslLeHSAFVlqaR/D5YcBTNAnggXZCEQuAQiBOATSgt2U6GZXkAcdnXhDV2CJCUG4Kn7p787iOn+7gv0L4Bagi3QewgGJeNy5W8ItYISjSErNFOYZaP8a9w0pgxa56B7TpVc8lWtiYODJRc4cffdQ2LhiYjavQm1QeZtcusupbK5Mf8KFMrYiBTH3ysP1fFj4p75Ww/u8qGxo/yMx/2a9hctIe5fuMY3YkCjrbVEBouNh1IVOhi5ArMxFktQEtYFp0XreLZ2upaSy9WAN5aX+JuSB3iTRn5Pb55QG67kfmfiwEQrT2cf/BVK5woysAU3gcrTLnXf/M1NxVhSVm+iUcf1K2DEpnXQ+K3mGtkVWF2nm0ulS2I74/CZlJXbep/qVW0svPtoC3j3p7cfJ/D23dX7D1cf3k/fv3932Op6l2AdiIxxG7oNopEqzWBNTDu/rUlZsjK7UT7pBbea6I0fG1aLEicFnu8V6hAoIpm/sJpIQ6ht4xHWaQs4qENnHdXiV6Rpr4WLeXjyiJu10my3o41W1QZ1u6ecQAWwLQ9Qa6U7Dqy0qqvdIF/cS0kBaUB0/CWMcTeWCOByqdzOpsR4/fI4ZprIEFUxGUzeRDFr7iefLD7b7OaIW61r0c60B0AV61sXSq6Ose6M9E07Wz3T3ZgdZD3QJKYoKlTN2hx17S6h0uqJM3TTtIQRS4bT1tf4FJZalcFS86pxsWoliDA29wPmyaQbSdEYpUezmBs69W9Nk9ntjY10z+79OUtvXQ+ncKOM4Y64PicZIBqdwQmsKE5AaWB8xS0RiiKR01HfuDSWSIpzvmfrzOJAmH1OLrkkAiWhBZfbW3cIYX9majDyvH4YShwwz3jWrLN9Ny2R8brcjf41mPAUOw48ljlccLuZZymv8aA2l0iMvXxL9whpZgh8RuRttuMmuMNNm+Z2UM5rYxPVxpX45PL5cOrFV5wv/1RqJTDstHF0jau9qfbWj9k3v7jRmaKPfv/Enf45XQ8YD8/AWGKd/AqB1OVsv83DM7dnTaG0nYcMcAVLIowLGpG0UDrhXTa7/KIrymnKjVswmB/GdDzmBNRTzk7TxH9J/q3G1iBwNqTqDVw5lD6OQsx54c2l6jQ64AqJRc2FBSV3uZKJwQs9uW4wna1dWIIsUJgeWqeWgN31xB5fZn4lAk5DWkfmlrI/hKsBIzNXDGREdVmuJz0tN939vcyM2Mfx8vSY/BDbin40zsT0IBADJCeaFtwitbU+wxw65uAPOF1N4fkvH+cf/zwBossJVBWdQMkr88e+K8pMK0GsK+lP8+SXO0iGog8UpVVmAvWilraewJpLptYjTnQ7npf7EO0MYixJycXmZIhgJk5SIyuInQDDBSdyAkuNuDBs12x51XOhc2sH+k/cWCdos5tLwphGY9D0AUpCT5tkgimIZmuisQWbQG1qIsQGvn66zn1IOvJYL1BLtGhaNfkxvzcA2z5vyuBuTdsahVxLdqfF9qW9AtRxGo6SoUqxM6SHbAUqxYK2DSqeqQg9H1ZrsQ/mGqOzTsxZHJnZoTnvMKBgDUpS9ZGIlMr6Y6mzwWUmhzHPWUdkuLRTUuyCPUMlNYgb7Kb21h+otpv+zXU4YS2QaOvPpUoluVX6zZYIjOzJOHp0Q44emnjU+Hb3oGR8F5/e8N8X2IBmp0Nw1oolAzHNgRCO6cWpVUwHbVkLAb+qheuoSTgidurcRHdgviweAPe8yMPY8+FeWSISrj8dR2OHbG3HMoeuTef2yOlQD/tzOrLmEkpOtTJIlWSmPzdDCzw1mp9C9oRai2hvCv9QOnW/8GBp9TCBByuM+09hrbskkoV/m4eBNc9K6Rd6lapil/8N6idOERboAhFjgmwK1+G8tOTGcLmaAG/H8u7SNy85tsxuBlw+oRya3ez0cpZ71fUkfVKYdOz5Dxu8egjcSkpn/H2NRoknZMAriHVP0/3QWmuU1lsdmKHr7juMHDxVf2G8ZpJxSpzs8GWjQFTVgsETEZwRG7retBJWucg137rayi1OMFNw31wIpR7r6kDRbm3AMaKdATWKHZ6MCfbvw/Nzk7XlTS3b3nnFn1COcUfb/jx36mejYYlkLuIxLkCkc8OfLhW97vd3ltRIO6Poo/mQse7ul+sf7z64Qv95cyDtGhtHsS4HAo3Cfy3rLsEY/Y6OylZm/ekOBNmgBu2ZYDWvwpevQ6NBlZTdYnXckT3OeId4iR3CoLFkIbgpgCQsF8QnTtKyuUGSVYrLbS8AFsSVB0pmn9QzI1Z1ln669frQtGEXEWEXGXuTHydkYqQVWe/6xsUKJdWb8IXbh+1AWlox3k+OfQJtmdEh5D49/K8TsiCSmYI84nej5JJLx0fnagOWMU1oJGyTMU6iXSv92DPcMvHVMC+dwVpb5en3/v7myKYpWhhe+LHk62COY1utRY9th3/XvYvZ1tW9qY+I08wjUtbC8nk/Jg3lyfqiH4h+DbCHafdZbTTkEdwX3Lj6kYBU8pJIIja/pZUKvx4IP5lZ1mKbT0oDWa00rsLxwFB6R1Mpafo14BGbN61nsgUV0aREi/rg3durUFtHuLS4ar4bHbSkALfJlWB45DP7iaLliXuaaqUK+WWi9e/evJstvkC7RpSw5NpYWGysrzPjVvtWu5Y2FJtrza1F6dq6nrUmoGFoPMUM9IyeO4K2oFti2DPYE8eOGPaG/6ysI8CyBYs/HwNn3rm0UGwDSoOSYgMEKo1L/jzxn0cH1ND9ybpcoAamMFha1kK46qvSaPyv7QrXolgifCBBIjLsr0wMk/KOhF8+KfZ6KokhsES1ufP07HxTeZC6lT9h/syg0wwOyE74C8v4fybAd2SC2/I4jzLwIibs5EH+q1KqykqgxY7yDChGXynGyqnXWD4NgSWGzwskrJe+XrjM3Zo0aXy+4MYSbbej4BZ/QNxDGnB7M8sSvjeP0eqov4tHG7kdLdf/euT8nj+7QOaaGBNpE5v8J55TuHMhNLDmtuiZ8z8kkdxyIuD++ibvaom1WFZ2Cl8kC28DWVrUrWT2rDHOgBZIHzua/Jrl97UQJzZMnJZ5wzS7/npzYKMU34RjGqXZDVRurQ/syMP+7n9m7BfUu37BFKLEl+AmB19ooW6jYS8x5zgxbCzDbaZJt1g5PnQL6wPL6nOfFaaDGZpH2+2/o05j6NERdxBJPV9yKlMp3Y/FUfFPvZ2zFLfsOUK+dfpzfWofdebjyEHVzo8kt7T3iManPYV/LVr2HRrTHSvaNgruylis2tXDZ278/7zSXd7XslD/CQAA//8Vezee"
+	return "eJzsW19z27gRf/en2MlTOyNrmqTJdPzQaeqkPc1d7jy2+yxDwErEGQQYALSsm374Dv6RoEjqj6Vc3ZnqJSYJ7C52f/jtLshcXMIjbq5ggcReAFhuBV7B38MVQ0M1ryxX8gr+egEAcK2kJVwaoKoslfTzYMlRMAPkiXBBFgKBSyBCAD6htGA3FZrpBcRhVxde0CVIUmJQPHV/+ruDOt3vvkA/AdQSbIHeQjAoGZcrf0OoFZRoDFmhmcIsG+WncdOIMmidge45VXLJV7UmTh0sucCJu+8eEgtPRNRuJtQGmZfJrbuUyubC/BQolLFRUxx/r7yqjh0T98zfenCXD40c5Vc8bte077Skcb/jGtuIAY221hIZLDZelarQqZErMBtjsQQlYV1wWrSGZ77TtZRcrgassbzE35Q8wJo08nta84TacCX3GxMHJlh5OPvgr1A6U5CBLbgJUJ52ofvmb24pxpKyehOFOqxfASM2+UHjt5prZFdgdZ1uLpUuie2Mw2dSVm7rfapXtbHw7qMt4N2f3n6cwNt3V+8/XH14P33//t1h3vUmwToAGeM2dBtEI1WawZqYdn1bi7JkZXZr+aQX3GqiN35s8BYljgo83ivUIVBEMn9hNZGGUNvGI/hpS3Fgh44f1eJXpGmvhYt5ePKIm7XSbLehDVfVBnW7pxxBBWVbFqDWSncMWGlVV7uVfHGTEgPSoNHhlzDG3VgigMulcjubEuP5y+sx0wSGyIpJYLImkllzP9lk8dlmN0fMak2LcqY9BVSxvnSh5OoY6U5IX7ST1RPdjdlB0gNMYoqiQtWszVHX7hIqrZ44Q7dMSxixZDhtfY1PYalVGSQ1U42LVUtBhLG5HzBPIt1IisYoPZrF3NCpnzVNYrc3NtI9u/fnLL11LZzCjTKGO+D6nGSAaHQCJ7CiOAGlgfEVt0QoikROR23j0lgiKc75nq0ziwNh9jmZ5JIIlIQWXG5v3SEN+zNToyPP64dpiQPmGc4aP9t30xIZr8vd2r8GER5ixymPZQ4X3G7mWcprLKjNJRJjL9/SPUSaCQKfEXmb7bgJ5nDTprkdkPPc2ES1MSU+uXw+HHpxirPln0qtBIadNq5d42pvqr31Y/atL250puij3z9xp39O1wPCwzMwllhHv0IgdTnbb/PwzO1ZUyht5yEDXMGSCOOCRiQtlE76LptdftEl5bTkxiwYzA9jPB5zAuopZ6dx4r8k/1ZjKxA4G2L1Rl05lD6O0pjjwotL1Wk0wBUSi5oLC0ruMiUjgxdact3odLJ26RJkgcL0tHVqCdhdT+yxZeY9EfQ0oHVgbiH7Q7gaEDJzxUAGVJfletTTYtPd34vMqPs4XJ4ekx9iW9GPxpmQHghiAORE04JbpLbWZ1hDRxz8AaerKTz/5eP8458nQHQ5gaqiEyh5Zf7YN0WZaSWIdSX9aZb8cgdJULSBorTKTKBe1NLWE1hzydR6xIhux/NyG6KcQR1LUnKxOVlFEBMXqZEVxE6A4YITOYGlRlwYtmu1vOqZ0Lm1Q/tP3FhHaLObS8KYRmPQ9BWUhJ62yKSmIJqticZW2QRqUxMhNvD103VuQ+KRx3qBWqJF07LJj/m9AbXt86YM7ta0rVDIuWR3Wmwn7SWgjtFwFA1Vip0hPWQeqBQL3Daoqj6VmrY0OXmD1GoqQs+3qFZiX5nrwM7qQSdxxIWHJtfDFAVpUJKqr4lIqaw//zqbukzksM5zFiyZXtqpXXapPUPJNqg3yE19tD+5bdnlzXU4yi2QaOsPwEoluVX6zRbbjGz+OHp054+eznitcXb3RGacLk4/WbgvsFGaHUPBWUujTIlpTp5wjJhO5aSOtmUtBPyqFq51J+Es2qWBJroD62XxpLlnRR7Gng33yhKR9PpjeDR2SNZ2LHPVtencHjmG6un+nM7GuYSSU60MUiWZ6a/N0AJPjeankKah1iLKm8I/lE5tNjxYWj1M4MEK4/4prHWXRLLwt3kY8HlWs7/QqlR+u0LDoH7iFGGBLhAxJsimcB0OZktuDJerCfB2LO+6vpnk0DK7GTD5hLprdrPTylluVdeS9O5i0pHn36Dw6iFgKzGd8fc1GiWekAGvIBZYTZtFa61RWi91YIXGEttB5ODx/QvjNZOMU+Johy8bBqKqFgyeiOCM2NBeJ09Y5SLXvFRrS8S4wIzBfRcjlHqsqwNJu5UBx5B2pqhh7PBkjLB/H5yfG6wtbmrZNukr/oRyDDva9te5kz8bDksgcxGPcQEinRn+GKvotdm/M6VG2BlFH82HDHV3v1z/ePfBdRTPmwNh18g4CnW5ItAo/Gu5rgvG4Hd0VLYy6093IMgGNWiPBKt5FV6xHRoNqqTsFqvjhuwxxhvES+wABo0lC8FNASTpckF84iS5zQ2SrFJcblsBsCCuPFAye3efCbGq4/rp1vShZcMuIMIuMPYWPw7IhEgrsib5jYsVSqo34VW6D9uBsLRivHEde9faIqMDyH18+F8HZEEkMwV5xO8GySWXDo/O1EZZhjShkbBNhjiJdq30Y09wi8RXg7x02Gttlaff+/ubI5umKGHY8WPJ16k5Dm21Fj20Hf4C+S5mW1f3pj4iLjOPSFkLy+f9mDSQJ+uLfiD6NcAepN1ntdGQRXBfcOPqRwJSyUsiidj8ljwVPlMI3+Ysa7GNJ6WBrFYaV+F4YCi9o6mUNP0a8IjNm/yZZEFFNCnRoj549/Yq1NYQLi2umhdUB7kU4DaZEgSPvM8/kbQ8cE9jrVQhv4y0/t1bd7PFF2jXiBKWXBsLi431dWbcat9q19KGYnOtubUoXVvXk9YENAyNx6UBntFyB9BW6RYZ9gT2yLFDhr3hPyvrALBslcXv1MCJdyYtFNuA0qCk2ACBSuOSP0/8e9gBNnQ/WZcL1MAUBknLWghXfVUajf+sr3AtiiXCBxIkIsO+Z2KYlDckfGKl2OupJIaUJajNnaVnx5vKg9St/AnzZwadZnCAdsIvuPH/SIDviAS35XEeaeBFSNiJg/zzVarKSqDFDvMMMEafKcbKqddYPg0pSwifF0hYL3290M3dmjRxfO5wY4m221Fwzh8g95AG3N7MsoTvzWO0Ouzv4tFGbkfL9b8eOb/nz06QOSfGRNrEJv+WdAp3LoQG1twWPXH+ixXJLScC7q9v8q6WWItlZafwRbIwG8jSom4psyeNcQa0QPrY4eTXTL+vBTixYeK0zBum2fXXmwMbpTgTjmmUZjdQOV8f2JGH/d1/zdgvqHd9KhWixJfgFgdfaKFuo2BPMec4MWwkw23GSbdYOTx0C+sDy+pznxWmgxmaR9vtv6NOY+jREXcqEnu+5FSmUrofi6Pin3o7Jylu2XOEfOv05/rUPurMx5GDrJ0fSW5x7xGNT3sK/1q47Ds0pjs82jYK7spYrFrv4TM3/n/JdN37Whz1nwAAAP//E8FTBQ=="
 }

libbeat/autodiscover/providers/kubernetes/config.go

@@ -19,6 +19,7 @@ type Config struct {
 	IncludeLabels      []string `config:"include_labels"`
 	ExcludeLabels      []string `config:"exclude_labels"`
 	IncludeAnnotations []string `config:"include_annotations"`
+	IncludePodUID      bool     `config:"include_pod_uid"`
 
 	Prefix       string                  `config:"prefix"`
 	HintsEnabled bool                    `config:"hints.enabled"`

libbeat/autodiscover/providers/kubernetes/kubernetes.go

@@ -43,7 +43,7 @@ func AutodiscoverBuilder(bus bus.Bus, c *common.Config) (autodiscover.Provider,
 		return nil, err
 	}
 
-	metagen := kubernetes.NewMetaGenerator(config.IncludeAnnotations, config.IncludeLabels, config.ExcludeLabels)
+	metagen := kubernetes.NewMetaGenerator(config.IncludeAnnotations, config.IncludeLabels, config.ExcludeLabels, config.IncludePodUID)
 
 	config.Host = kubernetes.DiscoverKubernetesNode(config.Host, config.InCluster, client)
 

libbeat/common/kubernetes/metadata.go

@@ -18,14 +18,16 @@ type metaGenerator struct {
 	annotations   []string
 	labels        []string
 	labelsExclude []string
+	poduid        bool
 }
 
 // NewMetaGenerator initializes and returns a new kubernetes metadata generator
-func NewMetaGenerator(annotations, labels, labelsExclude []string) MetaGenerator {
+func NewMetaGenerator(annotations, labels, labelsExclude []string, includePodUID bool) MetaGenerator {
 	return &metaGenerator{
 		annotations:   annotations,
 		labels:        labels,
 		labelsExclude: labelsExclude,
+		poduid:        includePodUID,
 	}
 }
 
@@ -56,6 +58,11 @@ func (g *metaGenerator) PodMetadata(pod *Pod) common.MapStr {
 		"namespace": pod.Metadata.Namespace,
 	}
 
+	// Add Pod UID metadata if enabled
+	if g.poduid {
+		safemapstr.Put(meta, "pod.uid", pod.Metadata.UID)
+	}
+
 	if len(labelMap) != 0 {
 		meta["labels"] = labelMap
 	}

libbeat/common/kubernetes/metadata_test.go

@@ -10,20 +10,43 @@ import (
 
 func TestPodMetadataDeDot(t *testing.T) {
 	tests := []struct {
-		pod  *Pod
-		meta common.MapStr
+		pod     *Pod
+		meta    common.MapStr
+		metaGen MetaGenerator
 	}{
 		{
 			pod: &Pod{
 				Metadata: ObjectMeta{
 					Labels: map[string]string{"a.key": "foo", "a": "bar"},
+					UID:    "005f3b90-4b9d-12f8-acf0-31020a840133",
 				},
 			},
-			meta: common.MapStr{"labels": common.MapStr{"a": common.MapStr{"value": "bar", "key": "foo"}}},
+			meta: common.MapStr{
+				"pod":       common.MapStr{"name": ""},
+				"namespace": "",
+				"node":      common.MapStr{"name": ""},
+				"labels":    common.MapStr{"a": common.MapStr{"value": "bar", "key": "foo"}},
+			},
+			metaGen: NewMetaGenerator(nil, nil, nil, false),
+		},
+		{
+			pod: &Pod{
+				Metadata: ObjectMeta{
+					Labels: map[string]string{"a.key": "foo", "a": "bar"},
+					UID:    "005f3b90-4b9d-12f8-acf0-31020a840133",
+				},
+			},
+			meta: common.MapStr{
+				"pod":       common.MapStr{"name": "", "uid": "005f3b90-4b9d-12f8-acf0-31020a840133"},
+				"namespace": "",
+				"node":      common.MapStr{"name": ""},
+				"labels":    common.MapStr{"a": common.MapStr{"value": "bar", "key": "foo"}},
+			},
+			metaGen: NewMetaGenerator(nil, nil, nil, true),
 		},
 	}
 
 	for _, test := range tests {
-		assert.Equal(t, NewMetaGenerator(nil, nil, nil).PodMetadata(test.pod)["labels"], test.meta["labels"])
+		assert.Equal(t, test.metaGen.PodMetadata(test.pod), test.meta)
 	}
 }

libbeat/processors/add_kubernetes_metadata/_meta/fields.yml

@@ -13,6 +13,11 @@
           description: >
             Kubernetes pod name
 
+        - name: pod.uid
+          type: keyword
+          description: >
+            Kubernetes pod uid
+
         - name: namespace
           type: keyword
           description: >

libbeat/processors/add_kubernetes_metadata/config.go

@@ -22,6 +22,7 @@ type kubeAnnotatorConfig struct {
 	IncludeLabels      []string      `config:"include_labels"`
 	ExcludeLabels      []string      `config:"exclude_labels"`
 	IncludeAnnotations []string      `config:"include_annotations"`
+	IncludePodUID      bool          `config:"include_pod_uid"`
 }
 
 type Enabled struct {

libbeat/processors/add_kubernetes_metadata/indexers.go

@@ -12,6 +12,7 @@ import (
 const (
 	ContainerIndexerName = "container"
 	PodNameIndexerName   = "pod_name"
+	PodUIDIndexerName    = "pod_uid"
 	IPPortIndexerName    = "ip_port"
 )
 
@@ -128,6 +129,32 @@ func (p *PodNameIndexer) GetIndexes(pod *kubernetes.Pod) []string {
 	return []string{fmt.Sprintf("%s/%s", pod.Metadata.Namespace, pod.Metadata.Name)}
 }
 
+// PodUIDIndexer indexes pods based on the pod UID
+type PodUIDIndexer struct {
+	metaGen kubernetes.MetaGenerator
+}
+
+// NewPodUIDIndexer initializes and returns a PodUIDIndexer
+func NewPodUIDIndexer(_ common.Config, metaGen kubernetes.MetaGenerator) (Indexer, error) {
+	return &PodUIDIndexer{metaGen: metaGen}, nil
+}
+
+// GetMetadata returns the composed metadata from PodNameIndexer and the pod UID
+func (p *PodUIDIndexer) GetMetadata(pod *kubernetes.Pod) []MetadataIndex {
+	data := p.metaGen.PodMetadata(pod)
+	return []MetadataIndex{
+		{
+			Index: pod.Metadata.UID,
+			Data:  data,
+		},
+	}
+}
+
+// GetIndexes returns the indexes for the given Pod
+func (p *PodUIDIndexer) GetIndexes(pod *kubernetes.Pod) []string {
+	return []string{pod.Metadata.UID}
+}
+
 // ContainerIndexer indexes pods based on all their containers IDs
 type ContainerIndexer struct {
 	metaGen kubernetes.MetaGenerator