From 183d02035341a2237ff3e261cc4468788a605b80 Mon Sep 17 00:00:00 2001 From: jtinkus <35308202+jtinkus@users.noreply.github.com> Date: Tue, 4 Aug 2020 10:09:19 +0300 Subject: [PATCH] add_process_metadata processor adds container id even if process metadata not accessible (#19767) --- CHANGELOG.next.asciidoc | 1 + .../add_process_metadata.go | 38 +++++--- .../add_process_metadata_test.go | 86 ++++++++++++++++++- 3 files changed, 110 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 81e6a650ee7..d62ce252c6d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -364,6 +364,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added the `max_cached_sessions` option to the script processor. {pull}19562[19562] - Add support for DNS over TLS for the dns_processor. {pull}19321[19321] - Set index.max_docvalue_fields_search in index template to increase value to 200 fields. {issue}20215[20215] +- Add capability of enriching process metadata with contianer id also for non-privileged containers in `add_process_metadata` processor. {pull}19767[19767] *Auditbeat* diff --git a/libbeat/processors/add_process_metadata/add_process_metadata.go b/libbeat/processors/add_process_metadata/add_process_metadata.go index 4aff1df8576..c41ca9a73d6 100644 --- a/libbeat/processors/add_process_metadata/add_process_metadata.go +++ b/libbeat/processors/add_process_metadata/add_process_metadata.go @@ -190,15 +190,29 @@ func (p *addProcessMetadata) enrich(event common.MapStr, pidField string) (resul return nil, errors.Errorf("cannot parse field '%s' (not an integer or string)", pidField) } + var meta common.MapStr + metaPtr, err := p.provider.GetProcessMetadata(pid) if err != nil || metaPtr == nil { + // no process metadata, lets still try to get container id p.log.Debugf("failed to get process metadata for PID=%d: %v", pid, err) - return nil, ErrNoProcess + meta = common.MapStr{} + } else { + meta = metaPtr.fields } - meta := metaPtr.fields - if err = p.enrichContainerID(pid, meta); err != nil { - return nil, err + cid, err := p.getContainerID(pid) + if cid == "" || err != nil { + p.log.Debugf("failed to get container id for PID=%d: %v", pid, err) + } else { + if _, err = meta.Put("container", common.MapStr{"id": cid}); err != nil { + return nil, err + } + } + + if len(meta) == 0 { + // no metadata nor container id + return nil, ErrNoProcess } result = event.Clone() @@ -216,8 +230,8 @@ func (p *addProcessMetadata) enrich(event common.MapStr, pidField string) (resul value, err := meta.GetValue(source) if err != nil { - // Should never happen - return nil, err + // skip missing values + continue } if _, err = result.Put(dest, value); err != nil { @@ -228,19 +242,15 @@ func (p *addProcessMetadata) enrich(event common.MapStr, pidField string) (resul return result, nil } -// enrichContainerID adds container.id into meta for mapping to pickup -func (p *addProcessMetadata) enrichContainerID(pid int, meta common.MapStr) error { +func (p *addProcessMetadata) getContainerID(pid int) (string, error) { if p.cidProvider == nil { - return nil + return "", nil } cid, err := p.cidProvider.GetCid(pid) if err != nil { - return err - } - if _, err = meta.Put("container", common.MapStr{"id": cid}); err != nil { - return err + return "", err } - return nil + return cid, nil } // String returns the processor representation formatted as a string diff --git a/libbeat/processors/add_process_metadata/add_process_metadata_test.go b/libbeat/processors/add_process_metadata/add_process_metadata_test.go index 8bb2cd4b625..f9b4aaa681c 100644 --- a/libbeat/processors/add_process_metadata/add_process_metadata_test.go +++ b/libbeat/processors/add_process_metadata/add_process_metadata_test.go @@ -49,12 +49,42 @@ func TestAddProcessMetadata(t *testing.T) { ppid: 0, startTime: startTime, }, + 3: { + name: "systemd", + title: "/usr/lib/systemd/systemd --switched-root --system --deserialize 22", + exe: "/usr/lib/systemd/systemd", + args: []string{"/usr/lib/systemd/systemd", "--switched-root", "--system", "--deserialize", "22"}, + env: map[string]string{ + "HOME": "/", + "TERM": "linux", + "BOOT_IMAGE": "/boot/vmlinuz-4.11.8-300.fc26.x86_64", + "LANG": "en_US.UTF-8", + }, + pid: 1, + ppid: 0, + startTime: startTime, + }, } // mock of the cgroup processCgroupPaths processCgroupPaths = func(_ string, pid int) (map[string]string, error) { testMap := map[int]map[string]string{ - 1: map[string]string{ + 1: { + "cpu": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "net_prio": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "blkio": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "perf_event": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "freezer": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "pids": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "hugetlb": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "cpuacct": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "cpuset": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "net_cls": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "devices": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "memory": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "name=systemd": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + }, + 2: { "cpu": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", "net_prio": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", "blkio": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", @@ -510,6 +540,60 @@ func TestAddProcessMetadata(t *testing.T) { }, }, }, + { + description: "no process metadata available", + config: common.MapStr{ + "match_pids": []string{"system.process.ppid"}, + "cgroup_regex": "\\/.+\\/.+\\/.+\\/([0-9a-f]{64}).*", + }, + event: common.MapStr{ + "system": common.MapStr{ + "process": common.MapStr{ + "ppid": "2", + }, + }, + }, + expected: common.MapStr{ + "system": common.MapStr{ + "process": common.MapStr{ + "ppid": "2", + }, + }, + "container": common.MapStr{ + "id": "b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + }, + }, + }, + { + description: "no container id available", + config: common.MapStr{ + "match_pids": []string{"system.process.ppid"}, + "cgroup_regex": "\\/.+\\/.+\\/.+\\/([0-9a-f]{64}).*", + }, + event: common.MapStr{ + "system": common.MapStr{ + "process": common.MapStr{ + "ppid": "3", + }, + }, + }, + expected: common.MapStr{ + "system": common.MapStr{ + "process": common.MapStr{ + "ppid": "3", + }, + }, + "process": common.MapStr{ + "name": "systemd", + "title": "/usr/lib/systemd/systemd --switched-root --system --deserialize 22", + "executable": "/usr/lib/systemd/systemd", + "args": []string{"/usr/lib/systemd/systemd", "--switched-root", "--system", "--deserialize", "22"}, + "pid": 1, + "ppid": 0, + "start_time": startTime, + }, + }, + }, { description: "without cgroup cache", config: common.MapStr{