Use sudo for privileged process checks on filedescriptors (take 2)

process/README.md

@@ -37,7 +37,21 @@ You can also configure the check to find any process by exact PID (`pid`) or pid
 
 To have the check search for processes in a path other than `/proc`, set `procfs_path: <your_proc_path>` in `datadog.conf`, NOT in `process.yaml` (its use has been deprecated there). Set this to `/host/proc` if you're running the Agent from a Docker container (i.e. [docker-dd-agent](https://github.com/DataDog/docker-dd-agent)) and want to monitor processes running on the server hosting your containers. You DON'T need to set this to monitor processes running _in_ your containers; the [Docker check][5] monitors those.  
 
-See the [example configuration][3] for more details on configuration options.  
+Some process metrics require either running the datadog collector as the same user as the monitored process or privileged access to be retrieved.
+Where the former option is not desired, and to avoid running the datadog collector as `root`, the `try_sudo` option lets the process check try using `sudo` to collect this metric.
+As of now, only the `open_fd` metric on Unix platforms is taking advantage of this setting.
+Note: the appropriate sudoers rules have to be configured for this to work, e.g. if packaged as a wheel archive with the datadog agent
+```
+dd-agent ALL=NOPASSWD: /opt/datadog-agent/embedded/bin/python /opt/datadog-agent/embedded/lib/python2.7/site-packages/datadog_checks/process/process.py num_fds *
+dd-agent ALL=NOPASSWD: /opt/datadog-agent/embedded/bin/python /opt/datadog-agent/embedded/lib/python2.7/site-packages/datadog_checks/process/process.pyc num_fds *
+```
+Before agent v5.22 and v6.0, integrations were not packaged as wheel archives and the path to the check was a little different:
+```
+dd-agent ALL=NOPASSWD: /opt/datadog-agent/embedded/bin/python /opt/datadog-agent/agent/checks.d/process.py num_fds *
+dd-agent ALL=NOPASSWD: /opt/datadog-agent/embedded/bin/python /opt/datadog-agent/agent/checks.d/process.pyc num_fds *
+```
+
+See the [example configuration][3] for more details on configuration options.
 
 [Restart the Agent][6] to start sending process metrics and service checks to Datadog.
 

process/datadog_checks/process/process.py

@@ -7,15 +7,25 @@
 from collections import defaultdict
 import time
 import os
+import subprocess
+import sys
 # 3p
 import psutil
 
+
 # project
 from checks import AgentCheck
 from config import _is_affirmative
 from utils.platform import Platform
 
 
+# Main entry point is meant for checks needing privilege escalation with sudo
+if __name__ == "__main__":
+    if sys.argv[1] == "num_fds":
+        print psutil.Process(int(sys.argv[2])).num_fds()
+    sys.exit(0)
+
+
 DEFAULT_AD_CACHE_DURATION = 120
 DEFAULT_PID_CACHE_DURATION = 120
 
@@ -164,7 +174,7 @@ def find_pids(self, name, search_string, exact_match, ignore_ad=True):
             self.last_ad_cache_ts[name] = time.time()
         return matching_pids
 
-    def psutil_wrapper(self, process, method, accessors, *args, **kwargs):
+    def psutil_wrapper(self, process, method, accessors, try_sudo, *args, **kwargs):
         """
         A psutil wrapper that is calling
         * psutil.method(*args, **kwargs) and returns the result
@@ -201,12 +211,20 @@ def psutil_wrapper(self, process, method, accessors, *args, **kwargs):
             self.log.debug("psutil method %s not implemented", method)
         except psutil.AccessDenied:
             self.log.debug("psutil was denied acccess for method %s", method)
+            if method == 'num_fds' and Platform.is_unix() and try_sudo:
+                try:
+                    # It is up the agent's packager to grant corresponding sudo policy on unix platforms
+                    result = int(subprocess.check_output(['sudo', sys.executable, __file__, method, str(process.pid)]))
+                except subprocess.CalledProcessError as e:
+                    self.log.exception("running psutil method %s with sudo failed with return code %d", method, e.returncode)
+                except:
+                    self.log.exception("running psutil method %s with sudo also failed", method)
         except psutil.NoSuchProcess:
             self.warning("Process {0} disappeared while scanning".format(process.pid))
 
         return result
 
-    def get_process_state(self, name, pids):
+    def get_process_state(self, name, pids, try_sudo):
         st = defaultdict(list)
 
         # Remove from cache the processes that are not in `pids`
@@ -234,36 +252,36 @@ def get_process_state(self, name, pids):
 
             p = self.process_cache[name][pid]
 
-            meminfo = self.psutil_wrapper(p, 'memory_info', ['rss', 'vms'])
+            meminfo = self.psutil_wrapper(p, 'memory_info', ['rss', 'vms'], try_sudo)
             st['rss'].append(meminfo.get('rss'))
             st['vms'].append(meminfo.get('vms'))
 
-            mem_percent = self.psutil_wrapper(p, 'memory_percent', None)
+            mem_percent = self.psutil_wrapper(p, 'memory_percent', None, try_sudo)
             st['mem_pct'].append(mem_percent)
 
             # will fail on win32 and solaris
-            shared_mem = self.psutil_wrapper(p, 'memory_info_ex', ['shared']).get('shared')
+            shared_mem = self.psutil_wrapper(p, 'memory_info_ex', ['shared'], try_sudo).get('shared')
             if shared_mem is not None and meminfo.get('rss') is not None:
                 st['real'].append(meminfo['rss'] - shared_mem)
             else:
                 st['real'].append(None)
 
-            ctxinfo = self.psutil_wrapper(p, 'num_ctx_switches', ['voluntary', 'involuntary'])
+            ctxinfo = self.psutil_wrapper(p, 'num_ctx_switches', ['voluntary', 'involuntary'], try_sudo)
             st['ctx_swtch_vol'].append(ctxinfo.get('voluntary'))
             st['ctx_swtch_invol'].append(ctxinfo.get('involuntary'))
 
-            st['thr'].append(self.psutil_wrapper(p, 'num_threads', None))
+            st['thr'].append(self.psutil_wrapper(p, 'num_threads', None, try_sudo))
 
-            cpu_percent = self.psutil_wrapper(p, 'cpu_percent', None)
+            cpu_percent = self.psutil_wrapper(p, 'cpu_percent', None, try_sudo)
             if not new_process:
                 # psutil returns `0.` for `cpu_percent` the first time it's sampled on a process,
                 # so save the value only on non-new processes
                 st['cpu'].append(cpu_percent)
 
-            st['open_fd'].append(self.psutil_wrapper(p, 'num_fds', None))
-            st['open_handle'].append(self.psutil_wrapper(p, 'num_handles', None))
+            st['open_fd'].append(self.psutil_wrapper(p, 'num_fds', None, try_sudo))
+            st['open_handle'].append(self.psutil_wrapper(p, 'num_handles', None, try_sudo))
 
-            ioinfo = self.psutil_wrapper(p, 'io_counters', ['read_count', 'write_count', 'read_bytes', 'write_bytes'])
+            ioinfo = self.psutil_wrapper(p, 'io_counters', ['read_count', 'write_count', 'read_bytes', 'write_bytes'], try_sudo)
             st['r_count'].append(ioinfo.get('read_count'))
             st['w_count'].append(ioinfo.get('write_count'))
             st['r_bytes'].append(ioinfo.get('read_bytes'))
@@ -283,7 +301,7 @@ def get_process_state(self, name, pids):
                 st['cmajflt'].append(None)
 
             #calculate process run time
-            create_time = self.psutil_wrapper(p, 'create_time', None)
+            create_time = self.psutil_wrapper(p, 'create_time', None, try_sudo)
             if create_time is not None:
                 now = time.time()
                 run_time = now - create_time
@@ -333,6 +351,7 @@ def check(self, instance):
         pid_file = instance.get('pid_file')
         collect_children = _is_affirmative(instance.get('collect_children', False))
         user = instance.get('user', False)
+        try_sudo = instance.get('try_sudo', False)
 
         if self._conflicting_procfs:
             self.warning('The `procfs_path` defined in `process.yaml` is different from the one defined in '
@@ -384,7 +403,7 @@ def check(self, instance):
         if user:
             pids = self._filter_by_user(user, pids)
 
-        proc_state = self.get_process_state(name, pids)
+        proc_state = self.get_process_state(name, pids, try_sudo)
 
         # FIXME 6.x remove the `name` tag
         tags.extend(['process_name:%s' % name, name])

process/test/test_process.py

@@ -188,7 +188,8 @@ def test_psutil_wrapper_simple(self):
         name = self.check.psutil_wrapper(
             self.get_psutil_proc(),
             'name',
-            None
+            None,
+            False
         )
 
         self.assertNotEquals(name, None)
@@ -199,7 +200,8 @@ def test_psutil_wrapper_simple_fail(self):
         name = self.check.psutil_wrapper(
             self.get_psutil_proc(),
             'blah',
-            None
+            None,
+            False
         )
 
         self.assertEquals(name, None)
@@ -210,7 +212,8 @@ def test_psutil_wrapper_accessors(self):
         meminfo = self.check.psutil_wrapper(
             self.get_psutil_proc(),
             'memory_info',
-            ['rss', 'vms', 'foo']
+            ['rss', 'vms', 'foo'],
+            False
         )
 
         self.assertIn('rss', meminfo)
@@ -223,7 +226,8 @@ def test_psutil_wrapper_accessors_fail(self):
         meminfo = self.check.psutil_wrapper(
             self.get_psutil_proc(),
             'memory_infoo',
-            ['rss', 'vms']
+            ['rss', 'vms'],
+            False
         )
 
         self.assertNotIn('rss', meminfo)
@@ -263,7 +267,7 @@ def mock_find_pids(self, name, search_string, exact_match=True, ignore_ad=True,
             idx = search_string[0].split('_')[1]
         return self.CONFIG_STUBS[int(idx)]['mocked_processes']
 
-    def mock_psutil_wrapper(self, process, method, accessors, *args, **kwargs):
+    def mock_psutil_wrapper(self, process, method, accessors, try_sudo, *args, **kwargs):
         if method == 'num_handles':  # remove num_handles as it's win32 only
             return None