fix #800: shared mem on linux

HISTORY.rst

@@ -1,5 +1,13 @@
 Bug tracker at https://github.com/giampaolo/psutil/issues
 
+4.2.0 - XXXX-XX-XX
+==================
+
+**Enhancements**
+
+- #800: [Linux] psutil.virtual_memory() returns a new "shared" memory field.
+
+
 4.1.0 - 2016-03-12
 ==================
 

IDEAS

@@ -91,18 +91,9 @@ FEATURES
   Also, we can probably reimplement wait_pid() on POSIX which is currently
   implemented as a busy-loop.
 
-- Certain systems provide CPU times about process children. On those systems
-  Process.cpu_times() might return a (user, system, user_children,
-  system_children) ntuple.
-  - Linux: /proc/{PID}/stat
-  - Solaris: pr_cutime and pr_cstime
-  - FreeBSD: none
-  - OSX: none
-  - Windows: none
-
-- ...also, os.times() provides 'elapsed' times as well.
-
-- ...also Linux provides guest_time and cguest_time.
+- os.times() provides 'elapsed' times (cpu_times() might).
+
+- ...also guest_time and cguest_time on Linux.
 
 - Enrich exception classes hierarchy on Python >= 3.3 / post PEP-3151 so that:
    - NoSuchProcess inherits from ProcessLookupError

README.rst

@@ -126,7 +126,7 @@ Memory
 .. code-block:: python
 
     >>> psutil.virtual_memory()
-    svmem(total=8374149120, available=2081050624, percent=75.1, used=8074080256, free=300068864, active=3294920704, inactive=1361616896, buffers=529895424, cached=1251086336)
+    svmem(total=10367352832, available=6472179712, percent=37.6, used=8186245120, free=2181107712, active=4748992512, inactive=2758115328, buffers=790724608, cached=3500347392, shared=787554304)
     >>> psutil.swap_memory()
     sswap(total=2097147904, used=296128512, free=1801019392, percent=14.1, sin=304193536, sout=677842944)
     >>>

docs/index.rst

@@ -197,10 +197,10 @@ Memory
   - **inactive** *(UNIX)*: memory that is marked as not used.
   - **buffers** *(Linux, BSD)*: cache for things like file system metadata.
   - **cached** *(Linux, BSD)*: cache for various things.
+  - **shared** *(Linux, BSD)*: memory that may be simultaneously accessed by
+    multiple processes.
   - **wired** *(BSD, OSX)*: memory that is marked to always stay in RAM. It is
     never moved to disk.
-  - **shared** *(BSD)*: memory that may be simultaneously accessed by multiple
-    processes.
 
   The sum of **used** and **available** does not necessarily equal **total**.
   On Windows **available** and **free** are the same.
@@ -210,14 +210,15 @@ Memory
     >>> import psutil
     >>> mem = psutil.virtual_memory()
     >>> mem
-    svmem(total=8374149120L, available=1247768576L, percent=85.1, used=8246628352L, free=127520768L, active=3208777728, inactive=1133408256, buffers=342413312L, cached=777834496)
+    svmem(total=10367352832, available=6472179712, percent=37.6, used=8186245120, free=2181107712, active=4748992512, inactive=2758115328, buffers=790724608, cached=3500347392, shared=787554304)
     >>>
     >>> THRESHOLD = 100 * 1024 * 1024  # 100MB
     >>> if mem.available <= THRESHOLD:
     ...     print("warning")
     ...
     >>>
 
+  .. versionchanged:: 4.2.0 added *shared* metrics on Linux.
 
 .. function:: swap_memory()
 

psutil/__init__.py

@@ -187,7 +187,7 @@
 ]
 __all__.extend(_psplatform.__extra__all__)
 __author__ = "Giampaolo Rodola'"
-__version__ = "4.1.0"
+__version__ = "4.2.0"
 version_info = tuple([int(num) for num in __version__.split('.')])
 AF_LINK = _psplatform.AF_LINK
 _TOTAL_PHYMEM = None

psutil/_pslinux.py

@@ -240,7 +240,7 @@ def set_scputimes_ntuple(procfs_path):
 
 svmem = namedtuple(
     'svmem', ['total', 'available', 'percent', 'used', 'free',
-              'active', 'inactive', 'buffers', 'cached'])
+              'active', 'inactive', 'buffers', 'cached', 'shared'])
 sdiskio = namedtuple('sdiskio', ['read_count', 'write_count',
                                  'read_bytes', 'write_bytes',
                                  'read_time', 'write_time',
@@ -266,36 +266,59 @@ def virtual_memory():
     total *= unit_multiplier
     free *= unit_multiplier
     buffers *= unit_multiplier
-    # XXX: tis is currently not used (neither returned) because it's
-    # always 0. It would be nice to have though ('free' provides it).
-    # shared *= unit_multiplier
+    # Note: this (on my Ubuntu 14.04, kernel 3.13 at least) may be 0.
+    # If so, it will be determined from /proc/meminfo.
+    shared *= unit_multiplier or None
+    if shared == 0:
+        shared = None
 
     cached = active = inactive = None
     with open_binary('%s/meminfo' % get_procfs_path()) as f:
         for line in f:
-            if line.startswith(b"Cached:"):
+            if cached is None and line.startswith(b"Cached:"):
                 cached = int(line.split()[1]) * 1024
-            elif line.startswith(b"Active:"):
+            elif active is None and line.startswith(b"Active:"):
                 active = int(line.split()[1]) * 1024
-            elif line.startswith(b"Inactive:"):
+            elif inactive is None and line.startswith(b"Inactive:"):
                 inactive = int(line.split()[1]) * 1024
-            if (cached is not None and
-                    active is not None and
-                    inactive is not None):
-                break
-        else:
-            # we might get here when dealing with exotic Linux flavors, see:
-            # https://github.com/giampaolo/psutil/issues/313
-            msg = "'cached', 'active' and 'inactive' memory stats couldn't " \
-                  "be determined and were set to 0"
-            warnings.warn(msg, RuntimeWarning)
-            cached = active = inactive = 0
+            # From "man free":
+            # The shared memory column represents either the MemShared
+            # value (2.4 kernels) or the Shmem value (2.6+ kernels) taken
+            # from the /proc/meminfo file. The value is zero if none of
+            # the entries is exported by the kernel.
+            elif shared is None and \
+                    line.startswith(b"MemShared:") or \
+                    line.startswith(b"Shmem:"):
+                shared = int(line.split()[1]) * 1024
+
+    missing = []
+    if cached is None:
+        missing.append('cached')
+        cached = 0
+    if active is None:
+        missing.append('active')
+        active = 0
+    if inactive is None:
+        missing.append('inactive')
+        inactive = 0
+    if shared is None:
+        missing.append('shared')
+        shared = 0
+    if missing:
+        msg = "%s memory stats couldn't be determined and %s set to 0" % (
+            ", ".join(missing),
+            "was" if len(missing) == 1 else "were")
+        warnings.warn(msg, RuntimeWarning)
 
+    # Note: this value matches "htop" perfectly.
     avail = free + buffers + cached
+    # Note: this value matches "free", but not all the time, see:
+    # https://github.com/giampaolo/psutil/issues/685#issuecomment-202914057
     used = total - free
+    # Note: this value matches "htop" perfectly.
     percent = usage_percent((total - avail), total, _round=1)
     return svmem(total, avail, percent, used, free,
-                 active, inactive, buffers, cached)
+                 active, inactive, buffers, cached, shared)
 
 
 def swap_memory():

psutil/tests/test_linux.py

@@ -156,6 +156,12 @@ def test_cached(self):
         self.assertAlmostEqual(cached, psutil.virtual_memory().cached,
                                delta=MEMORY_TOLERANCE)
 
+    @retry_before_failing()
+    def test_shared(self):
+        total, used, free, shared = free_physmem()
+        self.assertAlmostEqual(shared, psutil.virtual_memory().shared,
+                               delta=MEMORY_TOLERANCE)
+
     # --- mocked tests
 
     def test_warnings_mocked(self):
@@ -168,8 +174,7 @@ def test_warnings_mocked(self):
                 w = ws[0]
                 self.assertTrue(w.filename.endswith('psutil/_pslinux.py'))
                 self.assertIn(
-                    "'cached', 'active' and 'inactive' memory stats couldn't "
-                    "be determined", str(w.message))
+                    "memory stats couldn't be determined", str(w.message))
                 self.assertEqual(ret.cached, 0)
                 self.assertEqual(ret.active, 0)
                 self.assertEqual(ret.inactive, 0)