bpo-38119: Fix shmem resource tracking

Lib/multiprocessing/managers.py

@@ -1309,14 +1309,6 @@ class SharedMemoryManager(BaseManager):
         _Server = SharedMemoryServer
 
         def __init__(self, *args, **kwargs):
-            if os.name == "posix":
-                # bpo-36867: Ensure the resource_tracker is running before
-                # launching the manager process, so that concurrent
-                # shared_memory manipulation both in the manager and in the
-                # current process does not create two resource_tracker
-                # processes.
-                from . import resource_tracker
-                resource_tracker.ensure_running()
             BaseManager.__init__(self, *args, **kwargs)
             util.debug(f"{self.__class__.__name__} created by pid {getpid()}")
 

Lib/multiprocessing/resource_tracker.py

@@ -39,7 +39,6 @@
 
     _CLEANUP_FUNCS.update({
         'semaphore': _multiprocessing.sem_unlink,
-        'shared_memory': _posixshmem.shm_unlink,
     })
 
 

Lib/multiprocessing/shared_memory.py

@@ -113,9 +113,6 @@ def __init__(self, name=None, create=False, size=0):
                 self.unlink()
                 raise
 
-            from .resource_tracker import register
-            register(self._name, "shared_memory")
-
         else:
 
             # Windows Named Shared Memory
@@ -234,9 +231,7 @@ def unlink(self):
         called once (and only once) across all processes which have access
         to the shared memory block."""
         if _USE_POSIX and self._name:
-            from .resource_tracker import unregister
             _posixshmem.shm_unlink(self._name)
-            unregister(self._name, "shared_memory")
 
 
 _encoding = "utf8"

Lib/test/_test_multiprocessing.py

@@ -3829,6 +3829,27 @@ def test_shared_memory_across_processes(self):
 
         sms.close()
 
+    def test_shared_memory_across_independently_started_processes(self):
+        # A Process may not trigger the same exit path as an independently
+        # spawned process (for example, resource trackers on a separate
+        # process).  This tests that independently created processes can
+        # indeed continue to access shared memory and that that shared
+        # memory persists at least so long as one of the processes with a
+        # handle on it is alive.
+        sms = shared_memory.SharedMemory('test04_indep', True, size=256)
+        self.addCleanup(sms.unlink)
+
+        prog = (
+            "from multiprocessing import shared_memory; "
+            "sms =  shared_memory.SharedMemory('test04_indep'); "
+            "sms.buf[:4] = b'1234'; "
+            "sms.close()"
+        )
+        rc, out, err = test.support.script_helper.assert_python_ok('-c', prog)
+
+        self.assertFalse(err)
+        self.assertEqual(bytes(sms.buf[:4]), b'1234')
+
     @unittest.skipIf(os.name != "posix", "not feasible in non-posix platforms")
     def test_shared_memory_SharedMemoryServer_ignores_sigint(self):
         # bpo-36368: protect SharedMemoryManager server process from
@@ -3853,27 +3874,6 @@ def test_shared_memory_SharedMemoryServer_ignores_sigint(self):
 
         smm.shutdown()
 
-    @unittest.skipIf(os.name != "posix", "resource_tracker is posix only")
-    def test_shared_memory_SharedMemoryManager_reuses_resource_tracker(self):
-        # bpo-36867: test that a SharedMemoryManager uses the
-        # same resource_tracker process as its parent.
-        cmd = '''if 1:
-            from multiprocessing.managers import SharedMemoryManager
-
-
-            smm = SharedMemoryManager()
-            smm.start()
-            sl = smm.ShareableList(range(10))
-            smm.shutdown()
-        '''
-        rc, out, err = test.support.script_helper.assert_python_ok('-c', cmd)
-
-        # Before bpo-36867 was fixed, a SharedMemoryManager not using the same
-        # resource_tracker process as its parent would make the parent's
-        # tracker complain about sl being leaked even though smm.shutdown()
-        # properly released sl.
-        self.assertFalse(err)
-
     def test_shared_memory_SharedMemoryManager_basics(self):
         smm1 = multiprocessing.managers.SharedMemoryManager()
         with self.assertRaises(ValueError):
@@ -4012,49 +4012,6 @@ def test_shared_memory_ShareableList_pickling(self):
         deserialized_sl.shm.close()
         sl.shm.close()
 
-    def test_shared_memory_cleaned_after_process_termination(self):
-        cmd = '''if 1:
-            import os, time, sys
-            from multiprocessing import shared_memory
-
-            # Create a shared_memory segment, and send the segment name
-            sm = shared_memory.SharedMemory(create=True, size=10)
-            sys.stdout.write(sm.name + '\\n')
-            sys.stdout.flush()
-            time.sleep(100)
-        '''
-        with subprocess.Popen([sys.executable, '-E', '-c', cmd],
-                              stdout=subprocess.PIPE,
-                              stderr=subprocess.PIPE) as p:
-            name = p.stdout.readline().strip().decode()
-
-            # killing abruptly processes holding reference to a shared memory
-            # segment should not leak the given memory segment.
-            p.terminate()
-            p.wait()
-
-            deadline = time.monotonic() + 60
-            t = 0.1
-            while time.monotonic() < deadline:
-                time.sleep(t)
-                t = min(t*2, 5)
-                try:
-                    smm = shared_memory.SharedMemory(name, create=False)
-                except FileNotFoundError:
-                    break
-            else:
-                raise AssertionError("A SharedMemory segment was leaked after"
-                                     " a process was abruptly terminated.")
-
-            if os.name == 'posix':
-                # A warning was emitted by the subprocess' own
-                # resource_tracker (on Windows, shared memory segments
-                # are released automatically by the OS).
-                err = p.stderr.read().decode()
-                self.assertIn(
-                    "resource_tracker: There appear to be 1 leaked "
-                    "shared_memory objects to clean up at shutdown", err)
-
 #
 #
 #
@@ -5006,9 +4963,6 @@ def create_and_register_resource(rtype):
                 if rtype == "semaphore":
                     lock = mp.Lock()
                     return lock, lock._semlock.name
-                elif rtype == "shared_memory":
-                    sm = SharedMemory(create=True, size=10)
-                    return sm, sm._name
                 else:
                     raise ValueError(
                         "Resource type {{}} not understood".format(rtype))

Misc/NEWS.d/next/Library/2019-09-11-11-08-53.bpo-38119.da_hwK.rst

@@ -0,0 +1 @@
+Fix resource tracker treatment of shared memory as if it were a semaphore.