bpo-29861: release references to multiprocessing Pool tasks (python#743)

* bpo-29861: release references to multiprocessing Pool tasks

Release references to tasks, their arguments and their results as soon
as they are finished, instead of keeping them alive until another task
arrives.

* Comments in test

(cherry picked from commit 8988945)

Author: pitrou

Lib/multiprocessing/pool.py

@@ -128,6 +128,8 @@ def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None,
             util.debug("Possible encoding error while sending result: %s" % (
                 wrapped))
             put((job, i, (False, wrapped)))
+
+        task = job = result = func = args = kwds = None
         completed += 1
     util.debug('worker exiting after %d tasks' % completed)
 
@@ -402,10 +404,11 @@ def _handle_tasks(taskqueue, put, outqueue, pool, cache):
                 if set_length:
                     util.debug('doing set_length()')
                     set_length(i+1)
+            finally:
+                task = taskseq = job = None
         else:
             util.debug('task handler got sentinel')
 
-
         try:
             # tell result handler to finish when cache is empty
             util.debug('task handler sending sentinel to result handler')
@@ -445,6 +448,7 @@ def _handle_results(outqueue, get, cache):
                 cache[job]._set(i, obj)
             except KeyError:
                 pass
+            task = job = obj = None
 
         while cache and thread._state != TERMINATE:
             try:
@@ -461,6 +465,7 @@ def _handle_results(outqueue, get, cache):
                 cache[job]._set(i, obj)
             except KeyError:
                 pass
+            task = job = obj = None
 
         if hasattr(outqueue, '_reader'):
             util.debug('ensuring that outqueue is not full')

Lib/test/_test_multiprocessing.py

@@ -18,6 +18,7 @@
 import logging
 import struct
 import operator
+import weakref
 import test.support
 import test.support.script_helper
 
@@ -1668,6 +1669,19 @@ def sqr(x, wait=0.0):
 def mul(x, y):
     return x*y
 
+def identity(x):
+    return x
+
+class CountedObject(object):
+    n_instances = 0
+
+    def __new__(cls):
+        cls.n_instances += 1
+        return object.__new__(cls)
+
+    def __del__(self):
+        type(self).n_instances -= 1
+
 class SayWhenError(ValueError): pass
 
 def exception_throwing_generator(total, when):
@@ -1676,6 +1690,7 @@ def exception_throwing_generator(total, when):
             raise SayWhenError("Somebody said when")
         yield i
 
+
 class _TestPool(BaseTestCase):
 
     @classmethod
@@ -1910,6 +1925,19 @@ def test_wrapped_exception(self):
             with self.assertRaises(RuntimeError):
                 p.apply(self._test_wrapped_exception)
 
+    def test_release_task_refs(self):
+        # Issue #29861: task arguments and results should not be kept
+        # alive after we are done with them.
+        objs = [CountedObject() for i in range(10)]
+        refs = [weakref.ref(o) for o in objs]
+        self.pool.map(identity, objs)
+
+        del objs
+        self.assertEqual(set(wr() for wr in refs), {None})
+        # With a process pool, copies of the objects are returned, check
+        # they were released too.
+        self.assertEqual(CountedObject.n_instances, 0)
+
 
 def raising():
     raise KeyError("key")

Misc/NEWS

@@ -10,24 +10,6 @@ Release date: XXXX-XX-XX
 Core and Builtins
 -----------------
 
-- bpo-28876: ``bool(range)`` works even if ``len(range)``
-  raises :exc:`OverflowError`.
-
-- bpo-29600: Fix wrapping coroutine return values in StopIteration.
-
-- Issue #29537: Restore runtime compatibility with bytecode files generated by
-  CPython 3.5.0 to 3.5.2, and adjust the eval loop to avoid the problems that
-  could be caused by the malformed variant of the BUILD_MAP_UNPACK_WITH_CALL
-  opcode that they may contain. Patch by Petr Viktorin, Serhiy Storchaka,
-  and Nick Coghlan.
-
-- Issue #28598: Support __rmod__ for subclasses of str being called before
-  str.__mod__.  Patch by Martijn Pieters.
-
-- bpo-29602: Fix incorrect handling of signed zeros in complex constructor for
-  complex subclasses and for inputs having a __complex__ method. Patch
-  by Serhiy Storchaka.
-
 - bpo-29347: Fixed possibly dereferencing undefined pointers
   when creating weakref objects.
 
@@ -46,43 +28,8 @@ Extension Modules
 Library
 -------
 
-- bpo-29884: faulthandler: Restore the old sigaltstack during teardown.
-  Patch by Christophe Zeitouny.
-
-- bpo-25455: Fixed crashes in repr of recursive buffered file-like objects.
-
-- bpo-29800: Fix crashes in partial.__repr__ if the keys of partial.keywords
-  are not strings.  Patch by Michael Seifert.
-
-- bpo-29742: get_extra_info() raises exception if get called on closed ssl transport.
-  Patch by Nikolay Kim.
-
-- bpo-8256: Fixed possible failing or crashing input() if attributes "encoding"
-  or "errors" of sys.stdin or sys.stdout are not set or are not strings.
-
-- bpo-28298: Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big
-  intables (objects that have __int__) as elements.  Patch by Oren Milman.
-
-- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other
-  exception) to exception(s) raised in the dispatched methods.
-  Patch by Petr Motejlek.
-
-- bpo-29704: asyncio.subprocess.SubprocessStreamProtocol no longer closes before
-  all pipes are closed.
-
-- bpo-29703: Fix asyncio to support instantiation of new event loops
-  in child processes.
-
-- bpo-29376: Fix assertion error in threading._DummyThread.is_alive().
-
-- bpo-29110: Fix file object leak in aifc.open() when file is given as a
-  filesystem path and is not in valid AIFF format. Patch by Anthony Zhang.
-
-- Issue #28961: Fix unittest.mock._Call helper: don't ignore the name parameter
-  anymore. Patch written by Jiajun Huang.
-
-- bpo-29532: Altering a kwarg dictionary passed to functools.partial()
-  no longer affects a partial object after creation.
+- bpo-29861: Release references to tasks, their arguments and their results
+  as soon as they are finished in multiprocessing.Pool.
 
 - Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap,
   improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi,
@@ -141,11 +88,6 @@ C API
 Documentation
 -------------
 
-- bpo-28929: Link the documentation to its source file on GitHub.
-
-- bpo-25008: Document smtpd.py as effectively deprecated and add a pointer to
-  aiosmtpd, a third-party asyncio-based replacement.
-
 - Issue #26355: Add canonical header link on each page to corresponding major
   version of the documentation. Patch by Matthias Bussonnier.