Skip to content

Commit 3051f0b

Browse files
authored
bpo-30919: shared memory allocation performance regression in multiprocessing (#2708)
* Fix #30919: shared memory allocation performance regression in multiprocessing * Change strategy for Arena directory choice * Add blurb
1 parent 2b1e6e9 commit 3051f0b

File tree

2 files changed

+20
-10
lines changed

2 files changed

+20
-10
lines changed

Lib/multiprocessing/heap.py

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -60,26 +60,32 @@ def __setstate__(self, state):
6060
else:
6161

6262
class Arena(object):
63+
if sys.platform == 'linux':
64+
_dir_candidates = ['/dev/shm']
65+
else:
66+
_dir_candidates = []
6367

6468
def __init__(self, size, fd=-1):
6569
self.size = size
6670
self.fd = fd
6771
if fd == -1:
6872
self.fd, name = tempfile.mkstemp(
69-
prefix='pym-%d-'%os.getpid(), dir=util.get_temp_dir())
73+
prefix='pym-%d-'%os.getpid(),
74+
dir=self._choose_dir(size))
7075
os.unlink(name)
7176
util.Finalize(self, os.close, (self.fd,))
72-
with open(self.fd, 'wb', closefd=False) as f:
73-
bs = 1024 * 1024
74-
if size >= bs:
75-
zeros = b'\0' * bs
76-
for _ in range(size // bs):
77-
f.write(zeros)
78-
del zeros
79-
f.write(b'\0' * (size % bs))
80-
assert f.tell() == size
77+
os.ftruncate(self.fd, size)
8178
self.buffer = mmap.mmap(self.fd, self.size)
8279

80+
def _choose_dir(self, size):
81+
# Choose a non-storage backed directory if possible,
82+
# to improve performance
83+
for d in self._dir_candidates:
84+
st = os.statvfs(d)
85+
if st.f_bavail * st.f_frsize >= size: # enough free space?
86+
return d
87+
return util.get_temp_dir()
88+
8389
def reduce_arena(a):
8490
if a.fd == -1:
8591
raise ValueError('Arena is unpicklable because '
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Fix shared memory performance regression in multiprocessing in 3.x.
2+
3+
Shared memory used anonymous memory mappings in 2.x, while 3.x mmaps actual
4+
files. Try to be careful to do as little disk I/O as possible.

0 commit comments

Comments
 (0)