Skip to content

Commit 887ff8e

Browse files
authored
[3.8] bpo-40330: Fix utf-8 size check in ShareableList (GH-19606) (GH-19625)
The item size must be checked after encoding to bytes, not before. Automerge-Triggered-By: @pitrou. (cherry picked from commit eba9f61) Co-authored-by: Antoine Pitrou <[email protected]>
1 parent 81de3c2 commit 887ff8e

File tree

3 files changed

+22
-6
lines changed

3 files changed

+22
-6
lines changed

Lib/multiprocessing/shared_memory.py

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -433,9 +433,12 @@ def __setitem__(self, position, value):
433433

434434
if not isinstance(value, (str, bytes)):
435435
new_format = self._types_mapping[type(value)]
436+
encoded_value = value
436437
else:
437-
if len(value) > self._allocated_bytes[position]:
438-
raise ValueError("exceeds available storage for existing str")
438+
encoded_value = (value.encode(_encoding)
439+
if isinstance(value, str) else value)
440+
if len(encoded_value) > self._allocated_bytes[position]:
441+
raise ValueError("bytes/str item exceeds available storage")
439442
if current_format[-1] == "s":
440443
new_format = current_format
441444
else:
@@ -448,8 +451,7 @@ def __setitem__(self, position, value):
448451
new_format,
449452
value
450453
)
451-
value = value.encode(_encoding) if isinstance(value, str) else value
452-
struct.pack_into(new_format, self.shm.buf, offset, value)
454+
struct.pack_into(new_format, self.shm.buf, offset, encoded_value)
453455

454456
def __reduce__(self):
455457
return partial(self.__class__, name=self.shm.name), ()

Lib/test/_test_multiprocessing.py

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3964,9 +3964,21 @@ def test_shared_memory_ShareableList_basics(self):
39643964
sl[4] = 'some' # Change type at a given position.
39653965
self.assertEqual(sl[4], 'some')
39663966
self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q')
3967-
with self.assertRaises(ValueError):
3968-
sl[4] = 'far too many' # Exceeds available storage.
3967+
with self.assertRaisesRegex(ValueError,
3968+
"exceeds available storage"):
3969+
sl[4] = 'far too many'
39693970
self.assertEqual(sl[4], 'some')
3971+
sl[0] = 'encodés' # Exactly 8 bytes of UTF-8 data
3972+
self.assertEqual(sl[0], 'encodés')
3973+
self.assertEqual(sl[1], b'HoWdY') # no spillage
3974+
with self.assertRaisesRegex(ValueError,
3975+
"exceeds available storage"):
3976+
sl[0] = 'encodées' # Exactly 9 bytes of UTF-8 data
3977+
self.assertEqual(sl[1], b'HoWdY')
3978+
with self.assertRaisesRegex(ValueError,
3979+
"exceeds available storage"):
3980+
sl[1] = b'123456789'
3981+
self.assertEqual(sl[1], b'HoWdY')
39703982

39713983
# Exercise count().
39723984
with warnings.catch_warnings():
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
In :meth:`ShareableList.__setitem__`, check the size of a new string item
2+
after encoding it to utf-8, not before.

0 commit comments

Comments
 (0)