Skip to content

Commit 9e23f0a

Browse files
authored
[3.7] bpo-36389: _PyObject_IsFreed() now also detects uninitialized memory (GH-12770) (GH-12788)
* bpo-36389: _PyObject_IsFreed() now also detects uninitialized memory (GH-12770) Replace _PyMem_IsFreed() function with _PyMem_IsPtrFreed() inline function. The function is now way more efficient, it became a simple comparison on integers, rather than a short loop. It detects also uninitialized bytes and "forbidden bytes" filled by debug hooks on memory allocators. Add unit tests on _PyObject_IsFreed(). (cherry picked from commit 2b00db6) * bpo-36389: Change PyMem_SetupDebugHooks() constants (GH-12782) Modify CLEANBYTE, DEADDYTE and FORBIDDENBYTE constants: use 0xCD, 0xDD and 0xFD, rather than 0xCB, 0xBB and 0xFB, to use the same byte patterns than Windows CRT debug malloc() and free(). (cherry picked from commit 4c409be)
1 parent ac31da8 commit 9e23f0a

File tree

8 files changed

+140
-41
lines changed

8 files changed

+140
-41
lines changed

Doc/c-api/memory.rst

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -440,8 +440,9 @@ Customize Memory Allocators
440440
441441
Setup hooks to detect bugs in the Python memory allocator functions.
442442
443-
Newly allocated memory is filled with the byte ``0xCB``, freed memory is
444-
filled with the byte ``0xDB``.
443+
Newly allocated memory is filled with the byte ``0xCD`` (``CLEANBYTE``),
444+
freed memory is filled with the byte ``0xDD`` (``DEADBYTE``). Memory blocks
445+
are surrounded by "forbidden bytes" (``FORBIDDENBYTE``: byte ``0xFD``).
445446
446447
Runtime checks:
447448
@@ -471,6 +472,12 @@ Customize Memory Allocators
471472
if the GIL is held when functions of :c:data:`PYMEM_DOMAIN_OBJ` and
472473
:c:data:`PYMEM_DOMAIN_MEM` domains are called.
473474
475+
.. versionchanged:: 3.7.3
476+
Byte patterns ``0xCB`` (``CLEANBYTE``), ``0xDB`` (``DEADBYTE``) and
477+
``0xFB`` (``FORBIDDENBYTE``) have been replaced with ``0xCD``, ``0xDD``
478+
and ``0xFD`` to use the same values than Windows CRT debug ``malloc()``
479+
and ``free()``.
480+
474481
475482
.. _pymalloc:
476483

Include/internal/mem.h

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,30 @@ PyAPI_FUNC(void) _PyGC_Initialize(struct _gc_runtime_state *);
145145

146146
#define _PyGC_generation0 _PyRuntime.gc.generation0
147147

148+
/* Heuristic checking if a pointer value is newly allocated
149+
(uninitialized) or newly freed. The pointer is not dereferenced, only the
150+
pointer value is checked.
151+
152+
The heuristic relies on the debug hooks on Python memory allocators which
153+
fills newly allocated memory with CLEANBYTE (0xCD) and newly freed memory
154+
with DEADBYTE (0xDD). Detect also "untouchable bytes" marked
155+
with FORBIDDENBYTE (0xFD). */
156+
static inline int _PyMem_IsPtrFreed(void *ptr)
157+
{
158+
uintptr_t value = (uintptr_t)ptr;
159+
#if SIZEOF_VOID_P == 8
160+
return (value == (uintptr_t)0xCDCDCDCDCDCDCDCD
161+
|| value == (uintptr_t)0xDDDDDDDDDDDDDDDD
162+
|| value == (uintptr_t)0xFDFDFDFDFDFDFDFD);
163+
#elif SIZEOF_VOID_P == 4
164+
return (value == (uintptr_t)0xCDCDCDCD
165+
|| value == (uintptr_t)0xDDDDDDDD
166+
|| value == (uintptr_t)0xFDFDFDFD);
167+
#else
168+
# error "unknown pointer size"
169+
#endif
170+
}
171+
148172
#ifdef __cplusplus
149173
}
150174
#endif

Include/pymem.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,6 @@ PyAPI_FUNC(int) PyTraceMalloc_Untrack(
5555
PyAPI_FUNC(PyObject*) _PyTraceMalloc_GetTraceback(
5656
unsigned int domain,
5757
uintptr_t ptr);
58-
59-
PyAPI_FUNC(int) _PyMem_IsFreed(void *ptr, size_t size);
6058
#endif /* !defined(Py_LIMITED_API) */
6159

6260

Lib/test/test_capi.py

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -462,11 +462,11 @@ def test_buffer_overflow(self):
462462
r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n"
463463
r" The [0-9] pad bytes at tail={ptr} are not all FORBIDDENBYTE \(0x[0-9a-f]{{2}}\):\n"
464464
r" at tail\+0: 0x78 \*\*\* OUCH\n"
465-
r" at tail\+1: 0xfb\n"
466-
r" at tail\+2: 0xfb\n"
465+
r" at tail\+1: 0xfd\n"
466+
r" at tail\+2: 0xfd\n"
467467
r" .*\n"
468468
r" The block was made by call #[0-9]+ to debug malloc/realloc.\n"
469-
r" Data at p: cb cb cb .*\n"
469+
r" Data at p: cd cd cd .*\n"
470470
r"\n"
471471
r"Enable tracemalloc to get the memory block allocation traceback\n"
472472
r"\n"
@@ -482,7 +482,7 @@ def test_api_misuse(self):
482482
r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n"
483483
r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n"
484484
r" The block was made by call #[0-9]+ to debug malloc/realloc.\n"
485-
r" Data at p: cb cb cb .*\n"
485+
r" Data at p: cd cd cd .*\n"
486486
r"\n"
487487
r"Enable tracemalloc to get the memory block allocation traceback\n"
488488
r"\n"
@@ -508,6 +508,29 @@ def test_pyobject_malloc_without_gil(self):
508508
code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()'
509509
self.check_malloc_without_gil(code)
510510

511+
def check_pyobject_is_freed(self, func):
512+
code = textwrap.dedent('''
513+
import gc, os, sys, _testcapi
514+
# Disable the GC to avoid crash on GC collection
515+
gc.disable()
516+
obj = _testcapi.{func}()
517+
error = (_testcapi.pyobject_is_freed(obj) == False)
518+
# Exit immediately to avoid a crash while deallocating
519+
# the invalid object
520+
os._exit(int(error))
521+
''')
522+
code = code.format(func=func)
523+
assert_python_ok('-c', code, PYTHONMALLOC=self.PYTHONMALLOC)
524+
525+
def test_pyobject_is_freed_uninitialized(self):
526+
self.check_pyobject_is_freed('pyobject_uninitialized')
527+
528+
def test_pyobject_is_freed_forbidden_bytes(self):
529+
self.check_pyobject_is_freed('pyobject_forbidden_bytes')
530+
531+
def test_pyobject_is_freed_free(self):
532+
self.check_pyobject_is_freed('pyobject_freed')
533+
511534

512535
class PyMemMallocDebugTests(PyMemDebugTests):
513536
PYTHONMALLOC = 'malloc_debug'
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Change the value of ``CLEANBYTE``, ``DEADDYTE`` and ``FORBIDDENBYTE`` internal
2+
constants used by debug hooks on Python memory allocators
3+
(:c:func:`PyMem_SetupDebugHooks` function). Byte patterns ``0xCB``, ``0xDB``
4+
and ``0xFB`` have been replaced with ``0xCD``, ``0xDD`` and ``0xFD`` to use the
5+
same values than Windows CRT debug ``malloc()`` and ``free()``.

Modules/_testcapimodule.c

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4227,6 +4227,59 @@ test_pymem_getallocatorsname(PyObject *self, PyObject *args)
42274227
}
42284228

42294229

4230+
static PyObject*
4231+
pyobject_is_freed(PyObject *self, PyObject *op)
4232+
{
4233+
int res = _PyObject_IsFreed(op);
4234+
return PyBool_FromLong(res);
4235+
}
4236+
4237+
4238+
static PyObject*
4239+
pyobject_uninitialized(PyObject *self, PyObject *args)
4240+
{
4241+
PyObject *op = (PyObject *)PyObject_Malloc(sizeof(PyObject));
4242+
if (op == NULL) {
4243+
return NULL;
4244+
}
4245+
/* Initialize reference count to avoid early crash in ceval or GC */
4246+
Py_REFCNT(op) = 1;
4247+
/* object fields like ob_type are uninitialized! */
4248+
return op;
4249+
}
4250+
4251+
4252+
static PyObject*
4253+
pyobject_forbidden_bytes(PyObject *self, PyObject *args)
4254+
{
4255+
/* Allocate an incomplete PyObject structure: truncate 'ob_type' field */
4256+
PyObject *op = (PyObject *)PyObject_Malloc(offsetof(PyObject, ob_type));
4257+
if (op == NULL) {
4258+
return NULL;
4259+
}
4260+
/* Initialize reference count to avoid early crash in ceval or GC */
4261+
Py_REFCNT(op) = 1;
4262+
/* ob_type field is after the memory block: part of "forbidden bytes"
4263+
when using debug hooks on memory allocatrs! */
4264+
return op;
4265+
}
4266+
4267+
4268+
static PyObject*
4269+
pyobject_freed(PyObject *self, PyObject *args)
4270+
{
4271+
PyObject *op = _PyObject_CallNoArg((PyObject *)&PyBaseObject_Type);
4272+
if (op == NULL) {
4273+
return NULL;
4274+
}
4275+
Py_TYPE(op)->tp_dealloc(op);
4276+
/* Reset reference count to avoid early crash in ceval or GC */
4277+
Py_REFCNT(op) = 1;
4278+
/* object memory is freed! */
4279+
return op;
4280+
}
4281+
4282+
42304283
static PyObject*
42314284
pyobject_malloc_without_gil(PyObject *self, PyObject *args)
42324285
{
@@ -4788,6 +4841,10 @@ static PyMethodDef TestMethods[] = {
47884841
{"pymem_api_misuse", pymem_api_misuse, METH_NOARGS},
47894842
{"pymem_malloc_without_gil", pymem_malloc_without_gil, METH_NOARGS},
47904843
{"pymem_getallocatorsname", test_pymem_getallocatorsname, METH_NOARGS},
4844+
{"pyobject_is_freed", (PyCFunction)(void(*)(void))pyobject_is_freed, METH_O},
4845+
{"pyobject_uninitialized", pyobject_uninitialized, METH_NOARGS},
4846+
{"pyobject_forbidden_bytes", pyobject_forbidden_bytes, METH_NOARGS},
4847+
{"pyobject_freed", pyobject_freed, METH_NOARGS},
47914848
{"pyobject_malloc_without_gil", pyobject_malloc_without_gil, METH_NOARGS},
47924849
{"tracemalloc_track", tracemalloc_track, METH_VARARGS},
47934850
{"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS},

Objects/object.c

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -411,28 +411,26 @@ _Py_BreakPoint(void)
411411
}
412412

413413

414-
/* Heuristic checking if the object memory has been deallocated.
415-
Rely on the debug hooks on Python memory allocators which fills the memory
416-
with DEADBYTE (0xDB) when memory is deallocated.
414+
/* Heuristic checking if the object memory is uninitialized or deallocated.
415+
Rely on the debug hooks on Python memory allocators:
416+
see _PyMem_IsPtrFreed().
417417
418418
The function can be used to prevent segmentation fault on dereferencing
419-
pointers like 0xdbdbdbdbdbdbdbdb. Such pointer is very unlikely to be mapped
420-
in memory. */
419+
pointers like 0xDDDDDDDDDDDDDDDD. */
421420
int
422421
_PyObject_IsFreed(PyObject *op)
423422
{
424-
uintptr_t ptr = (uintptr_t)op;
425-
if (_PyMem_IsFreed(&ptr, sizeof(ptr))) {
423+
if (_PyMem_IsPtrFreed(op) || _PyMem_IsPtrFreed(op->ob_type)) {
426424
return 1;
427425
}
428-
int freed = _PyMem_IsFreed(&op->ob_type, sizeof(op->ob_type));
429-
/* ignore op->ob_ref: the value can have be modified
426+
/* ignore op->ob_ref: its value can have be modified
430427
by Py_INCREF() and Py_DECREF(). */
431428
#ifdef Py_TRACE_REFS
432-
freed &= _PyMem_IsFreed(&op->_ob_next, sizeof(op->_ob_next));
433-
freed &= _PyMem_IsFreed(&op->_ob_prev, sizeof(op->_ob_prev));
429+
if (_PyMem_IsPtrFreed(op->_ob_next) || _PyMem_IsPtrFreed(op->_ob_prev)) {
430+
return 1;
431+
}
434432
#endif
435-
return freed;
433+
return 0;
436434
}
437435

438436

@@ -449,7 +447,7 @@ _PyObject_Dump(PyObject* op)
449447
if (_PyObject_IsFreed(op)) {
450448
/* It seems like the object memory has been freed:
451449
don't access it to prevent a segmentation fault. */
452-
fprintf(stderr, "<freed object>\n");
450+
fprintf(stderr, "<Freed object>\n");
453451
return;
454452
}
455453

Objects/obmalloc.c

Lines changed: 7 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1946,14 +1946,17 @@ _Py_GetAllocatedBlocks(void)
19461946

19471947
/* Special bytes broadcast into debug memory blocks at appropriate times.
19481948
* Strings of these are unlikely to be valid addresses, floats, ints or
1949-
* 7-bit ASCII.
1949+
* 7-bit ASCII. If modified, _PyMem_IsPtrFreed() should be updated as well.
1950+
*
1951+
* Byte patterns 0xCB, 0xBB and 0xFB have been replaced with 0xCD, 0xDD and
1952+
* 0xFD to use the same values than Windows CRT debug malloc() and free().
19501953
*/
19511954
#undef CLEANBYTE
19521955
#undef DEADBYTE
19531956
#undef FORBIDDENBYTE
1954-
#define CLEANBYTE 0xCB /* clean (newly allocated) memory */
1955-
#define DEADBYTE 0xDB /* dead (newly freed) memory */
1956-
#define FORBIDDENBYTE 0xFB /* untouchable bytes at each end of a block */
1957+
#define CLEANBYTE 0xCD /* clean (newly allocated) memory */
1958+
#define DEADBYTE 0xDD /* dead (newly freed) memory */
1959+
#define FORBIDDENBYTE 0xFD /* untouchable bytes at each end of a block */
19571960

19581961
static size_t serialno = 0; /* incremented on each debug {m,re}alloc */
19591962

@@ -2091,22 +2094,6 @@ _PyMem_DebugRawCalloc(void *ctx, size_t nelem, size_t elsize)
20912094
}
20922095

20932096

2094-
/* Heuristic checking if the memory has been freed. Rely on the debug hooks on
2095-
Python memory allocators which fills the memory with DEADBYTE (0xDB) when
2096-
memory is deallocated. */
2097-
int
2098-
_PyMem_IsFreed(void *ptr, size_t size)
2099-
{
2100-
unsigned char *bytes = ptr;
2101-
for (size_t i=0; i < size; i++) {
2102-
if (bytes[i] != DEADBYTE) {
2103-
return 0;
2104-
}
2105-
}
2106-
return 1;
2107-
}
2108-
2109-
21102097
/* The debug free first checks the 2*SST bytes on each end for sanity (in
21112098
particular, that the FORBIDDENBYTEs with the api ID are still intact).
21122099
Then fills the original bytes with DEADBYTE.

0 commit comments

Comments
 (0)