Skip to content

bpo-36389: Fix _PyBytesWriter in release mode #16624

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 7, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions Include/internal/pycore_pymem.h
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,21 @@ PyAPI_FUNC(int) _PyMem_SetDefaultAllocator(
PyMemAllocatorDomain domain,
PyMemAllocatorEx *old_alloc);

/* Special bytes broadcast into debug memory blocks at appropriate times.
Strings of these are unlikely to be valid addresses, floats, ints or
7-bit ASCII.

- PYMEM_CLEANBYTE: clean (newly allocated) memory
- PYMEM_DEADBYTE dead (newly freed) memory
- PYMEM_FORBIDDENBYTE: untouchable bytes at each end of a block

Byte patterns 0xCB, 0xBB and 0xFB have been replaced with 0xCD, 0xDD and
0xFD to use the same values than Windows CRT debug malloc() and free().
If modified, _PyMem_IsPtrFreed() should be updated as well. */
#define PYMEM_CLEANBYTE 0xCD
#define PYMEM_DEADBYTE 0xDD
#define PYMEM_FORBIDDENBYTE 0xFD

/* Heuristic checking if a pointer value is newly allocated
(uninitialized), newly freed or NULL (is equal to zero).

Expand Down
21 changes: 10 additions & 11 deletions Objects/bytesobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -667,9 +667,6 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len,
Py_ssize_t len = 0;
char onechar; /* For byte_converter() */
Py_ssize_t alloc;
#ifdef Py_DEBUG
char *before;
#endif

fmt++;
if (*fmt == '%') {
Expand Down Expand Up @@ -981,8 +978,8 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len,
if (res == NULL)
goto error;
}
#ifdef Py_DEBUG
before = res;
#ifndef NDEBUG
char *before = res;
#endif

/* Write the sign if needed */
Expand Down Expand Up @@ -1047,7 +1044,7 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len,
}
Py_XDECREF(temp);

#ifdef Py_DEBUG
#ifndef NDEBUG
/* check that we computed the exact size for this write */
assert((res - before) == alloc);
#endif
Expand Down Expand Up @@ -3168,8 +3165,9 @@ _PyBytesWriter_Init(_PyBytesWriter *writer)
{
/* Set all attributes before small_buffer to 0 */
memset(writer, 0, offsetof(_PyBytesWriter, small_buffer));
#ifdef Py_DEBUG
memset(writer->small_buffer, 0xCB, sizeof(writer->small_buffer));
#ifndef NDEBUG
memset(writer->small_buffer, PYMEM_CLEANBYTE,
sizeof(writer->small_buffer));
#endif
}

Expand Down Expand Up @@ -3297,8 +3295,9 @@ _PyBytesWriter_Resize(_PyBytesWriter *writer, void *str, Py_ssize_t size)
}

writer->use_small_buffer = 0;
#ifdef Py_DEBUG
memset(writer->small_buffer, 0xDB, sizeof(writer->small_buffer));
#ifndef NDEBUG
memset(writer->small_buffer, PYMEM_CLEANBYTE,
sizeof(writer->small_buffer));
#endif
}
writer->allocated = allocated;
Expand Down Expand Up @@ -3350,7 +3349,7 @@ _PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size)
assert(size >= 0);

writer->use_small_buffer = 1;
#ifdef Py_DEBUG
#ifndef NDEBUG
writer->allocated = sizeof(writer->small_buffer) - 1;
/* In debug mode, don't use the full small buffer because it is less
efficient than bytes and bytearray objects to detect buffer underflow
Expand Down
59 changes: 23 additions & 36 deletions Objects/obmalloc.c
Original file line number Diff line number Diff line change
Expand Up @@ -2031,20 +2031,6 @@ _Py_GetAllocatedBlocks(void)
* it wraps a real allocator, adding extra debugging info to the memory blocks.
*/

/* Special bytes broadcast into debug memory blocks at appropriate times.
* Strings of these are unlikely to be valid addresses, floats, ints or
* 7-bit ASCII. If modified, _PyMem_IsPtrFreed() should be updated as well.
*
* Byte patterns 0xCB, 0xBB and 0xFB have been replaced with 0xCD, 0xDD and
* 0xFD to use the same values than Windows CRT debug malloc() and free().
*/
#undef CLEANBYTE
#undef DEADBYTE
#undef FORBIDDENBYTE
#define CLEANBYTE 0xCD /* clean (newly allocated) memory */
#define DEADBYTE 0xDD /* dead (newly freed) memory */
#define FORBIDDENBYTE 0xFD /* untouchable bytes at each end of a block */

/* Uncomment this define to add the "serialno" field */
/* #define PYMEM_DEBUG_SERIALNO */

Expand Down Expand Up @@ -2106,14 +2092,14 @@ p[0: S]
p[S]
API ID. See PEP 445. This is a character, but seems undocumented.
p[S+1: 2*S]
Copies of FORBIDDENBYTE. Used to catch under- writes and reads.
Copies of PYMEM_FORBIDDENBYTE. Used to catch under- writes and reads.
p[2*S: 2*S+n]
The requested memory, filled with copies of CLEANBYTE.
The requested memory, filled with copies of PYMEM_CLEANBYTE.
Used to catch reference to uninitialized memory.
&p[2*S] is returned. Note that this is 8-byte aligned if pymalloc
handled the request itself.
p[2*S+n: 2*S+n+S]
Copies of FORBIDDENBYTE. Used to catch over- writes and reads.
Copies of PYMEM_FORBIDDENBYTE. Used to catch over- writes and reads.
p[2*S+n+S: 2*S+n+2*S]
A serial number, incremented by 1 on each call to _PyMem_DebugMalloc
and _PyMem_DebugRealloc.
Expand Down Expand Up @@ -2170,15 +2156,15 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes)
/* at p, write size (SST bytes), id (1 byte), pad (SST-1 bytes) */
write_size_t(p, nbytes);
p[SST] = (uint8_t)api->api_id;
memset(p + SST + 1, FORBIDDENBYTE, SST-1);
memset(p + SST + 1, PYMEM_FORBIDDENBYTE, SST-1);

if (nbytes > 0 && !use_calloc) {
memset(data, CLEANBYTE, nbytes);
memset(data, PYMEM_CLEANBYTE, nbytes);
}

/* at tail, write pad (SST bytes) and serialno (SST bytes) */
tail = data + nbytes;
memset(tail, FORBIDDENBYTE, SST);
memset(tail, PYMEM_FORBIDDENBYTE, SST);
#ifdef PYMEM_DEBUG_SERIALNO
write_size_t(tail + SST, serialno);
#endif
Expand All @@ -2204,7 +2190,7 @@ _PyMem_DebugRawCalloc(void *ctx, size_t nelem, size_t elsize)

/* The debug free first checks the 2*SST bytes on each end for sanity (in
particular, that the FORBIDDENBYTEs with the api ID are still intact).
Then fills the original bytes with DEADBYTE.
Then fills the original bytes with PYMEM_DEADBYTE.
Then calls the underlying free.
*/
static void
Expand All @@ -2222,7 +2208,7 @@ _PyMem_DebugRawFree(void *ctx, void *p)
_PyMem_DebugCheckAddress(api->api_id, p);
nbytes = read_size_t(q);
nbytes += PYMEM_DEBUG_EXTRA_BYTES;
memset(q, DEADBYTE, nbytes);
memset(q, PYMEM_DEADBYTE, nbytes);
api->alloc.free(api->alloc.ctx, q);
}

Expand Down Expand Up @@ -2264,14 +2250,14 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes)
*/
if (original_nbytes <= sizeof(save)) {
memcpy(save, data, original_nbytes);
memset(data - 2 * SST, DEADBYTE,
memset(data - 2 * SST, PYMEM_DEADBYTE,
original_nbytes + PYMEM_DEBUG_EXTRA_BYTES);
}
else {
memcpy(save, data, ERASED_SIZE);
memset(head, DEADBYTE, ERASED_SIZE + 2 * SST);
memset(head, PYMEM_DEADBYTE, ERASED_SIZE + 2 * SST);
memcpy(&save[ERASED_SIZE], tail - ERASED_SIZE, ERASED_SIZE);
memset(tail - ERASED_SIZE, DEADBYTE,
memset(tail - ERASED_SIZE, PYMEM_DEADBYTE,
ERASED_SIZE + PYMEM_DEBUG_EXTRA_BYTES - 2 * SST);
}

Expand All @@ -2293,10 +2279,10 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes)

write_size_t(head, nbytes);
head[SST] = (uint8_t)api->api_id;
memset(head + SST + 1, FORBIDDENBYTE, SST-1);
memset(head + SST + 1, PYMEM_FORBIDDENBYTE, SST-1);

tail = data + nbytes;
memset(tail, FORBIDDENBYTE, SST);
memset(tail, PYMEM_FORBIDDENBYTE, SST);
#ifdef PYMEM_DEBUG_SERIALNO
write_size_t(tail + SST, block_serialno);
#endif
Expand All @@ -2320,7 +2306,8 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes)

if (nbytes > original_nbytes) {
/* growing: mark new extra memory clean */
memset(data + original_nbytes, CLEANBYTE, nbytes - original_nbytes);
memset(data + original_nbytes, PYMEM_CLEANBYTE,
nbytes - original_nbytes);
}

return data;
Expand Down Expand Up @@ -2399,7 +2386,7 @@ _PyMem_DebugCheckAddress(char api, const void *p)
* the tail could lead to a segfault then.
*/
for (i = SST-1; i >= 1; --i) {
if (*(q-i) != FORBIDDENBYTE) {
if (*(q-i) != PYMEM_FORBIDDENBYTE) {
msg = "bad leading pad byte";
goto error;
}
Expand All @@ -2408,7 +2395,7 @@ _PyMem_DebugCheckAddress(char api, const void *p)
nbytes = read_size_t(q - 2*SST);
tail = q + nbytes;
for (i = 0; i < SST; ++i) {
if (tail[i] != FORBIDDENBYTE) {
if (tail[i] != PYMEM_FORBIDDENBYTE) {
msg = "bad trailing pad byte";
goto error;
}
Expand Down Expand Up @@ -2448,7 +2435,7 @@ _PyObject_DebugDumpAddress(const void *p)
fprintf(stderr, " The %d pad bytes at p-%d are ", SST-1, SST-1);
ok = 1;
for (i = 1; i <= SST-1; ++i) {
if (*(q-i) != FORBIDDENBYTE) {
if (*(q-i) != PYMEM_FORBIDDENBYTE) {
ok = 0;
break;
}
Expand All @@ -2457,11 +2444,11 @@ _PyObject_DebugDumpAddress(const void *p)
fputs("FORBIDDENBYTE, as expected.\n", stderr);
else {
fprintf(stderr, "not all FORBIDDENBYTE (0x%02x):\n",
FORBIDDENBYTE);
PYMEM_FORBIDDENBYTE);
for (i = SST-1; i >= 1; --i) {
const uint8_t byte = *(q-i);
fprintf(stderr, " at p-%d: 0x%02x", i, byte);
if (byte != FORBIDDENBYTE)
if (byte != PYMEM_FORBIDDENBYTE)
fputs(" *** OUCH", stderr);
fputc('\n', stderr);
}
Expand All @@ -2476,7 +2463,7 @@ _PyObject_DebugDumpAddress(const void *p)
fprintf(stderr, " The %d pad bytes at tail=%p are ", SST, (void *)tail);
ok = 1;
for (i = 0; i < SST; ++i) {
if (tail[i] != FORBIDDENBYTE) {
if (tail[i] != PYMEM_FORBIDDENBYTE) {
ok = 0;
break;
}
Expand All @@ -2485,12 +2472,12 @@ _PyObject_DebugDumpAddress(const void *p)
fputs("FORBIDDENBYTE, as expected.\n", stderr);
else {
fprintf(stderr, "not all FORBIDDENBYTE (0x%02x):\n",
FORBIDDENBYTE);
PYMEM_FORBIDDENBYTE);
for (i = 0; i < SST; ++i) {
const uint8_t byte = tail[i];
fprintf(stderr, " at tail+%d: 0x%02x",
i, byte);
if (byte != FORBIDDENBYTE)
if (byte != PYMEM_FORBIDDENBYTE)
fputs(" *** OUCH", stderr);
fputc('\n', stderr);
}
Expand Down