Skip to content

Commit 3c137dc

Browse files
GH-91054: Add code object watchers API (GH-99859)
* Add API to allow extensions to set callback function on creation and destruction of PyCodeObject Co-authored-by: Ye11ow-Flash <[email protected]>
1 parent 0563be2 commit 3c137dc

File tree

11 files changed

+364
-0
lines changed

11 files changed

+364
-0
lines changed

Doc/c-api/code.rst

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,3 +115,51 @@ bound into a function.
115115
the free variables. On error, ``NULL`` is returned and an exception is raised.
116116
117117
.. versionadded:: 3.11
118+
119+
.. c:function:: int PyCode_AddWatcher(PyCode_WatchCallback callback)
120+
121+
Register *callback* as a code object watcher for the current interpreter.
122+
Return an ID which may be passed to :c:func:`PyCode_ClearWatcher`.
123+
In case of error (e.g. no more watcher IDs available),
124+
return ``-1`` and set an exception.
125+
126+
.. versionadded:: 3.12
127+
128+
.. c:function:: int PyCode_ClearWatcher(int watcher_id)
129+
130+
Clear watcher identified by *watcher_id* previously returned from
131+
:c:func:`PyCode_AddWatcher` for the current interpreter.
132+
Return ``0`` on success, or ``-1`` and set an exception on error
133+
(e.g. if the given *watcher_id* was never registered.)
134+
135+
.. versionadded:: 3.12
136+
137+
.. c:type:: PyCodeEvent
138+
139+
Enumeration of possible code object watcher events:
140+
- ``PY_CODE_EVENT_CREATE``
141+
- ``PY_CODE_EVENT_DESTROY``
142+
143+
.. versionadded:: 3.12
144+
145+
.. c:type:: int (*PyCode_WatchCallback)(PyCodeEvent event, PyCodeObject* co)
146+
147+
Type of a code object watcher callback function.
148+
149+
If *event* is ``PY_CODE_EVENT_CREATE``, then the callback is invoked
150+
after `co` has been fully initialized. Otherwise, the callback is invoked
151+
before the destruction of *co* takes place, so the prior state of *co*
152+
can be inspected.
153+
154+
Users of this API should not rely on internal runtime implementation
155+
details. Such details may include, but are not limited to, the exact
156+
order and timing of creation and destruction of code objects. While
157+
changes in these details may result in differences observable by watchers
158+
(including whether a callback is invoked or not), it does not change
159+
the semantics of the Python code being executed.
160+
161+
If the callback returns with an exception set, it must return ``-1``; this
162+
exception will be printed as an unraisable exception using
163+
:c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``.
164+
165+
.. versionadded:: 3.12

Doc/whatsnew/3.12.rst

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -773,6 +773,10 @@ New Features
773773
callbacks to receive notification on changes to a type.
774774
(Contributed by Carl Meyer in :gh:`91051`.)
775775

776+
* Added :c:func:`PyCode_AddWatcher` and :c:func:`PyCode_ClearWatcher`
777+
APIs to register callbacks to receive notification on creation and
778+
destruction of code objects.
779+
(Contributed by Itamar Ostricher in :gh:`91054`.)
776780

777781
* Add :c:func:`PyFrame_GetVar` and :c:func:`PyFrame_GetVarString` functions to
778782
get a frame variable by its name.

Include/cpython/code.h

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,41 @@ PyAPI_FUNC(int) PyCode_Addr2Line(PyCodeObject *, int);
181181

182182
PyAPI_FUNC(int) PyCode_Addr2Location(PyCodeObject *, int, int *, int *, int *, int *);
183183

184+
typedef enum PyCodeEvent {
185+
PY_CODE_EVENT_CREATE,
186+
PY_CODE_EVENT_DESTROY
187+
} PyCodeEvent;
188+
189+
190+
/*
191+
* A callback that is invoked for different events in a code object's lifecycle.
192+
*
193+
* The callback is invoked with a borrowed reference to co, after it is
194+
* created and before it is destroyed.
195+
*
196+
* If the callback returns with an exception set, it must return -1. Otherwise
197+
* it should return 0.
198+
*/
199+
typedef int (*PyCode_WatchCallback)(
200+
PyCodeEvent event,
201+
PyCodeObject* co);
202+
203+
/*
204+
* Register a per-interpreter callback that will be invoked for code object
205+
* lifecycle events.
206+
*
207+
* Returns a handle that may be passed to PyCode_ClearWatcher on success,
208+
* or -1 and sets an error if no more handles are available.
209+
*/
210+
PyAPI_FUNC(int) PyCode_AddWatcher(PyCode_WatchCallback callback);
211+
212+
/*
213+
* Clear the watcher associated with the watcher_id handle.
214+
*
215+
* Returns 0 on success or -1 if no watcher exists for the provided id.
216+
*/
217+
PyAPI_FUNC(int) PyCode_ClearWatcher(int watcher_id);
218+
184219
/* for internal use only */
185220
struct _opaque {
186221
int computed_line;

Include/internal/pycore_code.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
extern "C" {
55
#endif
66

7+
#define CODE_MAX_WATCHERS 8
8+
79
/* PEP 659
810
* Specialization and quickening structs and helper functions
911
*/

Include/internal/pycore_interp.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,9 @@ struct _is {
191191

192192
PyObject *audit_hooks;
193193
PyType_WatchCallback type_watchers[TYPE_MAX_WATCHERS];
194+
PyCode_WatchCallback code_watchers[CODE_MAX_WATCHERS];
195+
// One bit is set for each non-NULL entry in code_watchers
196+
uint8_t active_code_watchers;
194197

195198
struct _Py_unicode_state unicode;
196199
struct _Py_float_state float_state;

Lib/test/test_capi/test_watchers.py

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -336,6 +336,74 @@ def test_no_more_ids_available(self):
336336
self.add_watcher()
337337

338338

339+
class TestCodeObjectWatchers(unittest.TestCase):
340+
@contextmanager
341+
def code_watcher(self, which_watcher):
342+
wid = _testcapi.add_code_watcher(which_watcher)
343+
try:
344+
yield wid
345+
finally:
346+
_testcapi.clear_code_watcher(wid)
347+
348+
def assert_event_counts(self, exp_created_0, exp_destroyed_0,
349+
exp_created_1, exp_destroyed_1):
350+
self.assertEqual(
351+
exp_created_0, _testcapi.get_code_watcher_num_created_events(0))
352+
self.assertEqual(
353+
exp_destroyed_0, _testcapi.get_code_watcher_num_destroyed_events(0))
354+
self.assertEqual(
355+
exp_created_1, _testcapi.get_code_watcher_num_created_events(1))
356+
self.assertEqual(
357+
exp_destroyed_1, _testcapi.get_code_watcher_num_destroyed_events(1))
358+
359+
def test_code_object_events_dispatched(self):
360+
# verify that all counts are zero before any watchers are registered
361+
self.assert_event_counts(0, 0, 0, 0)
362+
363+
# verify that all counts remain zero when a code object is
364+
# created and destroyed with no watchers registered
365+
co1 = _testcapi.code_newempty("test_watchers", "dummy1", 0)
366+
self.assert_event_counts(0, 0, 0, 0)
367+
del co1
368+
self.assert_event_counts(0, 0, 0, 0)
369+
370+
# verify counts are as expected when first watcher is registered
371+
with self.code_watcher(0):
372+
self.assert_event_counts(0, 0, 0, 0)
373+
co2 = _testcapi.code_newempty("test_watchers", "dummy2", 0)
374+
self.assert_event_counts(1, 0, 0, 0)
375+
del co2
376+
self.assert_event_counts(1, 1, 0, 0)
377+
378+
# again with second watcher registered
379+
with self.code_watcher(1):
380+
self.assert_event_counts(1, 1, 0, 0)
381+
co3 = _testcapi.code_newempty("test_watchers", "dummy3", 0)
382+
self.assert_event_counts(2, 1, 1, 0)
383+
del co3
384+
self.assert_event_counts(2, 2, 1, 1)
385+
386+
# verify counts remain as they were after both watchers are cleared
387+
co4 = _testcapi.code_newempty("test_watchers", "dummy4", 0)
388+
self.assert_event_counts(2, 2, 1, 1)
389+
del co4
390+
self.assert_event_counts(2, 2, 1, 1)
391+
392+
def test_clear_out_of_range_watcher_id(self):
393+
with self.assertRaisesRegex(ValueError, r"Invalid code watcher ID -1"):
394+
_testcapi.clear_code_watcher(-1)
395+
with self.assertRaisesRegex(ValueError, r"Invalid code watcher ID 8"):
396+
_testcapi.clear_code_watcher(8) # CODE_MAX_WATCHERS = 8
397+
398+
def test_clear_unassigned_watcher_id(self):
399+
with self.assertRaisesRegex(ValueError, r"No code watcher set for ID 1"):
400+
_testcapi.clear_code_watcher(1)
401+
402+
def test_allocate_too_many_watchers(self):
403+
with self.assertRaisesRegex(RuntimeError, r"no more code watcher IDs available"):
404+
_testcapi.allocate_too_many_code_watchers()
405+
406+
339407
class TestFuncWatchers(unittest.TestCase):
340408
@contextmanager
341409
def add_watcher(self, func):

Misc/ACKS

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1320,6 +1320,7 @@ Michele Orrù
13201320
Tomáš Orsava
13211321
Oleg Oshmyan
13221322
Denis Osipov
1323+
Itamar Ostricher
13231324
Denis S. Otkidach
13241325
Peter Otten
13251326
Michael Otteneder
@@ -1627,6 +1628,7 @@ Silas Sewell
16271628
Ian Seyer
16281629
Dmitry Shachnev
16291630
Anish Shah
1631+
Jaineel Shah
16301632
Daniel Shahaf
16311633
Hui Shang
16321634
Geoff Shannon
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Add :c:func:`PyCode_AddWatcher` and :c:func:`PyCode_ClearWatcher` APIs to
2+
register callbacks to receive notification on creation and destruction of
3+
code objects.

Modules/_testcapi/watchers.c

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
#define Py_BUILD_CORE
44
#include "pycore_function.h" // FUNC_MAX_WATCHERS
5+
#include "pycore_code.h" // CODE_MAX_WATCHERS
56

67
// Test dict watching
78
static PyObject *g_dict_watch_events;
@@ -277,6 +278,126 @@ unwatch_type(PyObject *self, PyObject *args)
277278
Py_RETURN_NONE;
278279
}
279280

281+
282+
// Test code object watching
283+
284+
#define NUM_CODE_WATCHERS 2
285+
static int num_code_object_created_events[NUM_CODE_WATCHERS] = {0, 0};
286+
static int num_code_object_destroyed_events[NUM_CODE_WATCHERS] = {0, 0};
287+
288+
static int
289+
handle_code_object_event(int which_watcher, PyCodeEvent event, PyCodeObject *co) {
290+
if (event == PY_CODE_EVENT_CREATE) {
291+
num_code_object_created_events[which_watcher]++;
292+
}
293+
else if (event == PY_CODE_EVENT_DESTROY) {
294+
num_code_object_destroyed_events[which_watcher]++;
295+
}
296+
else {
297+
return -1;
298+
}
299+
return 0;
300+
}
301+
302+
static int
303+
first_code_object_callback(PyCodeEvent event, PyCodeObject *co)
304+
{
305+
return handle_code_object_event(0, event, co);
306+
}
307+
308+
static int
309+
second_code_object_callback(PyCodeEvent event, PyCodeObject *co)
310+
{
311+
return handle_code_object_event(1, event, co);
312+
}
313+
314+
static int
315+
noop_code_event_handler(PyCodeEvent event, PyCodeObject *co)
316+
{
317+
return 0;
318+
}
319+
320+
static PyObject *
321+
add_code_watcher(PyObject *self, PyObject *which_watcher)
322+
{
323+
int watcher_id;
324+
assert(PyLong_Check(which_watcher));
325+
long which_l = PyLong_AsLong(which_watcher);
326+
if (which_l == 0) {
327+
watcher_id = PyCode_AddWatcher(first_code_object_callback);
328+
}
329+
else if (which_l == 1) {
330+
watcher_id = PyCode_AddWatcher(second_code_object_callback);
331+
}
332+
else {
333+
return NULL;
334+
}
335+
if (watcher_id < 0) {
336+
return NULL;
337+
}
338+
return PyLong_FromLong(watcher_id);
339+
}
340+
341+
static PyObject *
342+
clear_code_watcher(PyObject *self, PyObject *watcher_id)
343+
{
344+
assert(PyLong_Check(watcher_id));
345+
long watcher_id_l = PyLong_AsLong(watcher_id);
346+
if (PyCode_ClearWatcher(watcher_id_l) < 0) {
347+
return NULL;
348+
}
349+
Py_RETURN_NONE;
350+
}
351+
352+
static PyObject *
353+
get_code_watcher_num_created_events(PyObject *self, PyObject *watcher_id)
354+
{
355+
assert(PyLong_Check(watcher_id));
356+
long watcher_id_l = PyLong_AsLong(watcher_id);
357+
assert(watcher_id_l >= 0 && watcher_id_l < NUM_CODE_WATCHERS);
358+
return PyLong_FromLong(num_code_object_created_events[watcher_id_l]);
359+
}
360+
361+
static PyObject *
362+
get_code_watcher_num_destroyed_events(PyObject *self, PyObject *watcher_id)
363+
{
364+
assert(PyLong_Check(watcher_id));
365+
long watcher_id_l = PyLong_AsLong(watcher_id);
366+
assert(watcher_id_l >= 0 && watcher_id_l < NUM_CODE_WATCHERS);
367+
return PyLong_FromLong(num_code_object_destroyed_events[watcher_id_l]);
368+
}
369+
370+
static PyObject *
371+
allocate_too_many_code_watchers(PyObject *self, PyObject *args)
372+
{
373+
int watcher_ids[CODE_MAX_WATCHERS + 1];
374+
int num_watchers = 0;
375+
for (unsigned long i = 0; i < sizeof(watcher_ids) / sizeof(int); i++) {
376+
int watcher_id = PyCode_AddWatcher(noop_code_event_handler);
377+
if (watcher_id == -1) {
378+
break;
379+
}
380+
watcher_ids[i] = watcher_id;
381+
num_watchers++;
382+
}
383+
PyObject *type, *value, *traceback;
384+
PyErr_Fetch(&type, &value, &traceback);
385+
for (int i = 0; i < num_watchers; i++) {
386+
if (PyCode_ClearWatcher(watcher_ids[i]) < 0) {
387+
PyErr_WriteUnraisable(Py_None);
388+
break;
389+
}
390+
}
391+
if (type) {
392+
PyErr_Restore(type, value, traceback);
393+
return NULL;
394+
}
395+
else if (PyErr_Occurred()) {
396+
return NULL;
397+
}
398+
Py_RETURN_NONE;
399+
}
400+
280401
// Test function watchers
281402

282403
#define NUM_FUNC_WATCHERS 2
@@ -509,6 +630,16 @@ static PyMethodDef test_methods[] = {
509630
{"unwatch_type", unwatch_type, METH_VARARGS, NULL},
510631
{"get_type_modified_events", get_type_modified_events, METH_NOARGS, NULL},
511632

633+
// Code object watchers.
634+
{"add_code_watcher", add_code_watcher, METH_O, NULL},
635+
{"clear_code_watcher", clear_code_watcher, METH_O, NULL},
636+
{"get_code_watcher_num_created_events",
637+
get_code_watcher_num_created_events, METH_O, NULL},
638+
{"get_code_watcher_num_destroyed_events",
639+
get_code_watcher_num_destroyed_events, METH_O, NULL},
640+
{"allocate_too_many_code_watchers",
641+
(PyCFunction) allocate_too_many_code_watchers, METH_NOARGS, NULL},
642+
512643
// Function watchers.
513644
{"add_func_watcher", add_func_watcher, METH_O, NULL},
514645
{"clear_func_watcher", clear_func_watcher, METH_O, NULL},

0 commit comments

Comments
 (0)