Add an optional callback that is invoked whenever a function is modified

JIT compilers may need to invalidate compiled code when a function is
modified (e.g. if its code object is modified). This adds the ability
to set a callback that, when set, is called each time a function is
modified.

Author: mpage

Doc/c-api/function.rst

@@ -118,3 +118,56 @@ There are a few functions specific to Python functions.
    must be a dictionary or ``Py_None``.
 
    Raises :exc:`SystemError` and returns ``-1`` on failure.
+
+
+.. c:function:: int PyFunction_AddWatcher(PyFunction_WatchCallback callback)
+
+   Register *callback* as a function watcher for the current interpreter. Returns
+   an id which may be passed to :c:func:`PyFunction_ClearWatcher`. In case
+   of error (e.g. no more watcher IDs available), return ``-1`` and set an
+   exception.
+
+   .. versionadded:: 3.12
+
+
+.. c:function:: int PyFunction_ClearWatcher(int watcher_id)
+
+   Clear watcher identified by *watcher_id* previously returned from
+   :c:func:`PyFunction_AddWatcher` for the current interpreter. Return ``0`` on
+   success or ``-1`` on error (e.g.  if the given *watcher_id* was never
+   registered.)
+
+   .. versionadded:: 3.12
+
+
+.. c:type:: PyFunction_WatchEvent
+
+   Enumeration of possible function watcher events: ``PyFunction_EVENT_CREATED``,
+   ``PyFunction_EVENT_DESTROY``, ``PyFunction_EVENT_MODIFY_CODE``,
+   ``PyFunction_EVENT_MODIFY_DEFAULTS``, or ``PyFunction_EVENT_MODIFY_KWDEFAULTS``.
+
+   .. versionadded:: 3.12
+
+
+.. c:type:: int (*PyFunction_WatchCallback)(PyFunction_WatchEvent event, PyFunctionObject *func, PyObject *new_value)
+
+   Type of a function watcher callback function.
+
+   If *event* is ``PyFunction_EVENT_CREATED`` or ``PyFunction_EVENT_DESTROY``
+   then *new_value* will be ``NULL``. Otherwise, *new_value* will hold a
+   borrowed reference to the new value that is about to be stored in *func* for
+   the attribute that is being modified.
+
+   The callback may inspect but must not modify *func*; doing so could have
+   unpredictable effects, including infinite recursion.
+
+   If *event* is ``PyFunction_EVENT_CREATED``, then the callback is invoked
+   after `func` has been fully initialized. Otherwise, the callback is invoked
+   before the modification to *func* takes place, so the prior state of *func*
+   can be inspected.
+
+   If the callback returns with an exception set, it must return ``-1``; this
+   exception will be printed as an unraisable exception using
+   :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``.
+
+   .. versionadded:: 3.12

Include/cpython/funcobject.h

@@ -131,6 +131,52 @@ PyAPI_DATA(PyTypeObject) PyStaticMethod_Type;
 PyAPI_FUNC(PyObject *) PyClassMethod_New(PyObject *);
 PyAPI_FUNC(PyObject *) PyStaticMethod_New(PyObject *);
 
+#define FOREACH_FUNC_EVENT(V) \
+    V(CREATED)                \
+    V(DESTROY)                \
+    V(MODIFY_CODE)            \
+    V(MODIFY_DEFAULTS)        \
+    V(MODIFY_KWDEFAULTS)
+
+typedef enum {
+    #define DEF_EVENT(EVENT) PyFunction_EVENT_##EVENT,
+    FOREACH_FUNC_EVENT(DEF_EVENT)
+    #undef DEF_EVENT
+} PyFunction_WatchEvent;
+
+/*
+ * A callback that is invoked for different events in a function's lifecycle.
+ *
+ * The callback is invoked with a borrowed reference to func, after it is
+ * created and before it is modified or destroyed. The callback should not
+ * modify func.
+ *
+ * When a function's code object, defaults, or kwdefaults are modified the
+ * callback will be invoked with the respective event and new_value will
+ * contain a borrowed reference to the new value that is about to be stored in
+ * the function. Otherwise the third argument is NULL.
+ */
+typedef int (*PyFunction_WatchCallback)(
+  PyFunction_WatchEvent event,
+  PyFunctionObject *func,
+  PyObject *new_value);
+
+/*
+ * Register a per-interpreter callback that will be invoked for function lifecycle
+ * events.
+ *
+ * Returns a handle that may be passed to PyFunction_ClearWatcher on success,
+ * or -1 and sets an error if no more handles are available.
+ */
+PyAPI_FUNC(int) PyFunction_AddWatcher(PyFunction_WatchCallback callback);
+
+/*
+ * Clear the watcher associated with the watcher_id handle.
+ *
+ * Returns 0 on success or -1 if no watcher exists for the supplied id.
+ */
+PyAPI_FUNC(int) PyFunction_ClearWatcher(int watcher_id);
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_function.h

@@ -8,6 +8,8 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#define FUNC_MAX_WATCHERS 8
+
 extern PyFunctionObject* _PyFunction_FromConstructor(PyFrameConstructor *constr);
 
 extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func);

Include/internal/pycore_interp.h

@@ -17,6 +17,7 @@ extern "C" {
 #include "pycore_dict.h"          // struct _Py_dict_state
 #include "pycore_exceptions.h"    // struct _Py_exc_state
 #include "pycore_floatobject.h"   // struct _Py_float_state
+#include "pycore_function.h"      // FUNC_MAX_WATCHERS
 #include "pycore_genobject.h"     // struct _Py_async_gen_state
 #include "pycore_gc.h"            // struct _gc_runtime_state
 #include "pycore_list.h"          // struct _Py_list_state
@@ -147,6 +148,7 @@ struct _is {
     _PyFrameEvalFunction eval_frame;
 
     PyDict_WatchCallback dict_watchers[DICT_MAX_WATCHERS];
+    PyFunction_WatchCallback func_watchers[FUNC_MAX_WATCHERS];
 
     Py_ssize_t co_extra_user_count;
     freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS];

Lib/test/test_func_events.py

@@ -0,0 +1,97 @@
+import unittest
+
+from contextlib import contextmanager
+from test.support import catch_unraisable_exception, import_helper
+
+_testcapi = import_helper.import_module("_testcapi")
+
+from _testcapi import (
+    PYFUNC_EVENT_CREATED,
+    PYFUNC_EVENT_DESTROY,
+    PYFUNC_EVENT_MODIFY_CODE,
+    PYFUNC_EVENT_MODIFY_DEFAULTS,
+    PYFUNC_EVENT_MODIFY_KWDEFAULTS,
+    _add_func_watcher,
+    _clear_func_watcher,
+)
+
+
+class FuncEventsTest(unittest.TestCase):
+    @contextmanager
+    def add_watcher(self, func):
+        wid = _add_func_watcher(func)
+        try:
+            yield
+        finally:
+            _clear_func_watcher(wid)
+
+    def test_func_events_dispatched(self):
+        events = []
+        def watcher(*args):
+            events.append(args)
+
+        with self.add_watcher(watcher):
+            def myfunc():
+                pass
+            self.assertIn((PYFUNC_EVENT_CREATED, myfunc, None), events)
+            myfunc_id = id(myfunc)
+
+            new_code = self.test_func_events_dispatched.__code__
+            myfunc.__code__ = new_code
+            self.assertIn((PYFUNC_EVENT_MODIFY_CODE, myfunc, new_code), events)
+
+            new_defaults = (123,)
+            myfunc.__defaults__ = new_defaults
+            self.assertIn((PYFUNC_EVENT_MODIFY_DEFAULTS, myfunc, new_defaults), events)
+
+            new_kwdefaults = {"self": 123}
+            myfunc.__kwdefaults__ = new_kwdefaults
+            self.assertIn((PYFUNC_EVENT_MODIFY_KWDEFAULTS, myfunc, new_kwdefaults), events)
+
+            # Clear events reference to func
+            events = []
+            del myfunc
+            self.assertIn((PYFUNC_EVENT_DESTROY, myfunc_id, None), events)
+
+    def test_multiple_watchers(self):
+        events0 = []
+        def first_watcher(*args):
+            events0.append(args)
+
+        events1 = []
+        def second_watcher(*args):
+            events1.append(args)
+
+        with self.add_watcher(first_watcher):
+            with self.add_watcher(second_watcher):
+                def myfunc():
+                    pass
+
+                event = (PYFUNC_EVENT_CREATED, myfunc, None)
+                self.assertIn(event, events0)
+                self.assertIn(event, events1)
+
+    def test_watcher_raises_error(self):
+        class MyError(Exception):
+            pass
+
+        def watcher(*args):
+            raise MyError("testing 123")
+
+        with self.add_watcher(watcher):
+            with catch_unraisable_exception() as cm:
+                def myfunc():
+                    pass
+
+                self.assertIs(cm.unraisable.object, myfunc)
+                self.assertIsInstance(cm.unraisable.exc_value, MyError)
+
+    def test_clear_out_of_range_watcher_id(self):
+        with self.assertRaisesRegex(ValueError, r"Invalid func watcher ID -1"):
+            _clear_func_watcher(-1)
+        with self.assertRaisesRegex(ValueError, r"Invalid func watcher ID 8"):
+            _clear_func_watcher(8)  # FUNC_MAX_WATCHERS = 8
+
+    def test_clear_unassigned_watcher_id(self):
+        with self.assertRaisesRegex(ValueError, r"No func watcher set for ID 1"):
+            _clear_func_watcher(1)

Misc/NEWS.d/next/Core and Builtins/2022-10-05-11-44-52.gh-issue-91053.f5Bo3p.rst

@@ -0,0 +1,4 @@
+Optimizing interpreters and JIT compilers may need to invalidate internal
+metadata when functions are modified. This change adds the ability to
+provide a callback that will be invoked each time a function is created,
+modified, or destroyed.

Modules/Setup.stdlib.in

@@ -169,7 +169,7 @@
 @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c
 @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c
 @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c
-@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c
+@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/func_events.c
 
 # Some testing modules MUST be built as shared libraries.
 *shared*