bpo-44525: Specialize CALL_FUNCTION for C function calls

Include/internal/pycore_code.h

@@ -53,6 +53,14 @@ typedef struct {
     uint32_t builtin_keys_version;
 } _PyLoadGlobalCache;
 
+typedef struct {
+    union {
+        PyCFunction cfunc;
+        /* TODO: func_version field for Python function calls*/
+        uint64_t _; /* Just for alignment on 32-bit */
+    };
+} _PyCallFunctionCache;
+
 /* Add specialized versions of entries to this union.
  *
  * Do not break the invariant: sizeof(SpecializedCacheEntry) == 8
@@ -68,6 +76,7 @@ typedef union {
     _PyAdaptiveEntry adaptive;
     _PyLoadAttrCache load_attr;
     _PyLoadGlobalCache load_global;
+    _PyCallFunctionCache call_function;
 } SpecializedCacheEntry;
 
 #define INSTRUCTIONS_PER_ENTRY (sizeof(SpecializedCacheEntry)/sizeof(_Py_CODEUNIT))
@@ -319,10 +328,25 @@ cache_backoff(_PyAdaptiveEntry *entry) {
     entry->counter = BACKOFF;
 }
 
+/* Corresponds to various function pointers
+https://docs.python.org/3/c-api/structures.html#implementing-functions-and-methods
+*/
+typedef enum {
+    PYCFUNCTION = 1,
+    PYCFUNCTION_O = 2,
+    PYCFUNCTION_NOARGS = 3,
+    PYCFUNCTION_WITH_KEYWORDS = 4,
+    _PYCFUNCTION_FAST = 5,
+    _PYCFUNCTION_FAST_WITH_KEYWORDS = 6,
+    PYCMETHOD = 7,
+} _BuiltinCallKinds;
+
 /* Specialization functions */
 
 int _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache);
 int _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache);
+int _Py_Specialize_CallFunction(PyObject **stack_pointer, uint8_t original_oparg,
+    _Py_CODEUNIT *instr, SpecializedCacheEntry *cache);
 
 #define SPECIALIZATION_STATS 0
 #define SPECIALIZATION_STATS_DETAILED 0

Lib/opcode.py

@@ -229,4 +229,6 @@ def jabs_op(name, op):
     "LOAD_GLOBAL_ADAPTIVE",
     "LOAD_GLOBAL_MODULE",
     "LOAD_GLOBAL_BUILTIN",
+    "CALL_FUNCTION_ADAPTIVE",
+    "CALL_CFUNCTION",
 ]

Misc/NEWS.d/next/Core and Builtins/2021-06-28-22-23-59.bpo-44525.sSvUKG.rst

@@ -0,0 +1,3 @@
+Specialize ``CALL_FUNCTION`` opcode with ``CALL_CFUNCTION``. This speeds up
+calls to ``PyCFunctionObject``. As a result, many builtin functions and
+C-extension functions should experience reduced call overhead.

Python/ceval.c

@@ -55,6 +55,12 @@ Py_LOCAL_INLINE(PyObject *) call_function(
 static PyObject * do_call_core(
     PyThreadState *tstate, PyObject *func,
     PyObject *callargs, PyObject *kwdict, int use_tracing);
+Py_LOCAL_INLINE(PyObject *) call_cfunction(
+    PyThreadState *tstate,
+    _PyAdaptiveEntry *cache0,
+    _PyCallFunctionCache *cache1,
+    PyObject ***pp_stack,
+    Py_ssize_t oparg, int use_tracing);
 
 #ifdef LLTRACE
 static int lltrace;
@@ -4066,7 +4072,48 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
             CHECK_EVAL_BREAKER();
             DISPATCH();
         }
+        case TARGET(CALL_FUNCTION_ADAPTIVE): {
+            SpecializedCacheEntry *cache = GET_CACHE();
+            if (cache->adaptive.counter == 0) {
+                next_instr--;
+                if (_Py_Specialize_CallFunction(stack_pointer, cache->adaptive.original_oparg, next_instr, cache) < 0) {
+                    goto error;
+                }
+                DISPATCH();
+            }
+            else {
+                STAT_INC(CALL_FUNCTION, deferred);
+                cache->adaptive.counter--;
+                oparg = cache->adaptive.original_oparg;
+                JUMP_TO_INSTRUCTION(CALL_FUNCTION);
+            }
+        }
+        case TARGET(CALL_CFUNCTION): {
+            /* Builtin functions, WITHOUT keywords */
+            SpecializedCacheEntry *caches = GET_CACHE();
+            _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
+            _PyCallFunctionCache *cache1 = &caches[-1].call_function;
+            PyObject *callable = PEEK(cache0->original_oparg + 1);
+            DEOPT_IF(!PyCFunction_CheckExact(callable), CALL_FUNCTION);
+            DEOPT_IF(PyCFunction_GET_FUNCTION(callable) != cache1->cfunc, CALL_FUNCTION);
 
+            PyObject **sp, *res;
+            sp = stack_pointer;
+            res = call_cfunction(tstate, cache0, cache1, &sp,
+                cache0->original_oparg, cframe.use_tracing);
+            stack_pointer = sp;
+            PUSH(res);
+            if (res == NULL) {
+                /* Not deopting because this doesn't mean our optimization was wrong.
+                   `res` can be NULL for valid reasons. Eg. getattr(x, 'invalid').
+                   In those cases an exception is set, so we must handle it.
+                */
+                goto error;
+            }
+            record_cache_hit(cache0);
+            STAT_INC(CALL_FUNCTION, hit);
+            DISPATCH();
+        }
         case TARGET(CALL_FUNCTION_KW): {
             PyObject **sp, *res, *names;
 
@@ -4297,6 +4344,7 @@ opname ## _miss: \
 
 MISS_WITH_CACHE(LOAD_ATTR)
 MISS_WITH_CACHE(LOAD_GLOBAL)
+MISS_WITH_CACHE(CALL_FUNCTION)
 
 error:
         /* Double-check exception status. */
@@ -5872,6 +5920,73 @@ do_call_core(PyThreadState *tstate,
     return PyObject_Call(func, callargs, kwdict);
 }
 
+/* Fast alternative for non-keyword calls to C functions. */
+Py_LOCAL_INLINE(PyObject *) _Py_HOT_FUNCTION
+call_cfunction(PyThreadState *tstate,
+              _PyAdaptiveEntry *cache0,
+              _PyCallFunctionCache *cache1,
+              PyObject ***pp_stack,
+              Py_ssize_t oparg,
+              int use_tracing)
+{
+#define MAYBE_TRACE(cfunc) if (use_tracing) {C_TRACE(x, cfunc);} else {x = cfunc;}
+
+    PyObject **pfunc = (*pp_stack) - oparg - 1;
+    PyObject *x = NULL, *w;
+    PyObject **stack = (*pp_stack) - oparg;
+
+    PyObject *func = *pfunc; /* Only for tracing purposes */
+    PyObject *self = PyCFunction_GET_SELF(func);
+    PyCFunction cfunc = PyCFunction_GET_FUNCTION(func);
+
+    switch ((_BuiltinCallKinds)cache0->index) {
+        case PYCFUNCTION_NOARGS:
+        case PYCFUNCTION_O:
+            MAYBE_TRACE(cfunc(self, *stack));
+            break;
+        case _PYCFUNCTION_FAST:
+            MAYBE_TRACE(((_PyCFunctionFast)(void(*)(void))cfunc)(self, stack, oparg));
+            break;
+        case _PYCFUNCTION_FAST_WITH_KEYWORDS:
+            MAYBE_TRACE(((_PyCFunctionFastWithKeywords)(void(*)(void))cfunc)(
+                self, stack, oparg, 0));
+            break;
+        case PYCFUNCTION: {
+            PyObject *args = _PyTuple_FromArray(stack, oparg);
+            if (args == NULL) {
+                break;
+            }
+            MAYBE_TRACE(cfunc(self, args));
+            Py_DECREF(args);
+            break;
+        }
+        case PYCFUNCTION_WITH_KEYWORDS: {
+            PyObject *args = _PyTuple_FromArray(stack, oparg);
+            if (args == NULL) {
+                break;
+            }
+            MAYBE_TRACE(((PyCFunctionWithKeywords)(void(*)(void))cfunc)(self, args, NULL));
+            Py_DECREF(args);
+            break;
+        }
+        /* This flag only applies to PyMethodObject.
+           We're only optimizing for PyCfunctionObject
+        */
+        case PYCMETHOD:
+        default:
+            Py_UNREACHABLE();
+            break;
+    }
+    assert((x != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+
+    /* Clear the stack of the function object. */
+    while ((*pp_stack) > pfunc) {
+        w = EXT_POP(*pp_stack);
+        Py_DECREF(w);
+    }
+
+    return x;
+}
 /* Extract a slice index from a PyLong or an object with the
    nb_index slot defined, and store in *pi.
    Silently reduce values larger than PY_SSIZE_T_MAX to PY_SSIZE_T_MAX,

Python/specialize.c

@@ -158,12 +158,14 @@ get_cache_count(SpecializedCacheOrInstruction *quickened) {
 static uint8_t adaptive_opcodes[256] = {
     [LOAD_ATTR] = LOAD_ATTR_ADAPTIVE,
     [LOAD_GLOBAL] = LOAD_GLOBAL_ADAPTIVE,
+    [CALL_FUNCTION] = CALL_FUNCTION_ADAPTIVE,
 };
 
 /* The number of cache entries required for a "family" of instructions. */
 static uint8_t cache_requirements[256] = {
     [LOAD_ATTR] = 2, /* _PyAdaptiveEntry and _PyLoadAttrCache */
     [LOAD_GLOBAL] = 2, /* _PyAdaptiveEntry and _PyLoadGlobalCache */
+    [CALL_FUNCTION] = 2, /* _PyAdaptiveEntry and _PyCallFunctionCache */
 };
 
 /* Return the oparg for the cache_offset and instruction index.
@@ -633,3 +635,71 @@ _Py_Specialize_LoadGlobal(
     cache0->counter = saturating_start();
     return 0;
 }
+
+/* TODO:
+    - Specialize calling C types like int() with CALL_CTYPE
+    - Specialize python function calls.
+*/
+int
+_Py_Specialize_CallFunction(PyObject **stack_pointer, uint8_t original_oparg,
+    _Py_CODEUNIT *instr, SpecializedCacheEntry *cache)
+{
+    PyObject *callable = stack_pointer[-(original_oparg + 1)];
+    _PyAdaptiveEntry *cache0 = &cache->adaptive;
+    _PyCallFunctionCache *cache1 = &cache[-1].call_function;
+    /* Specialize C functions */
+    if (PyCFunction_CheckExact(callable)) {
+        PyCFunctionObject *meth = (PyCFunctionObject *)callable;
+        if (meth->m_ml == NULL) {
+            goto fail;
+        }
+        const char *name_ascii = meth->m_ml->ml_name;
+        _BuiltinCallKinds kind = -1;
+        switch (PyCFunction_GET_FLAGS(meth) & (METH_VARARGS | METH_FASTCALL |
+            METH_NOARGS | METH_O | METH_KEYWORDS | METH_METHOD)) {
+            case METH_VARARGS:
+                kind = PYCFUNCTION;
+                break;
+            case METH_VARARGS | METH_KEYWORDS:
+                kind = PYCFUNCTION_WITH_KEYWORDS;
+                break;
+            case METH_FASTCALL:
+                kind = _PYCFUNCTION_FAST;
+                break;
+            case METH_FASTCALL | METH_KEYWORDS:
+                kind = _PYCFUNCTION_FAST_WITH_KEYWORDS;
+                break;
+            case METH_NOARGS:
+                kind = PYCFUNCTION_NOARGS;
+                break;
+            case METH_O:
+                kind = PYCFUNCTION_O;
+                break;
+            /* This case should never happen with PyCFunctionObject -- only
+               PyMethodObject. See zlib.compressobj()'s methods for an example.
+            */
+            case METH_METHOD | METH_FASTCALL | METH_KEYWORDS:
+                // kind = PYCMETHOD;
+            default:
+                SPECIALIZATION_FAIL(CALL_FUNCTION, type, callable, "bad call flags");
+                goto fail;
+        }
+        assert(kind > 0);
+        PyCFunction cfunc = PyCFunction_GET_FUNCTION(meth);
+        assert(cfunc != NULL);
+        *instr = _Py_MAKECODEUNIT(CALL_CFUNCTION, _Py_OPARG(*instr));
+        cache0->index = (uint16_t)kind;
+        cache1->cfunc = cfunc;
+        goto success;
+    }
+fail:
+    STAT_INC(CALL_FUNCTION, specialization_failure);
+    assert(!PyErr_Occurred());
+    cache_backoff(cache0);
+    return 0;
+success:
+    STAT_INC(CALL_FUNCTION, specialization_success);
+    assert(!PyErr_Occurred());
+    cache0->counter = saturating_start();
+    return 0;
+}