Split CALL_PY_EXACT_ARGS into uops

This is only the first step for doing `CALL` in Tier 2.
The next step involves tracing into the called code object.
After that we'll have to do the remaining `CALL` specialization.
Finally we'll have to tweak various things like `KW_NAMES`,
and possibly move the `NULL` (for method calls) *above* the callable.
But those are things for future PRs.

Note: this moves setting `frame->return_offset` directly in front of
`DISPATCH_INLINED()`, to make it easier to move it into `_PUSH_FRAME`.

Author: gvanrossum

Include/internal/pycore_opcode_metadata.h

@@ -941,13 +941,13 @@ dummy_func(
             {
                 PyGenObject *gen = (PyGenObject *)receiver;
                 _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe;
-                frame->return_offset = oparg;
                 STACK_SHRINK(1);
                 _PyFrame_StackPush(gen_frame, v);
                 gen->gi_frame_state = FRAME_EXECUTING;
                 gen->gi_exc_state.previous_item = tstate->exc_info;
                 tstate->exc_info = &gen->gi_exc_state;
                 SKIP_OVER(INLINE_CACHE_ENTRIES_SEND);
+                frame->return_offset = oparg;
                 DISPATCH_INLINED(gen_frame);
             }
             if (Py_IsNone(v) && PyIter_Check(receiver)) {
@@ -980,13 +980,13 @@ dummy_func(
             DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND);
             STAT_INC(SEND, hit);
             _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe;
-            frame->return_offset = oparg;
             STACK_SHRINK(1);
             _PyFrame_StackPush(gen_frame, v);
             gen->gi_frame_state = FRAME_EXECUTING;
             gen->gi_exc_state.previous_item = tstate->exc_info;
             tstate->exc_info = &gen->gi_exc_state;
             SKIP_OVER(INLINE_CACHE_ENTRIES_SEND);
+            frame->return_offset = oparg;
             DISPATCH_INLINED(gen_frame);
         }
 
@@ -2578,14 +2578,14 @@ dummy_func(
             DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER);
             STAT_INC(FOR_ITER, hit);
             _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe;
-            frame->return_offset = oparg;
             _PyFrame_StackPush(gen_frame, Py_None);
             gen->gi_frame_state = FRAME_EXECUTING;
             gen->gi_exc_state.previous_item = tstate->exc_info;
             tstate->exc_info = &gen->gi_exc_state;
             SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER);
             assert(next_instr[oparg].op.code == END_FOR ||
                    next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
+            frame->return_offset = oparg;
             DISPATCH_INLINED(gen_frame);
         }
 
@@ -2939,14 +2939,13 @@ dummy_func(
             GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS);
         }
 
-        inst(CALL_PY_EXACT_ARGS, (unused/1, func_version/2, method, callable, args[oparg] -- unused)) {
+        op(_CHECK_CALL_PY_EXACT_ARGS, (func_version/2, method, callable, unused[oparg] -- method, callable, unused[oparg])) {
             ASSERT_KWNAMES_IS_NULL();
             DEOPT_IF(tstate->interp->eval_frame, CALL);
             int is_meth = method != NULL;
             int argcount = oparg;
             if (is_meth) {
                 callable = method;
-                args--;
                 argcount++;
             }
             DEOPT_IF(!PyFunction_Check(callable), CALL);
@@ -2955,18 +2954,35 @@ dummy_func(
             PyCodeObject *code = (PyCodeObject *)func->func_code;
             DEOPT_IF(code->co_argcount != argcount, CALL);
             DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
+        }
+
+        op(_INIT_CALL_PY_EXACT_ARGS, (method, callable, args[oparg] -- new_frame: _PyInterpreterFrame*)) {
+            int is_meth = method != NULL;
+            int argcount = oparg;
+            if (is_meth) {
+                callable = method;
+                args--;
+                argcount++;
+            }
             STAT_INC(CALL, hit);
-            _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, argcount);
+            PyFunctionObject *func = (PyFunctionObject *)callable;
+            new_frame = _PyFrame_PushUnchecked(tstate, func, argcount);
             for (int i = 0; i < argcount; i++) {
                 new_frame->localsplus[i] = args[i];
             }
-            // Manipulate stack directly since we leave using DISPATCH_INLINED().
-            STACK_SHRINK(oparg + 2);
-            SKIP_OVER(INLINE_CACHE_ENTRIES_CALL);
+        }
+
+        op(_PUSH_FRAME, (new_frame: _PyInterpreterFrame* -- unused)) {
             frame->return_offset = 0;
             DISPATCH_INLINED(new_frame);
         }
 
+        macro(CALL_PY_EXACT_ARGS) =
+            unused/1 + // Skip over the counter
+            _CHECK_CALL_PY_EXACT_ARGS +
+            _INIT_CALL_PY_EXACT_ARGS +
+            _PUSH_FRAME;
+
         inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, method, callable, args[oparg] -- unused)) {
             ASSERT_KWNAMES_IS_NULL();
             DEOPT_IF(tstate->interp->eval_frame, CALL);

Python/bytecodes.c

@@ -30,6 +30,19 @@
 #undef ENABLE_SPECIALIZATION
 #define ENABLE_SPECIALIZATION 0
 
+#undef DISPATCH_INLINED
+#define DISPATCH_INLINED(NEW_FRAME)                             \
+    do {                                                        \
+        assert(tstate->interp->eval_frame == NULL);             \
+        _PyFrame_SetStackPointer(frame, stack_pointer);         \
+        frame->prev_instr -= 1;                                 \
+        (NEW_FRAME)->previous = frame;                          \
+        frame = tstate->cframe->current_frame = (NEW_FRAME);    \
+        CALL_STAT_INC(inlined_py_calls);                        \
+        stack_pointer = _PyFrame_GetStackPointer(frame);        \
+        ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; \
+    } while (0)
+
 
 _PyInterpreterFrame *
 _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer)

Python/executor.c

@@ -602,6 +602,10 @@ translate_bytecode_to_trace(
                             case OPARG_BOTTOM:  // Second half of super-instr
                                 oparg = orig_oparg & 0xF;
                                 break;
+                            case OPARG_SAVE_IP:  // op==SAVE_IP; oparg=next instr
+                                oparg = INSTR_IP(instr + offset, code);
+                                break;
+
                             default:
                                 fprintf(stderr,
                                         "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n",
@@ -611,6 +615,10 @@ translate_bytecode_to_trace(
                                 Py_FatalError("garbled expansion");
                         }
                         ADD_TO_TRACE(expansion->uops[i].uop, oparg, operand);
+                        if (expansion->uops[i].uop == _PUSH_FRAME) {
+                            ADD_TO_TRACE(SAVE_IP, 0, 0);
+                            goto done;
+                        }
                     }
                     break;
                 }