diff --git a/.gitattributes b/.gitattributes index 13289182400109..35d81d575bf1e1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -83,6 +83,7 @@ Parser/token.c generated Programs/test_frozenmain.h generated Python/Python-ast.c generated Python/generated_cases.c.h generated +Python/tier2_typepropagator.c.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated diff --git a/Include/cpython/code.h b/Include/cpython/code.h index bf087922432e1a..c1f8a0c27ccce3 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -56,18 +56,24 @@ typedef struct { PyObject *_co_freevars; } _PyCoCached; +// Tier 2 types meta interpreter +typedef struct _PyTier2TypeContext { + // points into type_stack, points to one element after the stack + PyTypeObject** type_stack_ptr; + int type_locals_len; + int type_stack_len; + PyTypeObject** type_stack; + PyTypeObject** type_locals; +} _PyTier2TypeContext; + // Tier 2 interpreter information typedef struct _PyTier2BBMetadata { // Index into _PyTier2Info->bb_data int id; - // Array of types. This corresponds to the fast locals array. - int type_context_len; - PyTypeObject **type_context; + _PyTier2TypeContext *type_context; _Py_CODEUNIT *tier2_start; // Note, this is the first tier 1 instruction to execute AFTER the BB ends. _Py_CODEUNIT *tier1_end; - // Type stack state - PyTypeObject **types_stack; } _PyTier2BBMetadata; // Bump allocator for basic blocks (overallocated) @@ -97,13 +103,19 @@ typedef struct _PyTier2Info { // will have [[BB_ID1, BB_ID2], [BB_ID3,], [], []] // etc. int **backward_jump_target_bb_ids; - PyTypeObject **types_stack; // Max len of bb_data int bb_data_len; // Current index to write into in bb_data. Incremented after each write. // This also assigns the BB ID. int bb_data_curr; _PyTier2BBMetadata **bb_data; + + // @TODO: + // Potentially optimise _PyTier2TypeContext by allocating the stacksize + // to the size needed for the snapshot, and the type propagation is performed + // on type_metainterpreter_stack_scratch which is allocated only once per + // code object. + // PyTypeObject** type_metainterpreter_stack_scratch; } _PyTier2Info; // To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 914c0c03e95c9b..5ca88116a31b42 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -259,7 +259,7 @@ extern _Py_CODEUNIT *_PyCode_Tier2Warmup(struct _PyInterpreterFrame *, _Py_CODEUNIT *); extern _Py_CODEUNIT *_PyTier2_GenerateNextBB( struct _PyInterpreterFrame *frame, uint16_t bb_id, int jumpby, - _Py_CODEUNIT **tier1_fallback); + _Py_CODEUNIT **tier1_fallback, char gen_bb_requires_pop); extern _Py_CODEUNIT *_PyTier2_LocateJumpBackwardsBB( struct _PyInterpreterFrame *frame, uint16_t bb_id, int jumpby, _Py_CODEUNIT **tier1_fallback); diff --git a/Objects/codeobject.c b/Objects/codeobject.c index cb956c643ba864..089139073a4d7e 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1672,6 +1672,11 @@ code_tier2_fini(PyCodeObject *co) if (co->_tier2_info == NULL) { return; } + // @TODO: + // Write a proper destructor for _PyTier2Info + // and it's children structures. + // Current implementation e.g., doesn't clear + // bb_data _PyTier2Info *t2_info = co->_tier2_info; t2_info->_entry_bb = NULL; if (t2_info->_bb_space != NULL) { @@ -1690,10 +1695,7 @@ code_tier2_fini(PyCodeObject *co) PyMem_Free(t2_info->backward_jump_offsets); t2_info->backward_jump_offsets = NULL; } - if (t2_info->types_stack != NULL) { - PyMem_Free(t2_info->types_stack); - t2_info->types_stack = NULL; - } + t2_info->backward_jump_count = 0; if (t2_info->bb_data != NULL && t2_info->bb_data_len > 0) { PyMem_Free(t2_info->bb_data); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6f707d2175a3e5..59f3db48f80283 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -102,31 +102,31 @@ dummy_func( } } - inst(LOAD_CLOSURE, (-- value)) { + inst(LOAD_CLOSURE, (-- value : locals[oparg])) { /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */ value = GETLOCAL(oparg); ERROR_IF(value == NULL, unbound_local_error); Py_INCREF(value); } - inst(LOAD_FAST_CHECK, (-- value)) { + inst(LOAD_FAST_CHECK, (-- value : locals[oparg])) { value = GETLOCAL(oparg); ERROR_IF(value == NULL, unbound_local_error); Py_INCREF(value); } - inst(LOAD_FAST, (-- value)) { + inst(LOAD_FAST, (-- value : locals[oparg])) { value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); } - inst(LOAD_CONST, (-- value)) { + inst(LOAD_CONST, (-- value : consts[oparg])) { value = GETITEM(consts, oparg); Py_INCREF(value); } - inst(STORE_FAST, (value --)) { + inst(STORE_FAST, (value --), locals[oparg] = *value) { SETLOCAL(oparg, value); } @@ -303,7 +303,7 @@ dummy_func( bb_test = PyLong_CheckExact(left) && (Py_TYPE(left) == Py_TYPE(right)); } - u_inst(BINARY_OP_ADD_INT_REST, (left, right -- sum)) { + u_inst(BINARY_OP_ADD_INT_REST, (left : PyLong_Type, right : PyLong_Type -- sum : PyLong_Type)) { STAT_INC(BINARY_OP, hit); sum = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -1180,13 +1180,13 @@ dummy_func( null = NULL; } - inst(DELETE_FAST, (--)) { + inst(DELETE_FAST, (--), locals[oparg] = NULL) { PyObject *v = GETLOCAL(oparg); ERROR_IF(v == NULL, unbound_local_error); SETLOCAL(oparg, NULL); } - inst(MAKE_CELL, (--)) { + inst(MAKE_CELL, (--), locals[oparg] = NULL) { // "initial" is probably NULL but not if it's an arg (or set // via PyFrame_LocalsToFast() before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); @@ -2084,6 +2084,10 @@ dummy_func( goto error; } } + // This gets set so BRANCH_BB knows whether to pop + // the type stack (type propagation) when generating the + // target BB + gen_bb_requires_pop = !jump; } inst(JUMP_IF_TRUE_OR_POP, (cond -- cond if (jump))) { @@ -2136,6 +2140,10 @@ dummy_func( goto error; } } + // This gets set so BRANCH_BB knows whether to pop + // the type stack (type propagation) when generating the + // target BB + gen_bb_requires_pop = !jump; } inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { @@ -3284,6 +3292,7 @@ dummy_func( } // Tier 2 instructions + // Type propagator assumes this doesn't affect type context inst(BB_BRANCH, (unused/1 --)) { _Py_CODEUNIT *t2_nextinstr = NULL; _PyBBBranchCache *cache = (_PyBBBranchCache *)next_instr; @@ -3293,7 +3302,8 @@ dummy_func( _py_set_opcode(next_instr - 1, BB_BRANCH_IF_FLAG_UNSET); // Generate consequent. t2_nextinstr = _PyTier2_GenerateNextBB( - frame, cache->bb_id, 0, &tier1_fallback); + frame, cache->bb_id, 0, &tier1_fallback, gen_bb_requires_pop); + gen_bb_requires_pop = false; if (t2_nextinstr == NULL) { // Fall back to tier 1. next_instr = tier1_fallback; @@ -3305,7 +3315,8 @@ dummy_func( _py_set_opcode(next_instr - 1, BB_BRANCH_IF_FLAG_SET); // Generate predicate. t2_nextinstr = _PyTier2_GenerateNextBB( - frame, cache->bb_id, oparg, &tier1_fallback); + frame, cache->bb_id, oparg, &tier1_fallback, gen_bb_requires_pop); + gen_bb_requires_pop = false; if (t2_nextinstr == NULL) { // Fall back to tier 1. next_instr = tier1_fallback + oparg; @@ -3327,7 +3338,8 @@ dummy_func( _Py_CODEUNIT *tier1_fallback = NULL; t2_nextinstr = _PyTier2_GenerateNextBB( - frame, cache->bb_id, oparg, &tier1_fallback); + frame, cache->bb_id, oparg, &tier1_fallback, gen_bb_requires_pop); + gen_bb_requires_pop = false; if (t2_nextinstr == NULL) { // Fall back to tier 1. next_instr = tier1_fallback; @@ -3358,7 +3370,8 @@ dummy_func( // @TODO: Rewrite TEST intruction above to a JUMP above.. t2_nextinstr = _PyTier2_GenerateNextBB( - frame, cache->bb_id, oparg, &tier1_fallback); + frame, cache->bb_id, oparg, &tier1_fallback, gen_bb_requires_pop); + gen_bb_requires_pop = false; if (t2_nextinstr == NULL) { // Fall back to tier 1. next_instr = tier1_fallback; @@ -3379,6 +3392,7 @@ dummy_func( // Fall through to next instruction. } + // Type propagator assumes this doesn't affect type context inst(BB_JUMP_BACKWARD_LAZY, (--)) { _Py_CODEUNIT *curr = next_instr - 1; _Py_CODEUNIT *t2_nextinstr = NULL; diff --git a/Python/ceval.c b/Python/ceval.c index 7d0f7a8813411f..9f30fbefb92cd8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -729,6 +729,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int // true = successor // false = alternate bool bb_test = true; + // For tier2 type propagation, handling of jump instructions with + // runtime-dependent stack effect. + // This flag is used to determine if the type context of a new bb + // requires a stack element to be popped. + bool gen_bb_requires_pop = false; /* WARNING: Because the _PyCFrame lives on the C stack, * but can be accessed from a heap allocated object (tstate) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2f7d223618a6ac..ba0aa72c68f22d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2643,6 +2643,10 @@ goto error; } } + // This gets set so BRANCH_BB knows whether to pop + // the type stack (type propagation) when generating the + // target BB + gen_bb_requires_pop = jump; STACK_SHRINK(1); STACK_GROW((jump ? 1 : 0)); DISPATCH(); @@ -2703,6 +2707,10 @@ goto error; } } + // This gets set so BRANCH_BB knows whether to pop + // the type stack (type propagation) when generating the + // target BB + gen_bb_requires_pop = jump; STACK_SHRINK(1); STACK_GROW((jump ? 1 : 0)); DISPATCH(); @@ -4129,7 +4137,8 @@ _py_set_opcode(next_instr - 1, BB_BRANCH_IF_FLAG_UNSET); // Generate consequent. t2_nextinstr = _PyTier2_GenerateNextBB( - frame, cache->bb_id, 0, &tier1_fallback); + frame, cache->bb_id, 0, &tier1_fallback, gen_bb_requires_pop); + gen_bb_requires_pop = false; if (t2_nextinstr == NULL) { // Fall back to tier 1. next_instr = tier1_fallback; @@ -4141,7 +4150,8 @@ _py_set_opcode(next_instr - 1, BB_BRANCH_IF_FLAG_SET); // Generate predicate. t2_nextinstr = _PyTier2_GenerateNextBB( - frame, cache->bb_id, oparg, &tier1_fallback); + frame, cache->bb_id, oparg, &tier1_fallback, gen_bb_requires_pop); + gen_bb_requires_pop = false; if (t2_nextinstr == NULL) { // Fall back to tier 1. next_instr = tier1_fallback + oparg; @@ -4163,7 +4173,8 @@ _Py_CODEUNIT *tier1_fallback = NULL; t2_nextinstr = _PyTier2_GenerateNextBB( - frame, cache->bb_id, oparg, &tier1_fallback); + frame, cache->bb_id, oparg, &tier1_fallback, gen_bb_requires_pop); + gen_bb_requires_pop = false; if (t2_nextinstr == NULL) { // Fall back to tier 1. next_instr = tier1_fallback; @@ -4198,7 +4209,8 @@ // @TODO: Rewrite TEST intruction above to a JUMP above.. t2_nextinstr = _PyTier2_GenerateNextBB( - frame, cache->bb_id, oparg, &tier1_fallback); + frame, cache->bb_id, oparg, &tier1_fallback, gen_bb_requires_pop); + gen_bb_requires_pop = false; if (t2_nextinstr == NULL) { // Fall back to tier 1. next_instr = tier1_fallback; diff --git a/Python/tier2.c b/Python/tier2.c index 0da0a7f6be37ef..c49ee8eea1dda8 100644 --- a/Python/tier2.c +++ b/Python/tier2.c @@ -9,31 +9,164 @@ #include "opcode.h" #define BB_DEBUG 1 +#define TYPEPROP_DEBUG 1 // Max typed version basic blocks per basic block #define MAX_BB_VERSIONS 5 // Number of potential extra instructions at end of a BB, for branch or cleanup purposes. #define BB_EPILOG 0 + static inline int IS_SCOPE_EXIT_OPCODE(int opcode); ////////// TYPE CONTEXT FUNCTIONS -static PyTypeObject ** -initialize_type_context(PyCodeObject *co, int *type_context_len) { +static _PyTier2TypeContext * +initialize_type_context(const PyCodeObject *co) +{ + +#if TYPEPROP_DEBUG + fprintf(stderr, " [*] Initialize type context\n"); +#endif + int nlocals = co->co_nlocals; - PyTypeObject **type_context = PyMem_Malloc(nlocals * sizeof(PyTypeObject *)); - if (type_context == NULL) { + int nstack = co->co_stacksize; + + PyTypeObject **type_locals = PyMem_Malloc(nlocals * sizeof(PyTypeObject *)); + if (type_locals == NULL) { return NULL; } - // Initialize to uknown type. + PyTypeObject **type_stack = PyMem_Malloc(nstack * sizeof(PyTypeObject *)); + if (type_stack == NULL) { + PyMem_Free(type_locals); + return NULL; + } + + // Initialize to unknown type. for (int i = 0; i < nlocals; i++) { - type_context[i] = NULL; + type_locals[i] = NULL; + } + for (int i = 0; i < nstack; i++) { + type_stack[i] = NULL; } - *type_context_len = nlocals; + + _PyTier2TypeContext *type_context = PyMem_Malloc(sizeof(_PyTier2TypeContext)); + if (type_context == NULL) { + PyMem_Free(type_locals); + PyMem_Free(type_stack); + return NULL; + } + type_context->type_locals_len = nlocals; + type_context->type_stack_len = nstack; + type_context->type_locals = type_locals; + type_context->type_stack = type_stack; + type_context->type_stack_ptr = type_stack; // init ptr at start of stack return type_context; } +static _PyTier2TypeContext * +_PyTier2TypeContext_Copy(const _PyTier2TypeContext *type_context) +{ + +#if TYPEPROP_DEBUG + fprintf(stderr, " [*] Copying type context\n"); +#endif + + int nlocals = type_context->type_locals_len; + int nstack = type_context->type_stack_len; + + PyTypeObject **type_locals = PyMem_Malloc(nlocals * sizeof(PyTypeObject *)); + if (type_locals == NULL) { + return NULL; + } + PyTypeObject **type_stack = PyMem_Malloc(nstack * sizeof(PyTypeObject *)); + if (type_stack == NULL) { + PyMem_Free(type_locals); + return NULL; + } + + memcpy(type_locals, type_context->type_locals, nlocals * sizeof(PyTypeObject *)); + memcpy(type_stack, type_context->type_stack, nstack * sizeof(PyTypeObject *)); + + _PyTier2TypeContext *new_type_context = PyMem_Malloc(sizeof(_PyTier2TypeContext)); + if (new_type_context == NULL) { + PyMem_Free(type_locals); + PyMem_Free(type_stack); + return NULL; + } + new_type_context->type_locals_len = nlocals; + new_type_context->type_stack_len = nstack; + new_type_context->type_locals = type_locals; + new_type_context->type_stack = type_stack; + new_type_context->type_stack_ptr = type_stack - type_context->type_stack + type_context->type_stack_ptr; + return new_type_context; +} + +static void +_PyTier2TypeContext_Free(_PyTier2TypeContext *type_context) +{ + +#if TYPEPROP_DEBUG + fprintf(stderr, " [*] Freeing type context\n"); +#endif + + PyMem_Free(type_context->type_locals); + PyMem_Free(type_context->type_stack); + PyMem_Free(type_context); +} + +// Type propagates across a single function. +static void +type_propagate( + int opcode, int oparg, + _PyTier2TypeContext *type_context, + const PyObject *consts) +{ + PyTypeObject **type_stack = type_context->type_stack; + PyTypeObject **type_locals = type_context->type_locals; + PyTypeObject **type_stackptr = type_context->type_stack_ptr; + +#define TARGET(op) case op: +#define TYPESTACK_PEEK(idx) (type_stackptr[-(idx)]) +#define TYPESTACK_POKE(idx, v) type_stackptr[-(idx)] = (v) +#define TYPELOCALS_SET(idx, v) type_locals[idx] = v; +#define TYPELOCALS_GET(idx) (type_locals[idx]) +#define TYPECONST_GET(idx) Py_TYPE(_PyTuple_CAST(consts)->ob_item[(idx)]) +#define STACK_ADJUST(idx) type_stackptr += (idx) +#define STACK_GROW(idx) STACK_ADJUST((idx)) +#define STACK_SHRINK(idx) STACK_ADJUST(-(idx)) + +#ifdef TYPEPROP_DEBUG + fprintf(stderr, " [-] Type stack bef: %llu\n", ((uint64_t)type_stackptr - (uint64_t)type_stack) / sizeof(PyTypeObject *)); +#ifdef Py_DEBUG + fprintf(stderr, " [-] Type propagating across: %s : %d\n", _PyOpcode_OpName[opcode], oparg); +#endif +#endif + + switch (opcode) { +#include "tier2_typepropagator.c.h" + default: + fprintf(stderr, "Unsupported opcode in type propagator: %d\n", opcode); + Py_UNREACHABLE(); + } + +#ifdef TYPEPROP_DEBUG + fprintf(stderr, " [-] Type stack aft: %llu\n", ((uint64_t)type_stackptr - (uint64_t)type_stack) / sizeof(PyTypeObject *)); +#endif + + type_context->type_stack_ptr = type_stackptr; + +#undef TARGET +#undef TYPESTACK_PEEK +#undef TYPESTACK_POKE +#undef TYPELOCALS_SET +#undef TYPELOCALS_GET +#undef TYPECONST_GET +#undef STACK_ADJUST +#undef STACK_GROW +#undef STACK_SHRINK +} + ////////// Utility functions // Gets end of the bytecode for a code object. @@ -108,18 +241,18 @@ _PyTier2_BBSpaceCheckAndReallocIfNeeded(PyCodeObject *co, Py_ssize_t space_reque static _PyTier2BBMetadata * allocate_bb_metadata(PyCodeObject *co, _Py_CODEUNIT *tier2_start, - _Py_CODEUNIT *tier1_end, int type_context_len, - PyTypeObject **type_context) + _Py_CODEUNIT *tier1_end, + _PyTier2TypeContext *type_context) { _PyTier2BBMetadata *metadata = PyMem_Malloc(sizeof(_PyTier2BBMetadata)); if (metadata == NULL) { return NULL; + } metadata->tier2_start = tier2_start; metadata->tier1_end = tier1_end; metadata->type_context = type_context; - metadata->type_context_len = type_context_len; return metadata; } @@ -161,11 +294,11 @@ write_bb_metadata(PyCodeObject *co, _PyTier2BBMetadata *metadata) static _PyTier2BBMetadata * _PyTier2_AllocateBBMetaData(PyCodeObject *co, _Py_CODEUNIT *tier2_start, - _Py_CODEUNIT *tier1_end, int type_context_len, - PyTypeObject **type_context) + _Py_CODEUNIT *tier1_end, + _PyTier2TypeContext *type_context) { _PyTier2BBMetadata *meta = allocate_bb_metadata(co, - tier2_start, tier1_end, type_context_len, type_context); + tier2_start, tier1_end, type_context); if (meta == NULL) { return NULL; } @@ -196,7 +329,7 @@ IS_JREL_OPCODE(int opcode) case JUMP_FORWARD: case JUMP_IF_FALSE_OR_POP: case JUMP_IF_TRUE_OR_POP: - // These two tend to be after a COMPARE_AND_BRANCH. + // These two tend to be after a COMPARE_AND_BRANCH. case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: case SEND: @@ -263,27 +396,27 @@ static inline int IS_FORBIDDEN_OPCODE(int opcode) { switch (opcode) { - // Generators and coroutines + // Generators and coroutines case SEND: case YIELD_VALUE: - // Raise keyword + // Raise keyword case RAISE_VARARGS: - // Exceptions, we could support these theoretically. - // Just too much work for now + // Exceptions, we could support these theoretically. + // Just too much work for now case PUSH_EXC_INFO: case RERAISE: case POP_EXCEPT: - // Closures + // Closures case LOAD_DEREF: case MAKE_CELL: - // DELETE_FAST + // DELETE_FAST case DELETE_FAST: - // Pattern matching + // Pattern matching case MATCH_MAPPING: case MATCH_SEQUENCE: case MATCH_KEYS: - // Too large arguments, we can handle this, just - // increases complexity + // Too large arguments, we can handle this, just + // increases complexity case EXTENDED_ARG: return 1; @@ -292,71 +425,6 @@ IS_FORBIDDEN_OPCODE(int opcode) } } -static inline PyTypeObject * -INSTR_LOCAL_READ_TYPE(PyCodeObject *co, _Py_CODEUNIT instr, PyTypeObject **type_context) -{ - int opcode = _PyOpcode_Deopt[_Py_OPCODE(instr)]; - int oparg = _Py_OPARG(instr); - switch (opcode) { - case LOAD_CONST: - return Py_TYPE(PyTuple_GET_ITEM(co->co_consts, oparg)); - case LOAD_FAST: - return type_context[oparg]; - // Note: Don't bother with LOAD_NAME, because those exist only in the global - // scope. - default: - return NULL; - } -} - -/* -* This does multiple things: -* 1. Gets the result type of an instruction, if it can figure it out. - 2. Determines whether thet instruction has a corresponding macro/micro instruction - that acts a guard, and whether it needs it. - 3. How many guards it needs, and what are the guards. - 4. Finally, the corresponding specialised micro-op to operate on this type. - - Returns the inferred type of an operation. NULL if unknown. - - how_many_guards returns the number of guards (0, 1) - guards should be a opcode corresponding to the guard instruction to use. -*/ -static PyTypeObject * -BINARY_OP_RESULT_TYPE(PyCodeObject *co, _Py_CODEUNIT *instr, int n_typecontext, - PyTypeObject **type_context, int *how_many_guards, _Py_CODEUNIT *guard, _Py_CODEUNIT *action) -{ - int opcode = _PyOpcode_Deopt[_Py_OPCODE(*instr)]; - int oparg = _Py_OPARG(*instr); - switch (opcode) { - case BINARY_OP: - if (oparg == NB_ADD) { - // For BINARY OP, read the previous two load instructions - // to see what variables we need to type check. - PyTypeObject *lhs_type = INSTR_LOCAL_READ_TYPE(co, *(instr - 2), type_context); - PyTypeObject *rhs_type = INSTR_LOCAL_READ_TYPE(co, *(instr - 1), type_context); - // The two instruction types are known. - if (lhs_type == &PyLong_Type) { - if (rhs_type == &PyLong_Type) { - *how_many_guards = 0; - action->op.code = BINARY_OP_ADD_INT_REST; - return &PyLong_Type; - } - //// Right side unknown. Emit single check. - //else if (rhs_type == NULL) { - // *how_many_guards = 1; - // guard->opcode = - // return NULL; - //} - } - // We don't know anything, need to emit guard. - // @TODO - } - break; - } - return NULL; -} - static inline _Py_CODEUNIT * emit_cache_entries(_Py_CODEUNIT *write_curr, int cache_entries) { @@ -387,7 +455,7 @@ emit_type_guard(_Py_CODEUNIT *write_curr, _Py_CODEUNIT guard, int bb_id) // Converts the tier 1 branch bytecode to tier 2 branch bytecode. static inline _Py_CODEUNIT * -emit_logical_branch(_Py_CODEUNIT *write_curr, _Py_CODEUNIT branch, int bb_id) +emit_logical_branch(_PyTier2TypeContext *type_context, _Py_CODEUNIT *write_curr, _Py_CODEUNIT branch, int bb_id) { int opcode; int oparg = _Py_OPARG(branch); @@ -399,27 +467,38 @@ emit_logical_branch(_Py_CODEUNIT *write_curr, _Py_CODEUNIT branch, int bb_id) // Subsequent jumps don't need to check this anymore. They can just // jump directly with JUMP_BACKWARD. opcode = BB_JUMP_BACKWARD_LAZY; + // v BB_JUMP_BACKWARD_LAZY has nothing to propagate + // type_propagate(opcode, oparg, type_context, NULL); break; case FOR_ITER: opcode = BB_TEST_ITER; + type_propagate(opcode, oparg, type_context, NULL); break; case JUMP_IF_FALSE_OR_POP: opcode = BB_TEST_IF_FALSE_OR_POP; + // This inst has conditional stack effect according to whether the branch is taken. + // This inst sets the `gen_bb_requires_pop` flag to handle stack effect of this opcode in BB_BRANCH break; case JUMP_IF_TRUE_OR_POP: opcode = BB_TEST_IF_TRUE_OR_POP; + // This inst has conditional stack effect according to whether the branch is taken. + // This inst sets the `gen_bb_requires_pop` flag to handle stack effect of this opcode in BB_BRANCH break; case POP_JUMP_IF_FALSE: opcode = BB_TEST_POP_IF_FALSE; + type_propagate(opcode, oparg, type_context, NULL); break; case POP_JUMP_IF_TRUE: opcode = BB_TEST_POP_IF_TRUE; + type_propagate(opcode, oparg, type_context, NULL); break; case POP_JUMP_IF_NOT_NONE: opcode = BB_TEST_POP_IF_NOT_NONE; + type_propagate(opcode, oparg, type_context, NULL); break; case POP_JUMP_IF_NONE: opcode = BB_TEST_POP_IF_NONE; + type_propagate(opcode, oparg, type_context, NULL); break; default: // Honestly shouldn't happen because branches that @@ -611,38 +690,32 @@ add_metadata_to_jump_2d_array(_PyTier2Info *t2_info, _PyTier2BBMetadata *meta, _PyTier2BBMetadata * _PyTier2_Code_DetectAndEmitBB(PyCodeObject *co, _PyTier2BBSpace *bb_space, _Py_CODEUNIT *tier1_start, - int n_typecontext, PyTypeObject **type_context) + // starting_type_context will be modified in this function, + // do make a copy if needed before calling this function + _PyTier2TypeContext *starting_type_context) { #define END() goto end; #define JUMPBY(x) i += x + 1; -#define BASIC_PUSH(v) (*stack_pointer++ = (v)) -#define BASIC_POP() (*--stack_pointer) #define DISPATCH() write_i = emit_i(write_i, opcode, oparg); \ write_i = copy_cache_entries(write_i, curr+1, caches); \ i += caches; \ + type_propagate(opcode, oparg, starting_type_context, consts); \ continue; #define DISPATCH_GOTO() goto dispatch_opcode; - + assert(co->_tier2_info != NULL); // There are only two cases that a BB ends. // 1. If there's a branch instruction / scope exit. // 2. If there's a type guard. - // Make a copy of the type context - PyTypeObject **type_context_copy = PyMem_Malloc(n_typecontext * sizeof(PyTypeObject *)); - if (type_context_copy == NULL) { - return NULL; - } - memcpy(type_context_copy, type_context, n_typecontext * sizeof(PyTypeObject *)); - _PyTier2BBMetadata *meta = NULL; _PyTier2BBMetadata *temp_meta = NULL; _PyTier2Info *t2_info = co->_tier2_info; + PyObject *consts = co->co_consts; _Py_CODEUNIT *t2_start = (_Py_CODEUNIT *)(((char *)bb_space->u_code) + bb_space->water_level); _Py_CODEUNIT *write_i = t2_start; - PyTypeObject **stack_pointer = co->_tier2_info->types_stack; int tos = -1; // For handling of backwards jumps @@ -661,25 +734,15 @@ _PyTier2_Code_DetectAndEmitBB(PyCodeObject *co, _PyTier2BBSpace *bb_space, // Just because an instruction requires a guard doesn't mean it's the end of a BB. // We need to check whether we can eliminate the guard based on the current type context. - int how_many_guards = 0; - _Py_CODEUNIT guard_instr; - _Py_CODEUNIT action; - + // int how_many_guards = 0; + // _Py_CODEUNIT guard_instr; + // _Py_CODEUNIT action; + dispatch_opcode: switch (opcode) { case RESUME: opcode = RESUME_QUICK; DISPATCH(); - //case EXTENDED_ARG: - // write_i = emit_i(write_i, EXTENDED_ARG, _Py_OPARG(*curr)); - // curr++; - // next_instr++; - // i++; - // oparg = oparg << 8 | _Py_OPARG(*curr); - // opcode = _Py_OPCODE(*curr); - // caches = _PyOpcode_Caches[opcode]; - // DISPATCH_GOTO(); - // We need to rewrite the pseudo-branch instruction. case COMPARE_AND_BRANCH: opcode = COMPARE_OP; DISPATCH(); @@ -691,7 +754,9 @@ _PyTier2_Code_DetectAndEmitBB(PyCodeObject *co, _PyTier2BBSpace *bb_space, t2_start++; DISPATCH(); default: +#if BB_DEBUG || TYPEPROP_DEBUG fprintf(stderr, "offset: %Id\n", curr - _PyCode_CODE(co)); +#endif if (IS_BACKWARDS_JUMP_TARGET(co, curr)) { #if BB_DEBUG fprintf(stderr, "Encountered a backward jump target\n"); @@ -710,10 +775,15 @@ _PyTier2_Code_DetectAndEmitBB(PyCodeObject *co, _PyTier2BBSpace *bb_space, // Else, create a virtual end to the basic block. // But generate the block after that so it can fall through. i--; + // Make a copy of the type context + _PyTier2TypeContext *type_context_copy = _PyTier2TypeContext_Copy(starting_type_context); + if (type_context_copy == NULL) { + return NULL; + } meta = _PyTier2_AllocateBBMetaData(co, - t2_start, _PyCode_CODE(co) + i, n_typecontext, type_context_copy); + t2_start, _PyCode_CODE(co) + i, type_context_copy); if (meta == NULL) { - PyMem_Free(type_context_copy); + _PyTier2TypeContext_Free(type_context_copy); return NULL; } bb_space->water_level += (write_i - t2_start) * sizeof(_Py_CODEUNIT); @@ -742,7 +812,7 @@ _PyTier2_Code_DetectAndEmitBB(PyCodeObject *co, _PyTier2BBSpace *bb_space, } // Get the BB ID without incrementing it. // AllocateBBMetaData will increment. - write_i = emit_logical_branch(write_i, *curr, + write_i = emit_logical_branch(starting_type_context, write_i, *curr, co->_tier2_info->bb_data_curr); i += caches; END(); @@ -753,9 +823,19 @@ _PyTier2_Code_DetectAndEmitBB(PyCodeObject *co, _PyTier2BBSpace *bb_space, } end: // Create the tier 2 BB + + // Make a copy of the type context + _PyTier2TypeContext *type_context_copy = _PyTier2TypeContext_Copy(starting_type_context); + if (type_context_copy == NULL) { + return NULL; + } temp_meta = _PyTier2_AllocateBBMetaData(co, t2_start, - // + 1 because we want to start with the NEXT instruction for the scan - _PyCode_CODE(co) + i + 1, n_typecontext, type_context_copy); + // + 1 because we want to start with the NEXT instruction for the scan + _PyCode_CODE(co) + i + 1, type_context_copy); + if (temp_meta == NULL) { + _PyTier2TypeContext_Free(type_context_copy); + return NULL; + } // We need to return the first block to enter into. If there is already a block generated // before us, then we use that instead of the most recent block. if (meta == NULL) { @@ -765,8 +845,10 @@ _PyTier2_Code_DetectAndEmitBB(PyCodeObject *co, _PyTier2BBSpace *bb_space, // Add the basic block to the jump ids if (add_metadata_to_jump_2d_array(t2_info, temp_meta, backwards_jump_target_offset) < 0) { PyMem_Free(meta); - PyMem_Free(temp_meta); - PyMem_Free(type_context_copy); + if (meta != temp_meta) { + PyMem_Free(temp_meta); + } + _PyTier2TypeContext_Free(type_context_copy); return NULL; } } @@ -802,7 +884,7 @@ allocate_jump_offset_2d_array(int backwards_jump_count, int **backward_jump_targ for (int i = 0; i < MAX_BB_VERSIONS; i++) { jump_offsets[i] = -1; } - done++; + done++; backward_jump_target_bb_ids[i] = jump_offsets; } return 0; @@ -876,8 +958,8 @@ _PyCode_Tier2FillJumpTargets(PyCodeObject *co) i += _PyOpcode_Caches[opcode]; } assert(curr_i == backwards_jump_count); - qsort(backward_jump_offsets,backwards_jump_count, - sizeof(int), compare_ints); + qsort(backward_jump_offsets, backwards_jump_count, + sizeof(int), compare_ints); #if BB_DEBUG fprintf(stderr, "BACKWARD JUMP COUNT : %Id\n", backwards_jump_count); fprintf(stderr, "BACKWARD JUMP TARGET OFFSETS (FROM START OF CODE): "); @@ -903,13 +985,6 @@ _PyTier2Info_Initialize(PyCodeObject *co) return NULL; } - // Next is to intitialize stack space for the tier 2 types meta-interpretr. - PyTypeObject **types_stack = PyMem_Malloc(co->co_stacksize * sizeof(PyObject *)); - if (types_stack == NULL) { - PyMem_Free(t2_info); - return NULL; - } - t2_info->types_stack = types_stack; t2_info->backward_jump_count = 0; t2_info->backward_jump_offsets = NULL; @@ -919,10 +994,9 @@ _PyTier2Info_Initialize(PyCodeObject *co) t2_info->bb_data_curr = 0; Py_ssize_t bb_data_len = (Py_SIZE(co) / 5 + 1); assert((int)bb_data_len == bb_data_len); - _PyTier2BBMetadata **bb_data = PyMem_Malloc(bb_data_len * sizeof(_PyTier2Info *)); + _PyTier2BBMetadata **bb_data = PyMem_Calloc(bb_data_len, sizeof(_PyTier2BBMetadata *)); if (bb_data == NULL) { PyMem_Free(t2_info); - PyMem_Free(types_stack); return NULL; } t2_info->bb_data_len = (int)bb_data_len; @@ -932,7 +1006,6 @@ _PyTier2Info_Initialize(PyCodeObject *co) return t2_info; } - ////////// OVERALL TIER2 FUNCTIONS @@ -1035,23 +1108,22 @@ _PyCode_Tier2Initialize(_PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr) t2_info->_bb_space = bb_space; - int type_context_len = 0; - PyTypeObject **type_context = initialize_type_context(co, &type_context_len); + _PyTier2TypeContext *type_context = initialize_type_context(co); if (type_context == NULL) { goto cleanup; } _PyTier2BBMetadata *meta = _PyTier2_Code_DetectAndEmitBB( co, bb_space, - _PyCode_CODE(co), type_context_len, type_context); + _PyCode_CODE(co), type_context); if (meta == NULL) { - PyMem_Free(type_context); + _PyTier2TypeContext_Free(type_context); goto cleanup; } #if BB_DEBUG fprintf(stderr, "ENTRY BB END IS: %d\n", (int)(meta->tier1_end - _PyCode_CODE(co))); #endif - + t2_info->_entry_bb = meta; // SET THE FRAME INFO @@ -1093,7 +1165,7 @@ _PyCode_Tier2Warmup(_PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr) // The second basic block created will always require a jump. _Py_CODEUNIT * _PyTier2_GenerateNextBB(_PyInterpreterFrame *frame, uint16_t bb_id, int jumpby, - _Py_CODEUNIT **tier1_fallback) + _Py_CODEUNIT **tier1_fallback, char gen_bb_requires_pop) { PyCodeObject *co = frame->f_code; assert(co->_tier2_info != NULL); @@ -1110,16 +1182,24 @@ _PyTier2_GenerateNextBB(_PyInterpreterFrame *frame, uint16_t bb_id, int jumpby, // DEOPTIMIZE TO TIER 1? return NULL; } - int type_context_len = 0; - PyTypeObject **type_context = initialize_type_context(frame->f_code, &type_context_len); - if (type_context == NULL) { + // Get type_context of previous BB + _PyTier2TypeContext *type_context = meta->type_context; + // Make a copy of the type context + _PyTier2TypeContext *type_context_copy = _PyTier2TypeContext_Copy(type_context); + if (type_context_copy == NULL) { return NULL; } + // If this flag is set, it means that either BB_TEST_IF_FALSE_OR_POP or + // BB_TEST_IF_TRUE_OR_POP was ran and the conditional stack effect was performed + // This means we have to pop an element from the type stack. + if (gen_bb_requires_pop) { + type_context_copy->type_stack_ptr--; + } _PyTier2BBMetadata *metadata = _PyTier2_Code_DetectAndEmitBB( - frame->f_code, space, tier1_end, type_context_len, - type_context); + frame->f_code, space, tier1_end, + type_context_copy); if (metadata == NULL) { - PyMem_Free(type_context); + _PyTier2TypeContext_Free(type_context_copy); return NULL; } return metadata->tier2_start; @@ -1145,11 +1225,9 @@ _PyTier2_LocateJumpBackwardsBB(_PyInterpreterFrame *frame, uint16_t bb_id, int j // DEOPTIMIZE TO TIER 1? return NULL; } - int type_context_len = 0; - PyTypeObject **type_context = initialize_type_context(frame->f_code, &type_context_len); - if (type_context == NULL) { - return NULL; - } + + // Get type_context of previous BB + _PyTier2TypeContext *type_context = meta->type_context; // Now, find the matching BB _PyTier2Info *t2_info = co->_tier2_info; int jump_offset = (int)(tier1_jump_target - _PyCode_CODE(co)); @@ -1166,8 +1244,8 @@ _PyTier2_LocateJumpBackwardsBB(_PyInterpreterFrame *frame, uint16_t bb_id, int j for (int x = 0; x < MAX_BB_VERSIONS; x++) { #if BB_DEBUG fprintf(stderr, "jump target BB ID: %d\n", -#endif t2_info->backward_jump_target_bb_ids[i][x]); +#endif // @TODO, this is where the diff function is supposed to be // it will calculate the closest type context BB // For now just any valid BB (>= 0) is used. @@ -1181,7 +1259,9 @@ _PyTier2_LocateJumpBackwardsBB(_PyInterpreterFrame *frame, uint16_t bb_id, int j } assert(matching_bb_id >= 0); assert(matching_bb_id <= t2_info->bb_data_curr); +#if BB_DEBUG fprintf(stderr, "Found jump target BB ID: %d\n", matching_bb_id); +#endif _PyTier2BBMetadata *target_metadata = t2_info->bb_data[matching_bb_id]; return target_metadata->tier2_start; } diff --git a/Python/tier2_typepropagator.c.h b/Python/tier2_typepropagator.c.h new file mode 100644 index 00000000000000..786eff5afbcf23 --- /dev/null +++ b/Python/tier2_typepropagator.c.h @@ -0,0 +1,1278 @@ +// This file is generated by Tools\cases_generator\generate_cases.py @TODO: make this a seperate argument +// from Python\bytecodes.c +// Do not edit! + + TARGET(NOP) { + break; + } + + TARGET(RESUME) { + break; + } + + TARGET(RESUME_QUICK) { + break; + } + + TARGET(LOAD_CLOSURE) { + STACK_GROW(1); + TYPESTACK_POKE(1, TYPELOCALS_GET(oparg)); + break; + } + + TARGET(LOAD_FAST_CHECK) { + STACK_GROW(1); + TYPESTACK_POKE(1, TYPELOCALS_GET(oparg)); + break; + } + + TARGET(LOAD_FAST) { + STACK_GROW(1); + TYPESTACK_POKE(1, TYPELOCALS_GET(oparg)); + break; + } + + TARGET(LOAD_CONST) { + STACK_GROW(1); + TYPESTACK_POKE(1, TYPECONST_GET(oparg)); + break; + } + + TARGET(STORE_FAST) { + PyTypeObject *value = TYPESTACK_PEEK(1); + TYPELOCALS_SET(oparg, value) + STACK_SHRINK(1); + break; + } + + TARGET(POP_TOP) { + PyTypeObject *value = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + break; + } + + TARGET(PUSH_NULL) { + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(END_FOR) { + { + PyTypeObject *value = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + } + { + PyTypeObject *value = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + } + break; + } + + TARGET(UNARY_NEGATIVE) { + PyTypeObject *value = TYPESTACK_PEEK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(UNARY_NOT) { + PyTypeObject *value = TYPESTACK_PEEK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(UNARY_INVERT) { + PyTypeObject *value = TYPESTACK_PEEK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(BINARY_OP_MULTIPLY_INT) { + PyTypeObject *right = TYPESTACK_PEEK(1); + PyTypeObject *left = TYPESTACK_PEEK(2); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(BINARY_OP_MULTIPLY_FLOAT) { + PyTypeObject *right = TYPESTACK_PEEK(1); + PyTypeObject *left = TYPESTACK_PEEK(2); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(BINARY_OP_SUBTRACT_INT) { + PyTypeObject *right = TYPESTACK_PEEK(1); + PyTypeObject *left = TYPESTACK_PEEK(2); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(BINARY_OP_SUBTRACT_FLOAT) { + PyTypeObject *right = TYPESTACK_PEEK(1); + PyTypeObject *left = TYPESTACK_PEEK(2); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(BINARY_OP_ADD_UNICODE) { + PyTypeObject *right = TYPESTACK_PEEK(1); + PyTypeObject *left = TYPESTACK_PEEK(2); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + PyTypeObject *right = TYPESTACK_PEEK(1); + PyTypeObject *left = TYPESTACK_PEEK(2); + STACK_SHRINK(2); + break; + } + + TARGET(BINARY_OP_ADD_FLOAT) { + PyTypeObject *right = TYPESTACK_PEEK(1); + PyTypeObject *left = TYPESTACK_PEEK(2); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(BINARY_OP_ADD_INT) { + PyTypeObject *right = TYPESTACK_PEEK(1); + PyTypeObject *left = TYPESTACK_PEEK(2); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(BINARY_CHECK_INT) { + PyTypeObject *right = TYPESTACK_PEEK(1); + PyTypeObject *left = TYPESTACK_PEEK(2); + TYPESTACK_POKE(1, &PyLong_Type); + TYPESTACK_POKE(2, &PyLong_Type); + break; + } + + TARGET(BINARY_OP_ADD_INT_REST) { + PyTypeObject *right = TYPESTACK_PEEK(1); + PyTypeObject *left = TYPESTACK_PEEK(2); + STACK_SHRINK(1); + TYPESTACK_POKE(1, &PyLong_Type); + break; + } + + TARGET(BINARY_SUBSCR) { + PyTypeObject *sub = TYPESTACK_PEEK(1); + PyTypeObject *container = TYPESTACK_PEEK(2); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(BINARY_SLICE) { + PyTypeObject *stop = TYPESTACK_PEEK(1); + PyTypeObject *start = TYPESTACK_PEEK(2); + PyTypeObject *container = TYPESTACK_PEEK(3); + STACK_SHRINK(2); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(STORE_SLICE) { + PyTypeObject *stop = TYPESTACK_PEEK(1); + PyTypeObject *start = TYPESTACK_PEEK(2); + PyTypeObject *container = TYPESTACK_PEEK(3); + PyTypeObject *v = TYPESTACK_PEEK(4); + STACK_SHRINK(4); + break; + } + + TARGET(BINARY_SUBSCR_LIST_INT) { + PyTypeObject *sub = TYPESTACK_PEEK(1); + PyTypeObject *list = TYPESTACK_PEEK(2); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(BINARY_SUBSCR_TUPLE_INT) { + PyTypeObject *sub = TYPESTACK_PEEK(1); + PyTypeObject *tuple = TYPESTACK_PEEK(2); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(BINARY_SUBSCR_DICT) { + PyTypeObject *sub = TYPESTACK_PEEK(1); + PyTypeObject *dict = TYPESTACK_PEEK(2); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(BINARY_SUBSCR_GETITEM) { + PyTypeObject *sub = TYPESTACK_PEEK(1); + PyTypeObject *container = TYPESTACK_PEEK(2); + STACK_SHRINK(1); + break; + } + + TARGET(LIST_APPEND) { + PyTypeObject *v = TYPESTACK_PEEK(1); + PyTypeObject *list = TYPESTACK_PEEK(2 + (oparg-1)); + STACK_SHRINK(1); + break; + } + + TARGET(SET_ADD) { + PyTypeObject *v = TYPESTACK_PEEK(1); + PyTypeObject *set = TYPESTACK_PEEK(2 + (oparg-1)); + STACK_SHRINK(1); + break; + } + + TARGET(STORE_SUBSCR) { + PyTypeObject *sub = TYPESTACK_PEEK(1); + PyTypeObject *container = TYPESTACK_PEEK(2); + PyTypeObject *v = TYPESTACK_PEEK(3); + STACK_SHRINK(3); + break; + } + + TARGET(STORE_SUBSCR_LIST_INT) { + PyTypeObject *sub = TYPESTACK_PEEK(1); + PyTypeObject *list = TYPESTACK_PEEK(2); + PyTypeObject *value = TYPESTACK_PEEK(3); + STACK_SHRINK(3); + break; + } + + TARGET(STORE_SUBSCR_DICT) { + PyTypeObject *sub = TYPESTACK_PEEK(1); + PyTypeObject *dict = TYPESTACK_PEEK(2); + PyTypeObject *value = TYPESTACK_PEEK(3); + STACK_SHRINK(3); + break; + } + + TARGET(DELETE_SUBSCR) { + PyTypeObject *sub = TYPESTACK_PEEK(1); + PyTypeObject *container = TYPESTACK_PEEK(2); + STACK_SHRINK(2); + break; + } + + TARGET(CALL_INTRINSIC_1) { + PyTypeObject *value = TYPESTACK_PEEK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(CALL_INTRINSIC_2) { + PyTypeObject *value1 = TYPESTACK_PEEK(1); + PyTypeObject *value2 = TYPESTACK_PEEK(2); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(RAISE_VARARGS) { + PyTypeObject **args = &TYPESTACK_PEEK(oparg); + STACK_SHRINK(oparg); + break; + } + + TARGET(INTERPRETER_EXIT) { + PyTypeObject *retval = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + break; + } + + TARGET(RETURN_VALUE) { + PyTypeObject *retval = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + break; + } + + TARGET(RETURN_CONST) { + break; + } + + TARGET(GET_AITER) { + PyTypeObject *obj = TYPESTACK_PEEK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(GET_ANEXT) { + PyTypeObject *aiter = TYPESTACK_PEEK(1); + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(GET_AWAITABLE) { + PyTypeObject *iterable = TYPESTACK_PEEK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(SEND) { + PyTypeObject *v = TYPESTACK_PEEK(1); + PyTypeObject *receiver = TYPESTACK_PEEK(2); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(SEND_GEN) { + PyTypeObject *v = TYPESTACK_PEEK(1); + PyTypeObject *receiver = TYPESTACK_PEEK(2); + STACK_SHRINK(1); + break; + } + + TARGET(YIELD_VALUE) { + PyTypeObject *retval = TYPESTACK_PEEK(1); + break; + } + + TARGET(POP_EXCEPT) { + PyTypeObject *exc_value = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + break; + } + + TARGET(RERAISE) { + PyTypeObject *exc = TYPESTACK_PEEK(1); + PyTypeObject **values = &TYPESTACK_PEEK(1 + oparg); + STACK_SHRINK(1); + break; + } + + TARGET(END_ASYNC_FOR) { + PyTypeObject *exc = TYPESTACK_PEEK(1); + PyTypeObject *awaitable = TYPESTACK_PEEK(2); + STACK_SHRINK(2); + break; + } + + TARGET(CLEANUP_THROW) { + PyTypeObject *exc_value = TYPESTACK_PEEK(1); + PyTypeObject *last_sent_val = TYPESTACK_PEEK(2); + PyTypeObject *sub_iter = TYPESTACK_PEEK(3); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + TYPESTACK_POKE(2, NULL); + break; + } + + TARGET(LOAD_ASSERTION_ERROR) { + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(LOAD_BUILD_CLASS) { + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(STORE_NAME) { + PyTypeObject *v = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + break; + } + + TARGET(DELETE_NAME) { + break; + } + + TARGET(UNPACK_SEQUENCE) { + PyTypeObject *seq = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + STACK_GROW(oparg); + break; + } + + TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + PyTypeObject *seq = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + STACK_GROW(oparg); + TYPESTACK_POKE(oparg, NULL); + break; + } + + TARGET(UNPACK_SEQUENCE_TUPLE) { + PyTypeObject *seq = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + STACK_GROW(oparg); + TYPESTACK_POKE(oparg, NULL); + break; + } + + TARGET(UNPACK_SEQUENCE_LIST) { + PyTypeObject *seq = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + STACK_GROW(oparg); + TYPESTACK_POKE(oparg, NULL); + break; + } + + TARGET(UNPACK_EX) { + PyTypeObject *seq = TYPESTACK_PEEK(1); + STACK_GROW((oparg & 0xFF) + (oparg >> 8)); + break; + } + + TARGET(STORE_ATTR) { + PyTypeObject *owner = TYPESTACK_PEEK(1); + PyTypeObject *v = TYPESTACK_PEEK(2); + STACK_SHRINK(2); + break; + } + + TARGET(DELETE_ATTR) { + PyTypeObject *owner = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + break; + } + + TARGET(STORE_GLOBAL) { + PyTypeObject *v = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + break; + } + + TARGET(DELETE_GLOBAL) { + break; + } + + TARGET(LOAD_NAME) { + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(LOAD_GLOBAL) { + STACK_GROW(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + TYPESTACK_POKE(1, NULL); + if (oparg & 1) { TYPESTACK_POKE(1 + ((oparg & 1) ? 1 : 0), NULL); } + break; + } + + TARGET(LOAD_GLOBAL_MODULE) { + STACK_GROW(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + TYPESTACK_POKE(1, NULL); + if (oparg & 1) { TYPESTACK_POKE(1 + ((oparg & 1) ? 1 : 0), NULL); } + break; + } + + TARGET(LOAD_GLOBAL_BUILTIN) { + STACK_GROW(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + TYPESTACK_POKE(1, NULL); + if (oparg & 1) { TYPESTACK_POKE(1 + ((oparg & 1) ? 1 : 0), NULL); } + break; + } + + TARGET(DELETE_FAST) { + TYPELOCALS_SET(oparg, NULL) + break; + } + + TARGET(MAKE_CELL) { + TYPELOCALS_SET(oparg, NULL) + break; + } + + TARGET(DELETE_DEREF) { + break; + } + + TARGET(LOAD_CLASSDEREF) { + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(LOAD_DEREF) { + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(STORE_DEREF) { + PyTypeObject *v = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + break; + } + + TARGET(COPY_FREE_VARS) { + break; + } + + TARGET(BUILD_STRING) { + PyTypeObject **pieces = &TYPESTACK_PEEK(oparg); + STACK_SHRINK(oparg); + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(BUILD_TUPLE) { + PyTypeObject **values = &TYPESTACK_PEEK(oparg); + STACK_SHRINK(oparg); + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(BUILD_LIST) { + PyTypeObject **values = &TYPESTACK_PEEK(oparg); + STACK_SHRINK(oparg); + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(LIST_EXTEND) { + PyTypeObject *iterable = TYPESTACK_PEEK(1); + PyTypeObject *list = TYPESTACK_PEEK(2 + (oparg-1)); + STACK_SHRINK(1); + break; + } + + TARGET(SET_UPDATE) { + PyTypeObject *iterable = TYPESTACK_PEEK(1); + PyTypeObject *set = TYPESTACK_PEEK(2 + (oparg-1)); + STACK_SHRINK(1); + break; + } + + TARGET(BUILD_SET) { + PyTypeObject **values = &TYPESTACK_PEEK(oparg); + STACK_SHRINK(oparg); + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(BUILD_MAP) { + PyTypeObject **values = &TYPESTACK_PEEK(oparg*2); + STACK_SHRINK(oparg*2); + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(SETUP_ANNOTATIONS) { + break; + } + + TARGET(BUILD_CONST_KEY_MAP) { + PyTypeObject *keys = TYPESTACK_PEEK(1); + PyTypeObject **values = &TYPESTACK_PEEK(1 + oparg); + STACK_SHRINK(oparg); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(DICT_UPDATE) { + PyTypeObject *update = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + break; + } + + TARGET(DICT_MERGE) { + PyTypeObject *update = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + break; + } + + TARGET(MAP_ADD) { + PyTypeObject *value = TYPESTACK_PEEK(1); + PyTypeObject *key = TYPESTACK_PEEK(2); + STACK_SHRINK(2); + break; + } + + TARGET(LOAD_ATTR) { + PyTypeObject *owner = TYPESTACK_PEEK(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + TYPESTACK_POKE(1, NULL); + if (oparg & 1) { TYPESTACK_POKE(1 + ((oparg & 1) ? 1 : 0), NULL); } + break; + } + + TARGET(LOAD_ATTR_INSTANCE_VALUE) { + PyTypeObject *owner = TYPESTACK_PEEK(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + TYPESTACK_POKE(1, NULL); + if (oparg & 1) { TYPESTACK_POKE(1 + ((oparg & 1) ? 1 : 0), NULL); } + break; + } + + TARGET(LOAD_ATTR_MODULE) { + PyTypeObject *owner = TYPESTACK_PEEK(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + TYPESTACK_POKE(1, NULL); + if (oparg & 1) { TYPESTACK_POKE(1 + ((oparg & 1) ? 1 : 0), NULL); } + break; + } + + TARGET(LOAD_ATTR_WITH_HINT) { + PyTypeObject *owner = TYPESTACK_PEEK(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + TYPESTACK_POKE(1, NULL); + if (oparg & 1) { TYPESTACK_POKE(1 + ((oparg & 1) ? 1 : 0), NULL); } + break; + } + + TARGET(LOAD_ATTR_SLOT) { + PyTypeObject *owner = TYPESTACK_PEEK(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + TYPESTACK_POKE(1, NULL); + if (oparg & 1) { TYPESTACK_POKE(1 + ((oparg & 1) ? 1 : 0), NULL); } + break; + } + + TARGET(LOAD_ATTR_CLASS) { + PyTypeObject *cls = TYPESTACK_PEEK(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + TYPESTACK_POKE(1, NULL); + if (oparg & 1) { TYPESTACK_POKE(1 + ((oparg & 1) ? 1 : 0), NULL); } + break; + } + + TARGET(LOAD_ATTR_PROPERTY) { + PyTypeObject *owner = TYPESTACK_PEEK(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + break; + } + + TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { + PyTypeObject *owner = TYPESTACK_PEEK(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + break; + } + + TARGET(STORE_ATTR_INSTANCE_VALUE) { + PyTypeObject *owner = TYPESTACK_PEEK(1); + PyTypeObject *value = TYPESTACK_PEEK(2); + STACK_SHRINK(2); + break; + } + + TARGET(STORE_ATTR_WITH_HINT) { + PyTypeObject *owner = TYPESTACK_PEEK(1); + PyTypeObject *value = TYPESTACK_PEEK(2); + STACK_SHRINK(2); + break; + } + + TARGET(STORE_ATTR_SLOT) { + PyTypeObject *owner = TYPESTACK_PEEK(1); + PyTypeObject *value = TYPESTACK_PEEK(2); + STACK_SHRINK(2); + break; + } + + TARGET(COMPARE_OP) { + PyTypeObject *right = TYPESTACK_PEEK(1); + PyTypeObject *left = TYPESTACK_PEEK(2); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(COMPARE_AND_BRANCH) { + PyTypeObject *right = TYPESTACK_PEEK(1); + PyTypeObject *left = TYPESTACK_PEEK(2); + STACK_SHRINK(2); + break; + } + + TARGET(COMPARE_AND_BRANCH_FLOAT) { + PyTypeObject *right = TYPESTACK_PEEK(1); + PyTypeObject *left = TYPESTACK_PEEK(2); + STACK_SHRINK(2); + break; + } + + TARGET(COMPARE_AND_BRANCH_INT) { + PyTypeObject *right = TYPESTACK_PEEK(1); + PyTypeObject *left = TYPESTACK_PEEK(2); + STACK_SHRINK(2); + break; + } + + TARGET(COMPARE_AND_BRANCH_STR) { + PyTypeObject *right = TYPESTACK_PEEK(1); + PyTypeObject *left = TYPESTACK_PEEK(2); + STACK_SHRINK(2); + break; + } + + TARGET(IS_OP) { + PyTypeObject *right = TYPESTACK_PEEK(1); + PyTypeObject *left = TYPESTACK_PEEK(2); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(CONTAINS_OP) { + PyTypeObject *right = TYPESTACK_PEEK(1); + PyTypeObject *left = TYPESTACK_PEEK(2); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(CHECK_EG_MATCH) { + PyTypeObject *match_type = TYPESTACK_PEEK(1); + PyTypeObject *exc_value = TYPESTACK_PEEK(2); + TYPESTACK_POKE(1, NULL); + TYPESTACK_POKE(2, NULL); + break; + } + + TARGET(CHECK_EXC_MATCH) { + PyTypeObject *right = TYPESTACK_PEEK(1); + PyTypeObject *left = TYPESTACK_PEEK(2); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(IMPORT_NAME) { + PyTypeObject *fromlist = TYPESTACK_PEEK(1); + PyTypeObject *level = TYPESTACK_PEEK(2); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(IMPORT_FROM) { + PyTypeObject *from = TYPESTACK_PEEK(1); + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(JUMP_FORWARD) { + break; + } + + TARGET(JUMP_BACKWARD) { + break; + } + + TARGET(JUMP_BACKWARD_QUICK) { + break; + } + + TARGET(POP_JUMP_IF_FALSE) { + PyTypeObject *cond = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + break; + } + + TARGET(BB_TEST_POP_IF_FALSE) { + PyTypeObject *cond = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + break; + } + + TARGET(POP_JUMP_IF_TRUE) { + PyTypeObject *cond = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + break; + } + + TARGET(BB_TEST_POP_IF_TRUE) { + PyTypeObject *cond = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + break; + } + + TARGET(POP_JUMP_IF_NOT_NONE) { + PyTypeObject *value = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + break; + } + + TARGET(BB_TEST_POP_IF_NOT_NONE) { + PyTypeObject *value = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + break; + } + + TARGET(POP_JUMP_IF_NONE) { + PyTypeObject *value = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + break; + } + + TARGET(BB_TEST_POP_IF_NONE) { + PyTypeObject *value = TYPESTACK_PEEK(1); + STACK_SHRINK(1); + break; + } + + TARGET(JUMP_IF_FALSE_OR_POP) { + fprintf(stderr, "Type propagation across `{self.name}` shouldn't be handled statically!\n"); + Py_UNREACHABLE(); + break; + } + + TARGET(BB_TEST_IF_FALSE_OR_POP) { + fprintf(stderr, "Type propagation across `{self.name}` shouldn't be handled statically!\n"); + Py_UNREACHABLE(); + break; + } + + TARGET(JUMP_IF_TRUE_OR_POP) { + fprintf(stderr, "Type propagation across `{self.name}` shouldn't be handled statically!\n"); + Py_UNREACHABLE(); + break; + } + + TARGET(BB_TEST_IF_TRUE_OR_POP) { + fprintf(stderr, "Type propagation across `{self.name}` shouldn't be handled statically!\n"); + Py_UNREACHABLE(); + break; + } + + TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + break; + } + + TARGET(GET_LEN) { + PyTypeObject *obj = TYPESTACK_PEEK(1); + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(MATCH_CLASS) { + PyTypeObject *names = TYPESTACK_PEEK(1); + PyTypeObject *type = TYPESTACK_PEEK(2); + PyTypeObject *subject = TYPESTACK_PEEK(3); + STACK_SHRINK(2); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(MATCH_MAPPING) { + PyTypeObject *subject = TYPESTACK_PEEK(1); + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(MATCH_SEQUENCE) { + PyTypeObject *subject = TYPESTACK_PEEK(1); + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(MATCH_KEYS) { + PyTypeObject *keys = TYPESTACK_PEEK(1); + PyTypeObject *subject = TYPESTACK_PEEK(2); + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(GET_ITER) { + PyTypeObject *iterable = TYPESTACK_PEEK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(GET_YIELD_FROM_ITER) { + PyTypeObject *iterable = TYPESTACK_PEEK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(FOR_ITER) { + PyTypeObject *iter = TYPESTACK_PEEK(1); + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(BB_TEST_ITER) { + PyTypeObject *iter = TYPESTACK_PEEK(1); + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(FOR_ITER_LIST) { + PyTypeObject *iter = TYPESTACK_PEEK(1); + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(FOR_ITER_TUPLE) { + PyTypeObject *iter = TYPESTACK_PEEK(1); + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(FOR_ITER_RANGE) { + PyTypeObject *iter = TYPESTACK_PEEK(1); + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(FOR_ITER_GEN) { + PyTypeObject *iter = TYPESTACK_PEEK(1); + STACK_GROW(1); + break; + } + + TARGET(BEFORE_ASYNC_WITH) { + PyTypeObject *mgr = TYPESTACK_PEEK(1); + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + TYPESTACK_POKE(2, NULL); + break; + } + + TARGET(BEFORE_WITH) { + PyTypeObject *mgr = TYPESTACK_PEEK(1); + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + TYPESTACK_POKE(2, NULL); + break; + } + + TARGET(WITH_EXCEPT_START) { + PyTypeObject *val = TYPESTACK_PEEK(1); + PyTypeObject *lasti = TYPESTACK_PEEK(3); + PyTypeObject *exit_func = TYPESTACK_PEEK(4); + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(PUSH_EXC_INFO) { + PyTypeObject *new_exc = TYPESTACK_PEEK(1); + STACK_GROW(1); + TYPESTACK_POKE(1, new_exc); + TYPESTACK_POKE(2, NULL); + break; + } + + TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + PyTypeObject *self = TYPESTACK_PEEK(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + TYPESTACK_POKE(1, NULL); + if (oparg & 1) { TYPESTACK_POKE(1 + ((oparg & 1) ? 1 : 0), NULL); } + break; + } + + TARGET(LOAD_ATTR_METHOD_NO_DICT) { + PyTypeObject *self = TYPESTACK_PEEK(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + TYPESTACK_POKE(1, NULL); + if (oparg & 1) { TYPESTACK_POKE(1 + ((oparg & 1) ? 1 : 0), NULL); } + break; + } + + TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + PyTypeObject *self = TYPESTACK_PEEK(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + TYPESTACK_POKE(1, NULL); + if (oparg & 1) { TYPESTACK_POKE(1 + ((oparg & 1) ? 1 : 0), NULL); } + break; + } + + TARGET(KW_NAMES) { + break; + } + + TARGET(CALL) { + PyTypeObject **args = &TYPESTACK_PEEK(oparg); + PyTypeObject *callable = TYPESTACK_PEEK(1 + oparg); + PyTypeObject *method = TYPESTACK_PEEK(2 + oparg); + STACK_SHRINK(oparg); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + PyTypeObject *callable = TYPESTACK_PEEK(1 + oparg); + PyTypeObject *method = TYPESTACK_PEEK(2 + oparg); + STACK_SHRINK(oparg); + STACK_SHRINK(1); + break; + } + + TARGET(CALL_PY_EXACT_ARGS) { + PyTypeObject **args = &TYPESTACK_PEEK(oparg); + PyTypeObject *callable = TYPESTACK_PEEK(1 + oparg); + PyTypeObject *method = TYPESTACK_PEEK(2 + oparg); + STACK_SHRINK(oparg); + STACK_SHRINK(1); + break; + } + + TARGET(CALL_PY_WITH_DEFAULTS) { + PyTypeObject **args = &TYPESTACK_PEEK(oparg); + PyTypeObject *callable = TYPESTACK_PEEK(1 + oparg); + PyTypeObject *method = TYPESTACK_PEEK(2 + oparg); + STACK_SHRINK(oparg); + STACK_SHRINK(1); + break; + } + + TARGET(CALL_NO_KW_TYPE_1) { + PyTypeObject **args = &TYPESTACK_PEEK(oparg); + PyTypeObject *callable = TYPESTACK_PEEK(1 + oparg); + PyTypeObject *null = TYPESTACK_PEEK(2 + oparg); + STACK_SHRINK(oparg); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(CALL_NO_KW_STR_1) { + PyTypeObject **args = &TYPESTACK_PEEK(oparg); + PyTypeObject *callable = TYPESTACK_PEEK(1 + oparg); + PyTypeObject *null = TYPESTACK_PEEK(2 + oparg); + STACK_SHRINK(oparg); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(CALL_NO_KW_TUPLE_1) { + PyTypeObject **args = &TYPESTACK_PEEK(oparg); + PyTypeObject *callable = TYPESTACK_PEEK(1 + oparg); + PyTypeObject *null = TYPESTACK_PEEK(2 + oparg); + STACK_SHRINK(oparg); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(CALL_BUILTIN_CLASS) { + PyTypeObject **args = &TYPESTACK_PEEK(oparg); + PyTypeObject *callable = TYPESTACK_PEEK(1 + oparg); + PyTypeObject *method = TYPESTACK_PEEK(2 + oparg); + STACK_SHRINK(oparg); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(CALL_NO_KW_BUILTIN_O) { + PyTypeObject **args = &TYPESTACK_PEEK(oparg); + PyTypeObject *callable = TYPESTACK_PEEK(1 + oparg); + PyTypeObject *method = TYPESTACK_PEEK(2 + oparg); + STACK_SHRINK(oparg); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(CALL_NO_KW_BUILTIN_FAST) { + PyTypeObject **args = &TYPESTACK_PEEK(oparg); + PyTypeObject *callable = TYPESTACK_PEEK(1 + oparg); + PyTypeObject *method = TYPESTACK_PEEK(2 + oparg); + STACK_SHRINK(oparg); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + PyTypeObject **args = &TYPESTACK_PEEK(oparg); + PyTypeObject *callable = TYPESTACK_PEEK(1 + oparg); + PyTypeObject *method = TYPESTACK_PEEK(2 + oparg); + STACK_SHRINK(oparg); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(CALL_NO_KW_LEN) { + PyTypeObject **args = &TYPESTACK_PEEK(oparg); + PyTypeObject *callable = TYPESTACK_PEEK(1 + oparg); + PyTypeObject *method = TYPESTACK_PEEK(2 + oparg); + STACK_SHRINK(oparg); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(CALL_NO_KW_ISINSTANCE) { + PyTypeObject **args = &TYPESTACK_PEEK(oparg); + PyTypeObject *callable = TYPESTACK_PEEK(1 + oparg); + PyTypeObject *method = TYPESTACK_PEEK(2 + oparg); + STACK_SHRINK(oparg); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(CALL_NO_KW_LIST_APPEND) { + PyTypeObject **args = &TYPESTACK_PEEK(oparg); + PyTypeObject *self = TYPESTACK_PEEK(1 + oparg); + PyTypeObject *method = TYPESTACK_PEEK(2 + oparg); + STACK_SHRINK(oparg); + STACK_SHRINK(1); + break; + } + + TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { + PyTypeObject **args = &TYPESTACK_PEEK(oparg); + PyTypeObject *method = TYPESTACK_PEEK(2 + oparg); + STACK_SHRINK(oparg); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { + PyTypeObject **args = &TYPESTACK_PEEK(oparg); + PyTypeObject *method = TYPESTACK_PEEK(2 + oparg); + STACK_SHRINK(oparg); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) { + PyTypeObject **args = &TYPESTACK_PEEK(oparg); + PyTypeObject *method = TYPESTACK_PEEK(2 + oparg); + STACK_SHRINK(oparg); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) { + PyTypeObject **args = &TYPESTACK_PEEK(oparg); + PyTypeObject *method = TYPESTACK_PEEK(2 + oparg); + STACK_SHRINK(oparg); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(CALL_FUNCTION_EX) { + PyTypeObject *kwargs = (oparg & 1) ? TYPESTACK_PEEK(((oparg & 1) ? 1 : 0)) : NULL; + PyTypeObject *callargs = TYPESTACK_PEEK(1 + ((oparg & 1) ? 1 : 0)); + PyTypeObject *func = TYPESTACK_PEEK(2 + ((oparg & 1) ? 1 : 0)); + STACK_SHRINK(((oparg & 1) ? 1 : 0)); + STACK_SHRINK(2); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(MAKE_FUNCTION) { + PyTypeObject *codeobj = TYPESTACK_PEEK(1); + PyTypeObject *closure = (oparg & 0x08) ? TYPESTACK_PEEK(1 + ((oparg & 0x08) ? 1 : 0)) : NULL; + PyTypeObject *annotations = (oparg & 0x04) ? TYPESTACK_PEEK(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0)) : NULL; + PyTypeObject *kwdefaults = (oparg & 0x02) ? TYPESTACK_PEEK(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0)) : NULL; + PyTypeObject *defaults = (oparg & 0x01) ? TYPESTACK_PEEK(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0)) : NULL; + STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(RETURN_GENERATOR) { + break; + } + + TARGET(BUILD_SLICE) { + PyTypeObject *step = (oparg == 3) ? TYPESTACK_PEEK(((oparg == 3) ? 1 : 0)) : NULL; + PyTypeObject *stop = TYPESTACK_PEEK(1 + ((oparg == 3) ? 1 : 0)); + PyTypeObject *start = TYPESTACK_PEEK(2 + ((oparg == 3) ? 1 : 0)); + STACK_SHRINK(((oparg == 3) ? 1 : 0)); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(FORMAT_VALUE) { + PyTypeObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? TYPESTACK_PEEK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)) : NULL; + PyTypeObject *value = TYPESTACK_PEEK(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); + STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(COPY) { + PyTypeObject *bottom = TYPESTACK_PEEK(1 + (oparg-1)); + STACK_GROW(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(BINARY_OP) { + PyTypeObject *rhs = TYPESTACK_PEEK(1); + PyTypeObject *lhs = TYPESTACK_PEEK(2); + STACK_SHRINK(1); + TYPESTACK_POKE(1, NULL); + break; + } + + TARGET(SWAP) { + PyTypeObject *top = TYPESTACK_PEEK(1); + PyTypeObject *bottom = TYPESTACK_PEEK(2 + (oparg-2)); + TYPESTACK_POKE(1, bottom); + TYPESTACK_POKE(2 + (oparg-2), top); + break; + } + + TARGET(EXTENDED_ARG) { + break; + } + + TARGET(CACHE) { + break; + } + + TARGET(BB_BRANCH) { + break; + } + + TARGET(BB_BRANCH_IF_FLAG_UNSET) { + break; + } + + TARGET(BB_JUMP_IF_FLAG_UNSET) { + break; + } + + TARGET(BB_BRANCH_IF_FLAG_SET) { + break; + } + + TARGET(BB_JUMP_IF_FLAG_SET) { + break; + } + + TARGET(BB_JUMP_BACKWARD_LAZY) { + break; + } diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 3dbb57236312c0..3c37a47d21c2cf 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -16,7 +16,8 @@ from enum import Enum, auto import parser -from parser import StackEffect +from parser import StackEffect, StackVarTypeLiteral, StackVarTypeIndex +from parser import LocalEffect, LocalEffectVarLiteral, LocalEffectVarStack HERE = os.path.dirname(__file__) ROOT = os.path.join(HERE, "../..") @@ -35,6 +36,11 @@ os.path.join(ROOT, "Include/internal/pycore_opcode_macro_to_micro.h") ) +# Tier 2 type propagator +TIER2_TYPE_PROPAGATOR_OUTPUT = os.path.relpath( + os.path.join(ROOT, "Python/tier2_typepropagator.c.h") +) + BEGIN_MARKER = "// BEGIN BYTECODES //" END_MARKER = "// END BYTECODES //" RE_PREDICTED = ( @@ -43,6 +49,17 @@ UNUSED = "unused" BITS_PER_CODE_UNIT = 16 +# Type propagation across these instructions are forbidden +# due to conditional effects that can't be determined statically +# The handling of type propagation across these opcodes are handled elsewhere +# within tier2. +TYPE_PROPAGATOR_FORBIDDEN = [ + "JUMP_IF_FALSE_OR_POP", + "JUMP_IF_TRUE_OR_POP", # Type propagator shouldn't see these + "BB_TEST_IF_FALSE_OR_POP", + "BB_TEST_IF_TRUE_OR_POP" # Type propagator handles this in BB_BRANCH +] + arg_parser = argparse.ArgumentParser( description="Generate the code for the interpreter switch.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, @@ -229,6 +246,7 @@ class Instruction: cache_effects: list[parser.CacheEffect] input_effects: list[StackEffect] output_effects: list[StackEffect] + local_effects: LocalEffect | None unmoved_names: frozenset[str] instr_fmt: str @@ -257,6 +275,7 @@ def __init__(self, inst: parser.InstDef): effect for effect in inst.inputs if isinstance(effect, StackEffect) ] self.output_effects = inst.outputs # For consistency/completeness + self.local_effects = inst.localeffect unmoved_names: set[str] = set() for ieffect, oeffect in zip(self.input_effects, self.output_effects): if ieffect.name == oeffect.name: @@ -294,6 +313,100 @@ def analyze_registers(self, a: "Analyzer") -> None: f"Instruction {self.name} has too many register effects", node=self.inst ) + def write_typeprop(self, out: Formatter) -> None: + """Write one instruction's type propagation rules""" + + if self.name in TYPE_PROPAGATOR_FORBIDDEN: + out.emit('fprintf(stderr, "Type propagation across `{self.name}` shouldn\'t be handled statically!\\n");') + out.emit("Py_UNREACHABLE();") + return + + # Write input stack effect variable declarations and initializations + ieffects = list(reversed(self.input_effects)) + usable_for_local_effect = {} + all_input_effect_names = {} + for i, ieffect in enumerate(ieffects): + isize = string_effect_size( + list_effect_size([ieff for ieff in ieffects[: i + 1]]) + ) + all_input_effect_names[ieffect.name] = (ieffect, i) + dst = StackEffect(ieffect.name, "PyTypeObject *") + if ieffect.size: + src = StackEffect(f"&TYPESTACK_PEEK({isize})", "PyTypeObject **") + dst = StackEffect(ieffect.name, "PyTypeObject **") + elif ieffect.cond: + src = StackEffect(f"({ieffect.cond}) ? TYPESTACK_PEEK({isize}) : NULL", "PyTypeObject *") + else: + usable_for_local_effect[ieffect.name] = ieffect + src = StackEffect(f"TYPESTACK_PEEK({isize})", "PyTypeObject *") + out.declare(dst, src) + + # Write localarr effect + if self.local_effects: + idx = self.local_effects.index + val = self.local_effects.value + match val: + case LocalEffectVarLiteral(name=valstr): + if valstr != "NULL": valstr = f"&{valstr}" + case LocalEffectVarStack(name=valstr): + assert valstr in usable_for_local_effect, \ + "`cond` and `size` stackvar not supported for localeffect" + case _: + typing.assert_never(val) + out.emit(f"TYPELOCALS_SET({idx}, {valstr})") + + # Update stack size + out.stack_adjust( + 0, + [ieff for ieff in self.input_effects], + [oeff for oeff in self.output_effects], + ) + + # Stack effect + oeffects = list(reversed(self.output_effects)) + for i, oeffect in enumerate(oeffects): + osize = string_effect_size( + list_effect_size([oeff for oeff in oeffects[: i + 1]]) + ) + + # Check if it's even used + if oeffect.name == UNUSED: continue + + # Check if there's type info + if typ := oeffect.type_annotation: + match typ: + case StackVarTypeLiteral(literal=val): + if val != "NULL": val = f"&{val}" + case StackVarTypeIndex(array=arr, index=idx): + val = f"{'TYPELOCALS_GET' if arr == 'locals' else 'TYPECONST_GET'}({idx})" + case _: + typing.assert_never(typ) + if oeffect.cond: + out.emit(f"if ({oeffect.cond}) {{ TYPESTACK_POKE({osize}, {val}); }}") + else: + out.emit(f"TYPESTACK_POKE({osize}, {val});") + continue + + # Check if it's part of input effect + # TODO: Can we assume that they have the same type? + if oeffect.name in all_input_effect_names: + ieffect, j = all_input_effect_names[oeffect.name] + assert not ieffect.cond, \ + "`cond` stackvar not supported for type prop" + # The stack var stays at the same pos + if len(oeffects) - i == len(ieffects) - j: continue + if oeffect.cond: + out.emit(f"if ({oeffect.cond}) {{ TYPESTACK_POKE({osize}, {oeffect.name}); }}") + else: + out.emit(f"TYPESTACK_POKE({osize}, {oeffect.name});") + continue + + # Just output null + if oeffect.cond: + out.emit(f"if ({oeffect.cond}) {{ TYPESTACK_POKE({osize}, NULL); }}") + else: + out.emit(f"TYPESTACK_POKE({osize}, NULL);") + def write(self, out: Formatter) -> None: """Write one instruction, sans prologue and epilogue.""" # Write a static assertion that a family's cache size is correct @@ -483,6 +596,10 @@ def write_body(self, out: Formatter, cache_adjust: int) -> None: for var, oeffect in self.output_mapping: out.assign(var, oeffect) + def write_typeprop(self, out: Formatter) -> None: + with out.block(""): + self.instr.write_typeprop(out) + @dataclasses.dataclass class SuperOrMacroInstruction: @@ -1013,11 +1130,38 @@ def write_uopguard_typedata(self): if types: max_types = max(max_types, len(types)) uop_to_type_output[instr_def.name] = types + if max_types > 0: + self.out.emit(f"extern const PyTypeObject *_Py_UOpGuardTypes[][{max_types}] = {{") + for name, types in uop_to_type_output.items(): + self.out.emit(f"[{name}] = {{{', '.join(['&' + type_ for type_ in types])}}},") + self.out.emit("};") - self.out.emit(f"extern const PyTypeObject *_Py_UOpGuardTypes[][{max_types}] = {{") - for name, types in uop_to_type_output.items(): - self.out.emit(f"[{name}] = {{{', '.join(['&' + type_ for type_ in types])}}},") - self.out.emit("};") + + def write_typepropagator(self) -> None: + """Write the type propagator""" + + with open(self.output_filename, "w") as f: + # Write provenance header + f.write(f"// This file is generated by {THIS} @TODO: make this a seperate argument\n") + f.write(f"// from {os.path.relpath(self.filename, ROOT).replace(os.path.sep, posixpath.sep)}\n") + f.write(f"// Do not edit!\n") + + # Create formatter + self.out = Formatter(f, 8) + + for thing in self.everything: + match thing: + case parser.InstDef(kind=kind, name=name): + match kind: + case "op": pass + case _: + self.write_instr_typeprop(self.instrs[name]) + case parser.Super(name=name): + self.write_super_typeprop(self.super_instrs[name]) + case parser.Macro(name=name): + self.write_macro_typeprop(self.macro_instrs[name]) + case _: + typing.assert_never(thing) def write_metadata(self) -> None: @@ -1237,6 +1381,31 @@ def write_macro(self, mac: MacroInstruction) -> None: f'{cache_adjust}, "incorrect cache size");' ) + def write_instr_typeprop(self, instr: Instruction) -> None: + name = instr.name + self.out.emit("") + with self.out.block(f"TARGET({name})"): + instr.write_typeprop(self.out) + self.out.emit("break;") + + def write_super_typeprop(self, sup: SuperInstruction) -> None: + # TODO: Support super instructions + # Currently not support because of the need for NEXTOPARG + ... + + def write_macro_typeprop(self, mac: MacroInstruction) -> None: + # TODO: Make the code emitted more efficient by + # combining stack effect + name = mac.name + self.out.emit("") + with self.out.block(f"TARGET({name})"): + for comp in mac.parts: + if not isinstance(comp, Component): continue + comp.write_typeprop(self.out) + self.out.emit("break;") + + + @contextlib.contextmanager def wrap_super_or_macro(self, up: SuperOrMacroInstruction): """Shared boilerplate for super- and macro instructions.""" @@ -1342,6 +1511,10 @@ def main(): # a.output_filename = TIER2_MACRO_TO_MICRO_MAP_OUTPUT # a.write_macromap_and_typedata() + # Quick hack. @TODO refactor + a.output_filename = TIER2_TYPE_PROPAGATOR_OUTPUT + a.write_typepropagator() + if __name__ == "__main__": main() diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index 4381abe5579462..1e84119854edf2 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -68,10 +68,25 @@ class Block(Node): pass +@dataclass +class StackVarTypeLiteral(Node): + literal: str + + +@dataclass +class StackVarTypeIndex(Node): + array: Literal["locals", "consts"] + index: str + + +StackVarType: TypeAlias = StackVarTypeLiteral | StackVarTypeIndex + + @dataclass class StackEffect(Node): name: str type: str = "" # Optional `:type` + type_annotation: StackVarType | None = None # Default is None cond: str = "" # Optional `if (cond)` size: str = "" # Optional `[size]` # Note: size cannot be combined with type or cond @@ -88,6 +103,25 @@ class CacheEffect(Node): size: int +@dataclass +class LocalEffectVarLiteral(Node): + name: str + + +@dataclass +class LocalEffectVarStack(Node): + name: str + + +LocalEffectVar: TypeAlias = LocalEffectVarLiteral | LocalEffectVarStack + + +@dataclass +class LocalEffect(Node): + index: str + value: LocalEffectVar + + @dataclass class OpName(Node): name: str @@ -125,6 +159,7 @@ class InstHeader(Node): name: str inputs: list[InputEffect] outputs: list[OutputEffect] + localeffect: LocalEffect | None = None @dataclass @@ -136,6 +171,7 @@ class InstDef(Node): outputs: list[OutputEffect] block: Block u_insts: list[str] + localeffect: LocalEffect | None = None @dataclass @@ -179,7 +215,7 @@ def inst_def(self) -> InstDef | None: space, label = m.groups() u_insts.append(label) return InstDef( - hdr.register, hdr.kind, hdr.name, hdr.inputs, hdr.outputs, block, u_insts + hdr.register, hdr.kind, hdr.name, hdr.inputs, hdr.outputs, block, u_insts, hdr.localeffect ) raise self.make_syntax_error("Expected block") return None @@ -199,6 +235,11 @@ def inst_header(self) -> InstHeader | None: if self.expect(lx.RPAREN): if (tkn := self.peek()) and tkn.kind == lx.LBRACE: return InstHeader(register, kind, name, inp, outp) + elif self.expect(lx.COMMA): + leffect = self.local_effect() + if self.expect(lx.RPAREN): + if (tkn := self.peek()) and tkn.kind == lx.LBRACE: + return InstHeader(register, kind, name, inp, outp, leffect) elif self.expect(lx.RPAREN) and kind == "inst": # No legacy stack effect if kind is "op". return InstHeader(register, "legacy", name, [], []) @@ -266,9 +307,12 @@ def stack_effect(self) -> StackEffect | None: # IDENTIFIER [':' IDENTIFIER] ['if' '(' expression ')'] # | IDENTIFIER '[' expression ']' if tkn := self.expect(lx.IDENTIFIER): - type_text = "" + _type = "" + has_type_annotation = False + type_annotation = None if self.expect(lx.COLON): - type_text = self.require(lx.IDENTIFIER).text.strip() + has_type_annotation = True + type_annotation = self.stackvar_type() cond_text = "" if self.expect(lx.IF): self.require(lx.LPAREN) @@ -278,14 +322,48 @@ def stack_effect(self) -> StackEffect | None: cond_text = cond.text.strip() size_text = "" if self.expect(lx.LBRACKET): - if type_text or cond_text: + if has_type_annotation or cond_text: raise self.make_syntax_error("Unexpected [") if not (size := self.expression()): raise self.make_syntax_error("Expected expression") self.require(lx.RBRACKET) - type_text = "PyObject **" + _type = "PyObject **" size_text = size.text.strip() - return StackEffect(tkn.text, type_text, cond_text, size_text) + return StackEffect(tkn.text, _type, type_annotation, cond_text, size_text) + + @contextual + def stackvar_type(self) -> StackVarType | None: + if id := self.expect(lx.IDENTIFIER): + idstr = id.text.strip() + if not self.expect(lx.LBRACKET): + return StackVarTypeLiteral(idstr) + if idstr not in ["locals", "consts"]: return + if id := self.expect(lx.IDENTIFIER): + index = id.text.strip() + self.require(lx.RBRACKET) + return StackVarTypeIndex( + "locals" if idstr == "locals" else "consts", + index) + + @contextual + def local_effect(self) -> LocalEffect | None: + if self.expect(lx.IDENTIFIER).text.strip() == "locals": + self.require(lx.LBRACKET) + if id := self.expect(lx.IDENTIFIER): + index = id.text.strip() + self.require(lx.RBRACKET) + self.require(lx.EQUALS) + if self.expect(lx.TIMES): # stackvar + value = self.require(lx.IDENTIFIER).text.strip() + return LocalEffect( + index, + LocalEffectVarStack(value) + ) + value = self.require(lx.IDENTIFIER).text.strip() + return LocalEffect( + index, + LocalEffectVarLiteral(value) + ) @contextual def expression(self) -> Expression | None: