From b0f7ad68b95182aace1b7c9c5018b8af87f8c835 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 25 Apr 2024 16:26:51 +0100 Subject: [PATCH 1/3] Allow RESUME_CHECK in tier 2 --- Include/internal/pycore_uop_ids.h | 204 +++++++++++++------------ Include/internal/pycore_uop_metadata.h | 8 + Python/bytecodes.c | 23 +++ Python/executor_cases.c.h | 28 ++++ Python/optimizer.c | 19 ++- Python/optimizer_cases.c.h | 8 + 6 files changed, 188 insertions(+), 102 deletions(-) diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index bb49d6e77d2562..0f57ef9e792047 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -93,46 +93,47 @@ extern "C" { #define _DICT_UPDATE DICT_UPDATE #define _END_SEND END_SEND #define _ERROR_POP_N 343 +#define _EVAL_BREAKER_EXIT 344 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _FATAL_ERROR 344 +#define _FATAL_ERROR 345 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 345 +#define _FOR_ITER 346 #define _FOR_ITER_GEN FOR_ITER_GEN -#define _FOR_ITER_TIER_TWO 346 +#define _FOR_ITER_TIER_TWO 347 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 347 -#define _GUARD_BOTH_INT 348 -#define _GUARD_BOTH_UNICODE 349 -#define _GUARD_BUILTINS_VERSION 350 -#define _GUARD_DORV_NO_DICT 351 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 352 -#define _GUARD_GLOBALS_VERSION 353 -#define _GUARD_IS_FALSE_POP 354 -#define _GUARD_IS_NONE_POP 355 -#define _GUARD_IS_NOT_NONE_POP 356 -#define _GUARD_IS_TRUE_POP 357 -#define _GUARD_KEYS_VERSION 358 -#define _GUARD_NOS_FLOAT 359 -#define _GUARD_NOS_INT 360 -#define _GUARD_NOT_EXHAUSTED_LIST 361 -#define _GUARD_NOT_EXHAUSTED_RANGE 362 -#define _GUARD_NOT_EXHAUSTED_TUPLE 363 -#define _GUARD_TOS_FLOAT 364 -#define _GUARD_TOS_INT 365 -#define _GUARD_TYPE_VERSION 366 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 367 -#define _INIT_CALL_PY_EXACT_ARGS 368 -#define _INIT_CALL_PY_EXACT_ARGS_0 369 -#define _INIT_CALL_PY_EXACT_ARGS_1 370 -#define _INIT_CALL_PY_EXACT_ARGS_2 371 -#define _INIT_CALL_PY_EXACT_ARGS_3 372 -#define _INIT_CALL_PY_EXACT_ARGS_4 373 +#define _GUARD_BOTH_FLOAT 348 +#define _GUARD_BOTH_INT 349 +#define _GUARD_BOTH_UNICODE 350 +#define _GUARD_BUILTINS_VERSION 351 +#define _GUARD_DORV_NO_DICT 352 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 353 +#define _GUARD_GLOBALS_VERSION 354 +#define _GUARD_IS_FALSE_POP 355 +#define _GUARD_IS_NONE_POP 356 +#define _GUARD_IS_NOT_NONE_POP 357 +#define _GUARD_IS_TRUE_POP 358 +#define _GUARD_KEYS_VERSION 359 +#define _GUARD_NOS_FLOAT 360 +#define _GUARD_NOS_INT 361 +#define _GUARD_NOT_EXHAUSTED_LIST 362 +#define _GUARD_NOT_EXHAUSTED_RANGE 363 +#define _GUARD_NOT_EXHAUSTED_TUPLE 364 +#define _GUARD_TOS_FLOAT 365 +#define _GUARD_TOS_INT 366 +#define _GUARD_TYPE_VERSION 367 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 368 +#define _INIT_CALL_PY_EXACT_ARGS 369 +#define _INIT_CALL_PY_EXACT_ARGS_0 370 +#define _INIT_CALL_PY_EXACT_ARGS_1 371 +#define _INIT_CALL_PY_EXACT_ARGS_2 372 +#define _INIT_CALL_PY_EXACT_ARGS_3 373 +#define _INIT_CALL_PY_EXACT_ARGS_4 374 #define _INSTRUMENTED_CALL INSTRUMENTED_CALL #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW @@ -149,65 +150,65 @@ extern "C" { #define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST #define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE #define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 374 -#define _IS_NONE 375 +#define _INTERNAL_INCREMENT_OPT_COUNTER 375 +#define _IS_NONE 376 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 376 -#define _ITER_CHECK_RANGE 377 -#define _ITER_CHECK_TUPLE 378 -#define _ITER_JUMP_LIST 379 -#define _ITER_JUMP_RANGE 380 -#define _ITER_JUMP_TUPLE 381 -#define _ITER_NEXT_LIST 382 -#define _ITER_NEXT_RANGE 383 -#define _ITER_NEXT_TUPLE 384 -#define _JUMP_TO_TOP 385 +#define _ITER_CHECK_LIST 377 +#define _ITER_CHECK_RANGE 378 +#define _ITER_CHECK_TUPLE 379 +#define _ITER_JUMP_LIST 380 +#define _ITER_JUMP_RANGE 381 +#define _ITER_JUMP_TUPLE 382 +#define _ITER_NEXT_LIST 383 +#define _ITER_NEXT_RANGE 384 +#define _ITER_NEXT_TUPLE 385 +#define _JUMP_TO_TOP 386 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND #define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR -#define _LOAD_ATTR 386 -#define _LOAD_ATTR_CLASS 387 -#define _LOAD_ATTR_CLASS_0 388 -#define _LOAD_ATTR_CLASS_1 389 +#define _LOAD_ATTR 387 +#define _LOAD_ATTR_CLASS 388 +#define _LOAD_ATTR_CLASS_0 389 +#define _LOAD_ATTR_CLASS_1 390 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 390 -#define _LOAD_ATTR_INSTANCE_VALUE_0 391 -#define _LOAD_ATTR_INSTANCE_VALUE_1 392 -#define _LOAD_ATTR_METHOD_LAZY_DICT 393 -#define _LOAD_ATTR_METHOD_NO_DICT 394 -#define _LOAD_ATTR_METHOD_WITH_VALUES 395 -#define _LOAD_ATTR_MODULE 396 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 397 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 398 +#define _LOAD_ATTR_INSTANCE_VALUE 391 +#define _LOAD_ATTR_INSTANCE_VALUE_0 392 +#define _LOAD_ATTR_INSTANCE_VALUE_1 393 +#define _LOAD_ATTR_METHOD_LAZY_DICT 394 +#define _LOAD_ATTR_METHOD_NO_DICT 395 +#define _LOAD_ATTR_METHOD_WITH_VALUES 396 +#define _LOAD_ATTR_MODULE 397 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 398 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 399 #define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY -#define _LOAD_ATTR_SLOT 399 -#define _LOAD_ATTR_SLOT_0 400 -#define _LOAD_ATTR_SLOT_1 401 -#define _LOAD_ATTR_WITH_HINT 402 +#define _LOAD_ATTR_SLOT 400 +#define _LOAD_ATTR_SLOT_0 401 +#define _LOAD_ATTR_SLOT_1 402 +#define _LOAD_ATTR_WITH_HINT 403 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 403 -#define _LOAD_CONST_INLINE_BORROW 404 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 405 -#define _LOAD_CONST_INLINE_WITH_NULL 406 +#define _LOAD_CONST_INLINE 404 +#define _LOAD_CONST_INLINE_BORROW 405 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 406 +#define _LOAD_CONST_INLINE_WITH_NULL 407 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 407 -#define _LOAD_FAST_0 408 -#define _LOAD_FAST_1 409 -#define _LOAD_FAST_2 410 -#define _LOAD_FAST_3 411 -#define _LOAD_FAST_4 412 -#define _LOAD_FAST_5 413 -#define _LOAD_FAST_6 414 -#define _LOAD_FAST_7 415 +#define _LOAD_FAST 408 +#define _LOAD_FAST_0 409 +#define _LOAD_FAST_1 410 +#define _LOAD_FAST_2 411 +#define _LOAD_FAST_3 412 +#define _LOAD_FAST_4 413 +#define _LOAD_FAST_5 414 +#define _LOAD_FAST_6 415 +#define _LOAD_FAST_7 416 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 416 -#define _LOAD_GLOBAL_BUILTINS 417 -#define _LOAD_GLOBAL_MODULE 418 +#define _LOAD_GLOBAL 417 +#define _LOAD_GLOBAL_BUILTINS 418 +#define _LOAD_GLOBAL_MODULE 419 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR @@ -221,50 +222,51 @@ extern "C" { #define _MATCH_SEQUENCE MATCH_SEQUENCE #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_FRAME 419 -#define _POP_JUMP_IF_FALSE 420 -#define _POP_JUMP_IF_TRUE 421 +#define _POP_FRAME 420 +#define _POP_JUMP_IF_FALSE 421 +#define _POP_JUMP_IF_TRUE 422 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 422 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 423 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 423 +#define _PUSH_FRAME 424 #define _PUSH_NULL PUSH_NULL -#define _REPLACE_WITH_TRUE 424 +#define _REPLACE_WITH_TRUE 425 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR -#define _SAVE_RETURN_OFFSET 425 -#define _SEND 426 +#define _SAVE_RETURN_OFFSET 426 +#define _SEND 427 #define _SEND_GEN SEND_GEN #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _SIDE_EXIT 427 -#define _START_EXECUTOR 428 -#define _STORE_ATTR 429 -#define _STORE_ATTR_INSTANCE_VALUE 430 -#define _STORE_ATTR_SLOT 431 +#define _SIDE_EXIT 428 +#define _START_EXECUTOR 429 +#define _STORE_ATTR 430 +#define _STORE_ATTR_INSTANCE_VALUE 431 +#define _STORE_ATTR_SLOT 432 #define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 432 -#define _STORE_FAST_0 433 -#define _STORE_FAST_1 434 -#define _STORE_FAST_2 435 -#define _STORE_FAST_3 436 -#define _STORE_FAST_4 437 -#define _STORE_FAST_5 438 -#define _STORE_FAST_6 439 -#define _STORE_FAST_7 440 +#define _STORE_FAST 433 +#define _STORE_FAST_0 434 +#define _STORE_FAST_1 435 +#define _STORE_FAST_2 436 +#define _STORE_FAST_3 437 +#define _STORE_FAST_4 438 +#define _STORE_FAST_5 439 +#define _STORE_FAST_6 440 +#define _STORE_FAST_7 441 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 441 +#define _STORE_SUBSCR 442 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TO_BOOL 442 +#define _TIER2_RESUME_CHECK 443 +#define _TO_BOOL 444 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -274,12 +276,12 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 443 +#define _UNPACK_SEQUENCE 445 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define MAX_UOP_ID 443 +#define MAX_UOP_ID 445 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index b8cdfae8391460..171db6489c3b38 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -251,6 +251,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_DEOPT] = 0, [_SIDE_EXIT] = 0, [_ERROR_POP_N] = HAS_ARG_FLAG, + [_TIER2_RESUME_CHECK] = HAS_EXIT_FLAG, + [_EVAL_BREAKER_EXIT] = HAS_ESCAPES_FLAG, }; const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { @@ -333,6 +335,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_DICT_UPDATE] = "_DICT_UPDATE", [_END_SEND] = "_END_SEND", [_ERROR_POP_N] = "_ERROR_POP_N", + [_EVAL_BREAKER_EXIT] = "_EVAL_BREAKER_EXIT", [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", [_EXIT_TRACE] = "_EXIT_TRACE", [_FATAL_ERROR] = "_FATAL_ERROR", @@ -477,6 +480,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_STORE_SUBSCR_DICT] = "_STORE_SUBSCR_DICT", [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", [_SWAP] = "_SWAP", + [_TIER2_RESUME_CHECK] = "_TIER2_RESUME_CHECK", [_TO_BOOL] = "_TO_BOOL", [_TO_BOOL_BOOL] = "_TO_BOOL_BOOL", [_TO_BOOL_INT] = "_TO_BOOL_INT", @@ -960,6 +964,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _ERROR_POP_N: return oparg; + case _TIER2_RESUME_CHECK: + return 0; + case _EVAL_BREAKER_EXIT: + return 0; default: return -1; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 485504914912f9..bdd8aa5abe99ab 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4222,6 +4222,29 @@ dummy_func( GOTO_UNWIND(); } + /* Special version of RESUME_CHECK that (when paired with _EVAL_BREAKER_EXIT) + * is safe for tier 2. Progress is guaranteed because _EVAL_BREAKER_EXIT calls + * _Py_HandlePending which clears the eval_breaker so that _TIER2_RESUME_CHECK + * will not exit if it is immediately executed again. */ + tier2 op(_TIER2_RESUME_CHECK, (--)) { +#if defined(__EMSCRIPTEN__) + EXIT_IF(_Py_emscripten_signal_clock == 0); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; +#endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + EXIT_IF(eval_breaker & _PY_EVAL_EVENTS_MASK); + EXIT_IF(eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + } + + tier2 op(_EVAL_BREAKER_EXIT, (--)) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_HandlePending(tstate) != 0) { + GOTO_UNWIND(); + } + EXIT_TO_TRACE(); + } + // END BYTECODES // } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 1eb3da9b70002c..6fab023dab2917 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4236,4 +4236,32 @@ break; } + case _TIER2_RESUME_CHECK: { + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + if (eval_breaker != version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _EVAL_BREAKER_EXIT: { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_HandlePending(tstate) != 0) { + GOTO_UNWIND(); + } + EXIT_TO_TRACE(); + break; + } + #undef TIER_TWO diff --git a/Python/optimizer.c b/Python/optimizer.c index e5c70f72f9c324..ba9edbf604f901 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -690,6 +690,12 @@ translate_bytecode_to_trace( break; } + case RESUME: + /* Use a special tier 2 version of RESUME_CHECK to allow traces to + * start with RESUME_CHECK */ + ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); + break; + default: { const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; @@ -961,7 +967,18 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) int32_t target = (int32_t)uop_get_target(inst); if (_PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) { if (target != current_jump_target) { - uint16_t exit_op = (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) ? _SIDE_EXIT : _DEOPT; + uint16_t exit_op; + if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { + if (opcode == _TIER2_RESUME_CHECK) { + exit_op = _EVAL_BREAKER_EXIT; + } + else { + exit_op = _SIDE_EXIT; + } + } + else { + exit_op = _DEOPT; + } make_exit(&buffer[next_spare], exit_op, target); current_jump_target = target; current_jump = next_spare; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 4f0941a3cc3e09..c399406ea49ebd 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2134,3 +2134,11 @@ break; } + case _TIER2_RESUME_CHECK: { + break; + } + + case _EVAL_BREAKER_EXIT: { + break; + } + From 1c91a5a0bc7c8edf7def2f728234a0019baa8bee Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 26 Apr 2024 14:59:24 +0100 Subject: [PATCH 2/3] Regen files --- Python/executor_cases.c.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 6fab023dab2917..264ba3de3d836b 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4245,9 +4245,11 @@ _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - if (eval_breaker != version) { + if (eval_breaker & _PY_EVAL_EVENTS_MASK) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } From f4d7149b27616c40b3e4ade3e86fc305be56c7d2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 26 Apr 2024 15:36:12 +0100 Subject: [PATCH 3/3] Fix incorrect edit and optimizer check. --- Python/bytecodes.c | 2 +- Python/executor_cases.c.h | 5 +---- Python/optimizer.c | 1 + 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index bdd8aa5abe99ab..3467e263b5987d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4233,7 +4233,7 @@ dummy_func( #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); EXIT_IF(eval_breaker & _PY_EVAL_EVENTS_MASK); - EXIT_IF(eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + assert(eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); } tier2 op(_EVAL_BREAKER_EXIT, (--)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 264ba3de3d836b..6b815b4c84fb55 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4249,10 +4249,7 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + assert(eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); break; } diff --git a/Python/optimizer.c b/Python/optimizer.c index ba9edbf604f901..bb7d867b33b92c 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1086,6 +1086,7 @@ sanity_check(_PyExecutorObject *executor) CHECK( opcode == _DEOPT || opcode == _SIDE_EXIT || + opcode == _EVAL_BREAKER_EXIT || opcode == _ERROR_POP_N); if (opcode == _SIDE_EXIT) { CHECK(inst->format == UOP_FORMAT_EXIT);