mirror of
https://github.com/python/cpython.git
synced 2025-10-19 16:03:42 +00:00
GH-136410: Faster side exits by using a cold exit stub (GH-136411)
This commit is contained in:
Mark Shannon
GitHub
parent
718e0c89ba
commit
e7b55f564d
15 changed files with 387 additions and 267 deletions
|
|
@ -198,6 +198,9 @@ struct _ts {
|
|||
|
||||
PyObject *current_executor;
|
||||
|
||||
/* Internal to the JIT */
|
||||
struct _PyExitData *jit_exit;
|
||||
|
||||
uint64_t dict_global_version;
|
||||
|
||||
/* Used to store/retrieve `threading.local` keys/values for this thread */
|
||||
|
|
|
|||
|
|
@ -943,6 +943,7 @@ struct _is {
|
|||
bool jit;
|
||||
struct _PyExecutorObject *executor_list_head;
|
||||
struct _PyExecutorObject *executor_deletion_list_head;
|
||||
struct _PyExecutorObject *cold_executor;
|
||||
int executor_deletion_list_remaining_capacity;
|
||||
size_t trace_run_counter;
|
||||
_rare_events rare_events;
|
||||
|
|
|
|||
|
|
@ -67,8 +67,9 @@ typedef struct {
|
|||
#endif
|
||||
} _PyUOpInstruction;
|
||||
|
||||
typedef struct {
|
||||
typedef struct _PyExitData {
|
||||
uint32_t target;
|
||||
uint16_t index;
|
||||
_Py_BackoffCounter temperature;
|
||||
struct _PyExecutorObject *executor;
|
||||
} _PyExitData;
|
||||
|
|
@ -354,6 +355,16 @@ PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored);
|
|||
|
||||
PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyExecutorObject **exec_ptr, int chain_depth);
|
||||
|
||||
static inline _PyExecutorObject *_PyExecutor_FromExit(_PyExitData *exit)
|
||||
{
|
||||
_PyExitData *exit0 = exit - exit->index;
|
||||
return (_PyExecutorObject *)(((char *)exit0) - offsetof(_PyExecutorObject, exits));
|
||||
}
|
||||
|
||||
extern _PyExecutorObject *_PyExecutor_GetColdExecutor(void);
|
||||
|
||||
PyAPI_FUNC(void) _PyExecutor_ClearExit(_PyExitData *exit);
|
||||
|
||||
static inline int is_terminator(const _PyUOpInstruction *uop)
|
||||
{
|
||||
int opcode = uop->opcode;
|
||||
|
|
@ -363,6 +374,8 @@ static inline int is_terminator(const _PyUOpInstruction *uop)
|
|||
);
|
||||
}
|
||||
|
||||
extern void _PyExecutor_Free(_PyExecutorObject *self);
|
||||
|
||||
PyAPI_FUNC(int) _PyDumpExecutors(FILE *out);
|
||||
#ifdef _Py_TIER2
|
||||
extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp);
|
||||
|
|
|
|||
379
Include/internal/pycore_uop_ids.h
generated
379
Include/internal/pycore_uop_ids.h
generated
|
|
@ -81,99 +81,100 @@ extern "C" {
|
|||
#define _CHECK_STACK_SPACE 357
|
||||
#define _CHECK_STACK_SPACE_OPERAND 358
|
||||
#define _CHECK_VALIDITY 359
|
||||
#define _COMPARE_OP 360
|
||||
#define _COMPARE_OP_FLOAT 361
|
||||
#define _COMPARE_OP_INT 362
|
||||
#define _COMPARE_OP_STR 363
|
||||
#define _CONTAINS_OP 364
|
||||
#define _CONTAINS_OP_DICT 365
|
||||
#define _CONTAINS_OP_SET 366
|
||||
#define _COLD_EXIT 360
|
||||
#define _COMPARE_OP 361
|
||||
#define _COMPARE_OP_FLOAT 362
|
||||
#define _COMPARE_OP_INT 363
|
||||
#define _COMPARE_OP_STR 364
|
||||
#define _CONTAINS_OP 365
|
||||
#define _CONTAINS_OP_DICT 366
|
||||
#define _CONTAINS_OP_SET 367
|
||||
#define _CONVERT_VALUE CONVERT_VALUE
|
||||
#define _COPY 367
|
||||
#define _COPY_1 368
|
||||
#define _COPY_2 369
|
||||
#define _COPY_3 370
|
||||
#define _COPY 368
|
||||
#define _COPY_1 369
|
||||
#define _COPY_2 370
|
||||
#define _COPY_3 371
|
||||
#define _COPY_FREE_VARS COPY_FREE_VARS
|
||||
#define _CREATE_INIT_FRAME 371
|
||||
#define _CREATE_INIT_FRAME 372
|
||||
#define _DELETE_ATTR DELETE_ATTR
|
||||
#define _DELETE_DEREF DELETE_DEREF
|
||||
#define _DELETE_FAST DELETE_FAST
|
||||
#define _DELETE_GLOBAL DELETE_GLOBAL
|
||||
#define _DELETE_NAME DELETE_NAME
|
||||
#define _DELETE_SUBSCR DELETE_SUBSCR
|
||||
#define _DEOPT 372
|
||||
#define _DEOPT 373
|
||||
#define _DICT_MERGE DICT_MERGE
|
||||
#define _DICT_UPDATE DICT_UPDATE
|
||||
#define _DO_CALL 373
|
||||
#define _DO_CALL_FUNCTION_EX 374
|
||||
#define _DO_CALL_KW 375
|
||||
#define _DO_CALL 374
|
||||
#define _DO_CALL_FUNCTION_EX 375
|
||||
#define _DO_CALL_KW 376
|
||||
#define _END_FOR END_FOR
|
||||
#define _END_SEND END_SEND
|
||||
#define _ERROR_POP_N 376
|
||||
#define _ERROR_POP_N 377
|
||||
#define _EXIT_INIT_CHECK EXIT_INIT_CHECK
|
||||
#define _EXPAND_METHOD 377
|
||||
#define _EXPAND_METHOD_KW 378
|
||||
#define _FATAL_ERROR 379
|
||||
#define _EXPAND_METHOD 378
|
||||
#define _EXPAND_METHOD_KW 379
|
||||
#define _FATAL_ERROR 380
|
||||
#define _FORMAT_SIMPLE FORMAT_SIMPLE
|
||||
#define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC
|
||||
#define _FOR_ITER 380
|
||||
#define _FOR_ITER_GEN_FRAME 381
|
||||
#define _FOR_ITER_TIER_TWO 382
|
||||
#define _FOR_ITER 381
|
||||
#define _FOR_ITER_GEN_FRAME 382
|
||||
#define _FOR_ITER_TIER_TWO 383
|
||||
#define _GET_AITER GET_AITER
|
||||
#define _GET_ANEXT GET_ANEXT
|
||||
#define _GET_AWAITABLE GET_AWAITABLE
|
||||
#define _GET_ITER GET_ITER
|
||||
#define _GET_LEN GET_LEN
|
||||
#define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER
|
||||
#define _GUARD_BINARY_OP_EXTEND 383
|
||||
#define _GUARD_CALLABLE_ISINSTANCE 384
|
||||
#define _GUARD_CALLABLE_LEN 385
|
||||
#define _GUARD_CALLABLE_LIST_APPEND 386
|
||||
#define _GUARD_CALLABLE_STR_1 387
|
||||
#define _GUARD_CALLABLE_TUPLE_1 388
|
||||
#define _GUARD_CALLABLE_TYPE_1 389
|
||||
#define _GUARD_DORV_NO_DICT 390
|
||||
#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 391
|
||||
#define _GUARD_GLOBALS_VERSION 392
|
||||
#define _GUARD_IS_FALSE_POP 393
|
||||
#define _GUARD_IS_NONE_POP 394
|
||||
#define _GUARD_IS_NOT_NONE_POP 395
|
||||
#define _GUARD_IS_TRUE_POP 396
|
||||
#define _GUARD_KEYS_VERSION 397
|
||||
#define _GUARD_NOS_DICT 398
|
||||
#define _GUARD_NOS_FLOAT 399
|
||||
#define _GUARD_NOS_INT 400
|
||||
#define _GUARD_NOS_LIST 401
|
||||
#define _GUARD_NOS_NOT_NULL 402
|
||||
#define _GUARD_NOS_NULL 403
|
||||
#define _GUARD_NOS_OVERFLOWED 404
|
||||
#define _GUARD_NOS_TUPLE 405
|
||||
#define _GUARD_NOS_UNICODE 406
|
||||
#define _GUARD_NOT_EXHAUSTED_LIST 407
|
||||
#define _GUARD_NOT_EXHAUSTED_RANGE 408
|
||||
#define _GUARD_NOT_EXHAUSTED_TUPLE 409
|
||||
#define _GUARD_THIRD_NULL 410
|
||||
#define _GUARD_TOS_ANY_SET 411
|
||||
#define _GUARD_TOS_DICT 412
|
||||
#define _GUARD_TOS_FLOAT 413
|
||||
#define _GUARD_TOS_INT 414
|
||||
#define _GUARD_TOS_LIST 415
|
||||
#define _GUARD_TOS_OVERFLOWED 416
|
||||
#define _GUARD_TOS_SLICE 417
|
||||
#define _GUARD_TOS_TUPLE 418
|
||||
#define _GUARD_TOS_UNICODE 419
|
||||
#define _GUARD_TYPE_VERSION 420
|
||||
#define _GUARD_TYPE_VERSION_AND_LOCK 421
|
||||
#define _GUARD_BINARY_OP_EXTEND 384
|
||||
#define _GUARD_CALLABLE_ISINSTANCE 385
|
||||
#define _GUARD_CALLABLE_LEN 386
|
||||
#define _GUARD_CALLABLE_LIST_APPEND 387
|
||||
#define _GUARD_CALLABLE_STR_1 388
|
||||
#define _GUARD_CALLABLE_TUPLE_1 389
|
||||
#define _GUARD_CALLABLE_TYPE_1 390
|
||||
#define _GUARD_DORV_NO_DICT 391
|
||||
#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 392
|
||||
#define _GUARD_GLOBALS_VERSION 393
|
||||
#define _GUARD_IS_FALSE_POP 394
|
||||
#define _GUARD_IS_NONE_POP 395
|
||||
#define _GUARD_IS_NOT_NONE_POP 396
|
||||
#define _GUARD_IS_TRUE_POP 397
|
||||
#define _GUARD_KEYS_VERSION 398
|
||||
#define _GUARD_NOS_DICT 399
|
||||
#define _GUARD_NOS_FLOAT 400
|
||||
#define _GUARD_NOS_INT 401
|
||||
#define _GUARD_NOS_LIST 402
|
||||
#define _GUARD_NOS_NOT_NULL 403
|
||||
#define _GUARD_NOS_NULL 404
|
||||
#define _GUARD_NOS_OVERFLOWED 405
|
||||
#define _GUARD_NOS_TUPLE 406
|
||||
#define _GUARD_NOS_UNICODE 407
|
||||
#define _GUARD_NOT_EXHAUSTED_LIST 408
|
||||
#define _GUARD_NOT_EXHAUSTED_RANGE 409
|
||||
#define _GUARD_NOT_EXHAUSTED_TUPLE 410
|
||||
#define _GUARD_THIRD_NULL 411
|
||||
#define _GUARD_TOS_ANY_SET 412
|
||||
#define _GUARD_TOS_DICT 413
|
||||
#define _GUARD_TOS_FLOAT 414
|
||||
#define _GUARD_TOS_INT 415
|
||||
#define _GUARD_TOS_LIST 416
|
||||
#define _GUARD_TOS_OVERFLOWED 417
|
||||
#define _GUARD_TOS_SLICE 418
|
||||
#define _GUARD_TOS_TUPLE 419
|
||||
#define _GUARD_TOS_UNICODE 420
|
||||
#define _GUARD_TYPE_VERSION 421
|
||||
#define _GUARD_TYPE_VERSION_AND_LOCK 422
|
||||
#define _IMPORT_FROM IMPORT_FROM
|
||||
#define _IMPORT_NAME IMPORT_NAME
|
||||
#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 422
|
||||
#define _INIT_CALL_PY_EXACT_ARGS 423
|
||||
#define _INIT_CALL_PY_EXACT_ARGS_0 424
|
||||
#define _INIT_CALL_PY_EXACT_ARGS_1 425
|
||||
#define _INIT_CALL_PY_EXACT_ARGS_2 426
|
||||
#define _INIT_CALL_PY_EXACT_ARGS_3 427
|
||||
#define _INIT_CALL_PY_EXACT_ARGS_4 428
|
||||
#define _INSERT_NULL 429
|
||||
#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 423
|
||||
#define _INIT_CALL_PY_EXACT_ARGS 424
|
||||
#define _INIT_CALL_PY_EXACT_ARGS_0 425
|
||||
#define _INIT_CALL_PY_EXACT_ARGS_1 426
|
||||
#define _INIT_CALL_PY_EXACT_ARGS_2 427
|
||||
#define _INIT_CALL_PY_EXACT_ARGS_3 428
|
||||
#define _INIT_CALL_PY_EXACT_ARGS_4 429
|
||||
#define _INSERT_NULL 430
|
||||
#define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER
|
||||
#define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION
|
||||
#define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD
|
||||
|
|
@ -183,177 +184,177 @@ extern "C" {
|
|||
#define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE
|
||||
#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE
|
||||
#define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE
|
||||
#define _IS_NONE 430
|
||||
#define _IS_NONE 431
|
||||
#define _IS_OP IS_OP
|
||||
#define _ITER_CHECK_LIST 431
|
||||
#define _ITER_CHECK_RANGE 432
|
||||
#define _ITER_CHECK_TUPLE 433
|
||||
#define _ITER_JUMP_LIST 434
|
||||
#define _ITER_JUMP_RANGE 435
|
||||
#define _ITER_JUMP_TUPLE 436
|
||||
#define _ITER_NEXT_LIST 437
|
||||
#define _ITER_NEXT_LIST_TIER_TWO 438
|
||||
#define _ITER_NEXT_RANGE 439
|
||||
#define _ITER_NEXT_TUPLE 440
|
||||
#define _JUMP_TO_TOP 441
|
||||
#define _ITER_CHECK_LIST 432
|
||||
#define _ITER_CHECK_RANGE 433
|
||||
#define _ITER_CHECK_TUPLE 434
|
||||
#define _ITER_JUMP_LIST 435
|
||||
#define _ITER_JUMP_RANGE 436
|
||||
#define _ITER_JUMP_TUPLE 437
|
||||
#define _ITER_NEXT_LIST 438
|
||||
#define _ITER_NEXT_LIST_TIER_TWO 439
|
||||
#define _ITER_NEXT_RANGE 440
|
||||
#define _ITER_NEXT_TUPLE 441
|
||||
#define _JUMP_TO_TOP 442
|
||||
#define _LIST_APPEND LIST_APPEND
|
||||
#define _LIST_EXTEND LIST_EXTEND
|
||||
#define _LOAD_ATTR 442
|
||||
#define _LOAD_ATTR_CLASS 443
|
||||
#define _LOAD_ATTR 443
|
||||
#define _LOAD_ATTR_CLASS 444
|
||||
#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN
|
||||
#define _LOAD_ATTR_INSTANCE_VALUE 444
|
||||
#define _LOAD_ATTR_METHOD_LAZY_DICT 445
|
||||
#define _LOAD_ATTR_METHOD_NO_DICT 446
|
||||
#define _LOAD_ATTR_METHOD_WITH_VALUES 447
|
||||
#define _LOAD_ATTR_MODULE 448
|
||||
#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 449
|
||||
#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 450
|
||||
#define _LOAD_ATTR_PROPERTY_FRAME 451
|
||||
#define _LOAD_ATTR_SLOT 452
|
||||
#define _LOAD_ATTR_WITH_HINT 453
|
||||
#define _LOAD_ATTR_INSTANCE_VALUE 445
|
||||
#define _LOAD_ATTR_METHOD_LAZY_DICT 446
|
||||
#define _LOAD_ATTR_METHOD_NO_DICT 447
|
||||
#define _LOAD_ATTR_METHOD_WITH_VALUES 448
|
||||
#define _LOAD_ATTR_MODULE 449
|
||||
#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 450
|
||||
#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 451
|
||||
#define _LOAD_ATTR_PROPERTY_FRAME 452
|
||||
#define _LOAD_ATTR_SLOT 453
|
||||
#define _LOAD_ATTR_WITH_HINT 454
|
||||
#define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS
|
||||
#define _LOAD_BYTECODE 454
|
||||
#define _LOAD_BYTECODE 455
|
||||
#define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT
|
||||
#define _LOAD_CONST LOAD_CONST
|
||||
#define _LOAD_CONST_INLINE 455
|
||||
#define _LOAD_CONST_INLINE_BORROW 456
|
||||
#define _LOAD_CONST_UNDER_INLINE 457
|
||||
#define _LOAD_CONST_UNDER_INLINE_BORROW 458
|
||||
#define _LOAD_CONST_INLINE 456
|
||||
#define _LOAD_CONST_INLINE_BORROW 457
|
||||
#define _LOAD_CONST_UNDER_INLINE 458
|
||||
#define _LOAD_CONST_UNDER_INLINE_BORROW 459
|
||||
#define _LOAD_DEREF LOAD_DEREF
|
||||
#define _LOAD_FAST 459
|
||||
#define _LOAD_FAST_0 460
|
||||
#define _LOAD_FAST_1 461
|
||||
#define _LOAD_FAST_2 462
|
||||
#define _LOAD_FAST_3 463
|
||||
#define _LOAD_FAST_4 464
|
||||
#define _LOAD_FAST_5 465
|
||||
#define _LOAD_FAST_6 466
|
||||
#define _LOAD_FAST_7 467
|
||||
#define _LOAD_FAST 460
|
||||
#define _LOAD_FAST_0 461
|
||||
#define _LOAD_FAST_1 462
|
||||
#define _LOAD_FAST_2 463
|
||||
#define _LOAD_FAST_3 464
|
||||
#define _LOAD_FAST_4 465
|
||||
#define _LOAD_FAST_5 466
|
||||
#define _LOAD_FAST_6 467
|
||||
#define _LOAD_FAST_7 468
|
||||
#define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR
|
||||
#define _LOAD_FAST_BORROW 468
|
||||
#define _LOAD_FAST_BORROW_0 469
|
||||
#define _LOAD_FAST_BORROW_1 470
|
||||
#define _LOAD_FAST_BORROW_2 471
|
||||
#define _LOAD_FAST_BORROW_3 472
|
||||
#define _LOAD_FAST_BORROW_4 473
|
||||
#define _LOAD_FAST_BORROW_5 474
|
||||
#define _LOAD_FAST_BORROW_6 475
|
||||
#define _LOAD_FAST_BORROW_7 476
|
||||
#define _LOAD_FAST_BORROW 469
|
||||
#define _LOAD_FAST_BORROW_0 470
|
||||
#define _LOAD_FAST_BORROW_1 471
|
||||
#define _LOAD_FAST_BORROW_2 472
|
||||
#define _LOAD_FAST_BORROW_3 473
|
||||
#define _LOAD_FAST_BORROW_4 474
|
||||
#define _LOAD_FAST_BORROW_5 475
|
||||
#define _LOAD_FAST_BORROW_6 476
|
||||
#define _LOAD_FAST_BORROW_7 477
|
||||
#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW
|
||||
#define _LOAD_FAST_CHECK LOAD_FAST_CHECK
|
||||
#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST
|
||||
#define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF
|
||||
#define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS
|
||||
#define _LOAD_GLOBAL 477
|
||||
#define _LOAD_GLOBAL_BUILTINS 478
|
||||
#define _LOAD_GLOBAL_MODULE 479
|
||||
#define _LOAD_GLOBAL 478
|
||||
#define _LOAD_GLOBAL_BUILTINS 479
|
||||
#define _LOAD_GLOBAL_MODULE 480
|
||||
#define _LOAD_LOCALS LOAD_LOCALS
|
||||
#define _LOAD_NAME LOAD_NAME
|
||||
#define _LOAD_SMALL_INT 480
|
||||
#define _LOAD_SMALL_INT_0 481
|
||||
#define _LOAD_SMALL_INT_1 482
|
||||
#define _LOAD_SMALL_INT_2 483
|
||||
#define _LOAD_SMALL_INT_3 484
|
||||
#define _LOAD_SPECIAL 485
|
||||
#define _LOAD_SMALL_INT 481
|
||||
#define _LOAD_SMALL_INT_0 482
|
||||
#define _LOAD_SMALL_INT_1 483
|
||||
#define _LOAD_SMALL_INT_2 484
|
||||
#define _LOAD_SMALL_INT_3 485
|
||||
#define _LOAD_SPECIAL 486
|
||||
#define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR
|
||||
#define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD
|
||||
#define _MAKE_CALLARGS_A_TUPLE 486
|
||||
#define _MAKE_CALLARGS_A_TUPLE 487
|
||||
#define _MAKE_CELL MAKE_CELL
|
||||
#define _MAKE_FUNCTION MAKE_FUNCTION
|
||||
#define _MAKE_WARM 487
|
||||
#define _MAKE_WARM 488
|
||||
#define _MAP_ADD MAP_ADD
|
||||
#define _MATCH_CLASS MATCH_CLASS
|
||||
#define _MATCH_KEYS MATCH_KEYS
|
||||
#define _MATCH_MAPPING MATCH_MAPPING
|
||||
#define _MATCH_SEQUENCE MATCH_SEQUENCE
|
||||
#define _MAYBE_EXPAND_METHOD 488
|
||||
#define _MAYBE_EXPAND_METHOD_KW 489
|
||||
#define _MONITOR_CALL 490
|
||||
#define _MONITOR_CALL_KW 491
|
||||
#define _MONITOR_JUMP_BACKWARD 492
|
||||
#define _MONITOR_RESUME 493
|
||||
#define _MAYBE_EXPAND_METHOD 489
|
||||
#define _MAYBE_EXPAND_METHOD_KW 490
|
||||
#define _MONITOR_CALL 491
|
||||
#define _MONITOR_CALL_KW 492
|
||||
#define _MONITOR_JUMP_BACKWARD 493
|
||||
#define _MONITOR_RESUME 494
|
||||
#define _NOP NOP
|
||||
#define _POP_CALL 494
|
||||
#define _POP_CALL_LOAD_CONST_INLINE_BORROW 495
|
||||
#define _POP_CALL_ONE 496
|
||||
#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 497
|
||||
#define _POP_CALL_TWO 498
|
||||
#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 499
|
||||
#define _POP_CALL 495
|
||||
#define _POP_CALL_LOAD_CONST_INLINE_BORROW 496
|
||||
#define _POP_CALL_ONE 497
|
||||
#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 498
|
||||
#define _POP_CALL_TWO 499
|
||||
#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 500
|
||||
#define _POP_EXCEPT POP_EXCEPT
|
||||
#define _POP_ITER POP_ITER
|
||||
#define _POP_JUMP_IF_FALSE 500
|
||||
#define _POP_JUMP_IF_TRUE 501
|
||||
#define _POP_JUMP_IF_FALSE 501
|
||||
#define _POP_JUMP_IF_TRUE 502
|
||||
#define _POP_TOP POP_TOP
|
||||
#define _POP_TOP_FLOAT 502
|
||||
#define _POP_TOP_INT 503
|
||||
#define _POP_TOP_LOAD_CONST_INLINE 504
|
||||
#define _POP_TOP_LOAD_CONST_INLINE_BORROW 505
|
||||
#define _POP_TOP_NOP 506
|
||||
#define _POP_TOP_UNICODE 507
|
||||
#define _POP_TWO 508
|
||||
#define _POP_TWO_LOAD_CONST_INLINE_BORROW 509
|
||||
#define _POP_TOP_FLOAT 503
|
||||
#define _POP_TOP_INT 504
|
||||
#define _POP_TOP_LOAD_CONST_INLINE 505
|
||||
#define _POP_TOP_LOAD_CONST_INLINE_BORROW 506
|
||||
#define _POP_TOP_NOP 507
|
||||
#define _POP_TOP_UNICODE 508
|
||||
#define _POP_TWO 509
|
||||
#define _POP_TWO_LOAD_CONST_INLINE_BORROW 510
|
||||
#define _PUSH_EXC_INFO PUSH_EXC_INFO
|
||||
#define _PUSH_FRAME 510
|
||||
#define _PUSH_FRAME 511
|
||||
#define _PUSH_NULL PUSH_NULL
|
||||
#define _PUSH_NULL_CONDITIONAL 511
|
||||
#define _PY_FRAME_GENERAL 512
|
||||
#define _PY_FRAME_KW 513
|
||||
#define _QUICKEN_RESUME 514
|
||||
#define _REPLACE_WITH_TRUE 515
|
||||
#define _PUSH_NULL_CONDITIONAL 512
|
||||
#define _PY_FRAME_GENERAL 513
|
||||
#define _PY_FRAME_KW 514
|
||||
#define _QUICKEN_RESUME 515
|
||||
#define _REPLACE_WITH_TRUE 516
|
||||
#define _RESUME_CHECK RESUME_CHECK
|
||||
#define _RETURN_GENERATOR RETURN_GENERATOR
|
||||
#define _RETURN_VALUE RETURN_VALUE
|
||||
#define _SAVE_RETURN_OFFSET 516
|
||||
#define _SEND 517
|
||||
#define _SEND_GEN_FRAME 518
|
||||
#define _SAVE_RETURN_OFFSET 517
|
||||
#define _SEND 518
|
||||
#define _SEND_GEN_FRAME 519
|
||||
#define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS
|
||||
#define _SET_ADD SET_ADD
|
||||
#define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE
|
||||
#define _SET_UPDATE SET_UPDATE
|
||||
#define _START_EXECUTOR 519
|
||||
#define _STORE_ATTR 520
|
||||
#define _STORE_ATTR_INSTANCE_VALUE 521
|
||||
#define _STORE_ATTR_SLOT 522
|
||||
#define _STORE_ATTR_WITH_HINT 523
|
||||
#define _START_EXECUTOR 520
|
||||
#define _STORE_ATTR 521
|
||||
#define _STORE_ATTR_INSTANCE_VALUE 522
|
||||
#define _STORE_ATTR_SLOT 523
|
||||
#define _STORE_ATTR_WITH_HINT 524
|
||||
#define _STORE_DEREF STORE_DEREF
|
||||
#define _STORE_FAST 524
|
||||
#define _STORE_FAST_0 525
|
||||
#define _STORE_FAST_1 526
|
||||
#define _STORE_FAST_2 527
|
||||
#define _STORE_FAST_3 528
|
||||
#define _STORE_FAST_4 529
|
||||
#define _STORE_FAST_5 530
|
||||
#define _STORE_FAST_6 531
|
||||
#define _STORE_FAST_7 532
|
||||
#define _STORE_FAST 525
|
||||
#define _STORE_FAST_0 526
|
||||
#define _STORE_FAST_1 527
|
||||
#define _STORE_FAST_2 528
|
||||
#define _STORE_FAST_3 529
|
||||
#define _STORE_FAST_4 530
|
||||
#define _STORE_FAST_5 531
|
||||
#define _STORE_FAST_6 532
|
||||
#define _STORE_FAST_7 533
|
||||
#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST
|
||||
#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST
|
||||
#define _STORE_GLOBAL STORE_GLOBAL
|
||||
#define _STORE_NAME STORE_NAME
|
||||
#define _STORE_SLICE 533
|
||||
#define _STORE_SUBSCR 534
|
||||
#define _STORE_SUBSCR_DICT 535
|
||||
#define _STORE_SUBSCR_LIST_INT 536
|
||||
#define _SWAP 537
|
||||
#define _SWAP_2 538
|
||||
#define _SWAP_3 539
|
||||
#define _TIER2_RESUME_CHECK 540
|
||||
#define _TO_BOOL 541
|
||||
#define _STORE_SLICE 534
|
||||
#define _STORE_SUBSCR 535
|
||||
#define _STORE_SUBSCR_DICT 536
|
||||
#define _STORE_SUBSCR_LIST_INT 537
|
||||
#define _SWAP 538
|
||||
#define _SWAP_2 539
|
||||
#define _SWAP_3 540
|
||||
#define _TIER2_RESUME_CHECK 541
|
||||
#define _TO_BOOL 542
|
||||
#define _TO_BOOL_BOOL TO_BOOL_BOOL
|
||||
#define _TO_BOOL_INT TO_BOOL_INT
|
||||
#define _TO_BOOL_LIST 542
|
||||
#define _TO_BOOL_LIST 543
|
||||
#define _TO_BOOL_NONE TO_BOOL_NONE
|
||||
#define _TO_BOOL_STR 543
|
||||
#define _TO_BOOL_STR 544
|
||||
#define _UNARY_INVERT UNARY_INVERT
|
||||
#define _UNARY_NEGATIVE UNARY_NEGATIVE
|
||||
#define _UNARY_NOT UNARY_NOT
|
||||
#define _UNPACK_EX UNPACK_EX
|
||||
#define _UNPACK_SEQUENCE 544
|
||||
#define _UNPACK_SEQUENCE_LIST 545
|
||||
#define _UNPACK_SEQUENCE_TUPLE 546
|
||||
#define _UNPACK_SEQUENCE_TWO_TUPLE 547
|
||||
#define _UNPACK_SEQUENCE 545
|
||||
#define _UNPACK_SEQUENCE_LIST 546
|
||||
#define _UNPACK_SEQUENCE_TUPLE 547
|
||||
#define _UNPACK_SEQUENCE_TWO_TUPLE 548
|
||||
#define _WITH_EXCEPT_START WITH_EXCEPT_START
|
||||
#define _YIELD_VALUE YIELD_VALUE
|
||||
#define MAX_UOP_ID 547
|
||||
#define MAX_UOP_ID 548
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
6
Include/internal/pycore_uop_metadata.h
generated
6
Include/internal/pycore_uop_metadata.h
generated
|
|
@ -330,12 +330,13 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
|
|||
[_LOAD_CONST_UNDER_INLINE] = 0,
|
||||
[_LOAD_CONST_UNDER_INLINE_BORROW] = 0,
|
||||
[_CHECK_FUNCTION] = HAS_DEOPT_FLAG,
|
||||
[_START_EXECUTOR] = 0,
|
||||
[_START_EXECUTOR] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG,
|
||||
[_MAKE_WARM] = 0,
|
||||
[_FATAL_ERROR] = 0,
|
||||
[_DEOPT] = 0,
|
||||
[_ERROR_POP_N] = HAS_ARG_FLAG,
|
||||
[_TIER2_RESUME_CHECK] = HAS_DEOPT_FLAG,
|
||||
[_COLD_EXIT] = HAS_ESCAPES_FLAG,
|
||||
};
|
||||
|
||||
const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = {
|
||||
|
|
@ -419,6 +420,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
|
|||
[_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE",
|
||||
[_CHECK_STACK_SPACE_OPERAND] = "_CHECK_STACK_SPACE_OPERAND",
|
||||
[_CHECK_VALIDITY] = "_CHECK_VALIDITY",
|
||||
[_COLD_EXIT] = "_COLD_EXIT",
|
||||
[_COMPARE_OP] = "_COMPARE_OP",
|
||||
[_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT",
|
||||
[_COMPARE_OP_INT] = "_COMPARE_OP_INT",
|
||||
|
|
@ -1301,6 +1303,8 @@ int _PyUop_num_popped(int opcode, int oparg)
|
|||
return 0;
|
||||
case _TIER2_RESUME_CHECK:
|
||||
return 0;
|
||||
case _COLD_EXIT:
|
||||
return 0;
|
||||
default:
|
||||
return -1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2964,6 +2964,8 @@ dummy_func(
|
|||
else {
|
||||
this_instr[1].counter = initial_jump_backoff_counter();
|
||||
assert(tstate->current_executor == NULL);
|
||||
assert(executor != tstate->interp->cold_executor);
|
||||
tstate->jit_exit = NULL;
|
||||
GOTO_TIER_TWO(executor);
|
||||
}
|
||||
}
|
||||
|
|
@ -3028,6 +3030,8 @@ dummy_func(
|
|||
}
|
||||
DISPATCH_GOTO();
|
||||
}
|
||||
assert(executor != tstate->interp->cold_executor);
|
||||
tstate->jit_exit = NULL;
|
||||
GOTO_TIER_TWO(executor);
|
||||
#else
|
||||
Py_FatalError("ENTER_EXECUTOR is not supported in this build");
|
||||
|
|
@ -5238,9 +5242,8 @@ dummy_func(
|
|||
|
||||
tier2 op(_EXIT_TRACE, (exit_p/4 --)) {
|
||||
_PyExitData *exit = (_PyExitData *)exit_p;
|
||||
PyCodeObject *code = _PyFrame_GetCode(frame);
|
||||
_Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target;
|
||||
#if defined(Py_DEBUG) && !defined(_Py_JIT)
|
||||
_Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target;
|
||||
OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
|
||||
if (frame->lltrace >= 2) {
|
||||
printf("SIDE EXIT: [UOp ");
|
||||
|
|
@ -5251,32 +5254,7 @@ dummy_func(
|
|||
_PyOpcode_OpName[target->op.code]);
|
||||
}
|
||||
#endif
|
||||
if (exit->executor && !exit->executor->vm_data.valid) {
|
||||
exit->temperature = initial_temperature_backoff_counter();
|
||||
Py_CLEAR(exit->executor);
|
||||
}
|
||||
if (exit->executor == NULL) {
|
||||
_Py_BackoffCounter temperature = exit->temperature;
|
||||
if (!backoff_counter_triggers(temperature)) {
|
||||
exit->temperature = advance_backoff_counter(temperature);
|
||||
GOTO_TIER_ONE(target);
|
||||
}
|
||||
_PyExecutorObject *executor;
|
||||
if (target->op.code == ENTER_EXECUTOR) {
|
||||
executor = code->co_executors->executors[target->op.arg];
|
||||
Py_INCREF(executor);
|
||||
}
|
||||
else {
|
||||
int chain_depth = current_executor->vm_data.chain_depth + 1;
|
||||
int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth);
|
||||
if (optimized <= 0) {
|
||||
exit->temperature = restart_backoff_counter(temperature);
|
||||
GOTO_TIER_ONE(optimized < 0 ? NULL : target);
|
||||
}
|
||||
exit->temperature = initial_temperature_backoff_counter();
|
||||
}
|
||||
exit->executor = executor;
|
||||
}
|
||||
tstate->jit_exit = exit;
|
||||
GOTO_TIER_TWO(exit->executor);
|
||||
}
|
||||
|
||||
|
|
@ -5375,7 +5353,14 @@ dummy_func(
|
|||
#ifndef _Py_JIT
|
||||
current_executor = (_PyExecutorObject*)executor;
|
||||
#endif
|
||||
assert(((_PyExecutorObject *)executor)->vm_data.valid);
|
||||
assert(tstate->jit_exit == NULL || tstate->jit_exit->executor == current_executor);
|
||||
tstate->current_executor = (PyObject *)executor;
|
||||
if (!current_executor->vm_data.valid) {
|
||||
assert(tstate->jit_exit->executor == current_executor);
|
||||
assert(tstate->current_executor == executor);
|
||||
_PyExecutor_ClearExit(tstate->jit_exit);
|
||||
DEOPT_IF(true);
|
||||
}
|
||||
}
|
||||
|
||||
tier2 op(_MAKE_WARM, (--)) {
|
||||
|
|
@ -5414,6 +5399,37 @@ dummy_func(
|
|||
assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version));
|
||||
}
|
||||
|
||||
tier2 op(_COLD_EXIT, ( -- )) {
|
||||
_PyExitData *exit = tstate->jit_exit;
|
||||
assert(exit != NULL);
|
||||
_Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target;
|
||||
_Py_BackoffCounter temperature = exit->temperature;
|
||||
if (!backoff_counter_triggers(temperature)) {
|
||||
exit->temperature = advance_backoff_counter(temperature);
|
||||
GOTO_TIER_ONE(target);
|
||||
}
|
||||
_PyExecutorObject *executor;
|
||||
if (target->op.code == ENTER_EXECUTOR) {
|
||||
PyCodeObject *code = _PyFrame_GetCode(frame);
|
||||
executor = code->co_executors->executors[target->op.arg];
|
||||
Py_INCREF(executor);
|
||||
}
|
||||
else {
|
||||
_PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit);
|
||||
assert(tstate->current_executor == (PyObject *)previous_executor);
|
||||
int chain_depth = previous_executor->vm_data.chain_depth + 1;
|
||||
int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth);
|
||||
if (optimized <= 0) {
|
||||
exit->temperature = restart_backoff_counter(temperature);
|
||||
GOTO_TIER_ONE(optimized < 0 ? NULL : target);
|
||||
}
|
||||
exit->temperature = initial_temperature_backoff_counter();
|
||||
}
|
||||
assert(tstate->jit_exit == exit);
|
||||
exit->executor = executor;
|
||||
GOTO_TIER_TWO(exit->executor);
|
||||
}
|
||||
|
||||
label(pop_2_error) {
|
||||
stack_pointer -= 2;
|
||||
assert(WITHIN_STACK_BOUNDS());
|
||||
|
|
|
|||
|
|
@ -1158,7 +1158,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
|
|||
uint64_t trace_uop_execution_counter = 0;
|
||||
#endif
|
||||
|
||||
assert(next_uop->opcode == _START_EXECUTOR);
|
||||
assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT);
|
||||
tier2_dispatch:
|
||||
for (;;) {
|
||||
uopcode = next_uop->opcode;
|
||||
|
|
|
|||
|
|
@ -359,7 +359,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer)
|
|||
do { \
|
||||
OPT_STAT_INC(traces_executed); \
|
||||
_PyExecutorObject *_executor = (EXECUTOR); \
|
||||
tstate->current_executor = (PyObject *)_executor; \
|
||||
jit_func jitted = _executor->jit_code; \
|
||||
/* Keep the shim frame alive via the executor: */ \
|
||||
Py_INCREF(_executor); \
|
||||
|
|
@ -378,9 +377,8 @@ do { \
|
|||
do { \
|
||||
OPT_STAT_INC(traces_executed); \
|
||||
_PyExecutorObject *_executor = (EXECUTOR); \
|
||||
tstate->current_executor = (PyObject *)_executor; \
|
||||
next_uop = _executor->trace; \
|
||||
assert(next_uop->opcode == _START_EXECUTOR); \
|
||||
assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); \
|
||||
goto enter_tier_two; \
|
||||
} while (0)
|
||||
#endif
|
||||
|
|
@ -390,7 +388,6 @@ do { \
|
|||
{ \
|
||||
tstate->current_executor = NULL; \
|
||||
next_instr = (TARGET); \
|
||||
assert(tstate->current_executor == NULL); \
|
||||
OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \
|
||||
_PyFrame_SetStackPointer(frame, stack_pointer); \
|
||||
stack_pointer = _PyFrame_GetStackPointer(frame); \
|
||||
|
|
|
|||
84
Python/executor_cases.c.h
generated
84
Python/executor_cases.c.h
generated
|
|
@ -7113,9 +7113,8 @@
|
|||
case _EXIT_TRACE: {
|
||||
PyObject *exit_p = (PyObject *)CURRENT_OPERAND0();
|
||||
_PyExitData *exit = (_PyExitData *)exit_p;
|
||||
PyCodeObject *code = _PyFrame_GetCode(frame);
|
||||
_Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target;
|
||||
#if defined(Py_DEBUG) && !defined(_Py_JIT)
|
||||
_Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target;
|
||||
OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
|
||||
if (frame->lltrace >= 2) {
|
||||
_PyFrame_SetStackPointer(frame, stack_pointer);
|
||||
|
|
@ -7128,36 +7127,7 @@
|
|||
stack_pointer = _PyFrame_GetStackPointer(frame);
|
||||
}
|
||||
#endif
|
||||
if (exit->executor && !exit->executor->vm_data.valid) {
|
||||
exit->temperature = initial_temperature_backoff_counter();
|
||||
_PyFrame_SetStackPointer(frame, stack_pointer);
|
||||
Py_CLEAR(exit->executor);
|
||||
stack_pointer = _PyFrame_GetStackPointer(frame);
|
||||
}
|
||||
if (exit->executor == NULL) {
|
||||
_Py_BackoffCounter temperature = exit->temperature;
|
||||
if (!backoff_counter_triggers(temperature)) {
|
||||
exit->temperature = advance_backoff_counter(temperature);
|
||||
GOTO_TIER_ONE(target);
|
||||
}
|
||||
_PyExecutorObject *executor;
|
||||
if (target->op.code == ENTER_EXECUTOR) {
|
||||
executor = code->co_executors->executors[target->op.arg];
|
||||
Py_INCREF(executor);
|
||||
}
|
||||
else {
|
||||
int chain_depth = current_executor->vm_data.chain_depth + 1;
|
||||
_PyFrame_SetStackPointer(frame, stack_pointer);
|
||||
int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth);
|
||||
stack_pointer = _PyFrame_GetStackPointer(frame);
|
||||
if (optimized <= 0) {
|
||||
exit->temperature = restart_backoff_counter(temperature);
|
||||
GOTO_TIER_ONE(optimized < 0 ? NULL : target);
|
||||
}
|
||||
exit->temperature = initial_temperature_backoff_counter();
|
||||
}
|
||||
exit->executor = executor;
|
||||
}
|
||||
tstate->jit_exit = exit;
|
||||
GOTO_TIER_TWO(exit->executor);
|
||||
break;
|
||||
}
|
||||
|
|
@ -7438,7 +7408,19 @@
|
|||
#ifndef _Py_JIT
|
||||
current_executor = (_PyExecutorObject*)executor;
|
||||
#endif
|
||||
assert(((_PyExecutorObject *)executor)->vm_data.valid);
|
||||
assert(tstate->jit_exit == NULL || tstate->jit_exit->executor == current_executor);
|
||||
tstate->current_executor = (PyObject *)executor;
|
||||
if (!current_executor->vm_data.valid) {
|
||||
assert(tstate->jit_exit->executor == current_executor);
|
||||
assert(tstate->current_executor == executor);
|
||||
_PyFrame_SetStackPointer(frame, stack_pointer);
|
||||
_PyExecutor_ClearExit(tstate->jit_exit);
|
||||
stack_pointer = _PyFrame_GetStackPointer(frame);
|
||||
if (true) {
|
||||
UOP_STAT_INC(uopcode, miss);
|
||||
JUMP_TO_JUMP_TARGET();
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -7487,4 +7469,40 @@
|
|||
break;
|
||||
}
|
||||
|
||||
case _COLD_EXIT: {
|
||||
_PyExitData *exit = tstate->jit_exit;
|
||||
assert(exit != NULL);
|
||||
_Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target;
|
||||
_Py_BackoffCounter temperature = exit->temperature;
|
||||
if (!backoff_counter_triggers(temperature)) {
|
||||
exit->temperature = advance_backoff_counter(temperature);
|
||||
GOTO_TIER_ONE(target);
|
||||
}
|
||||
_PyExecutorObject *executor;
|
||||
if (target->op.code == ENTER_EXECUTOR) {
|
||||
PyCodeObject *code = _PyFrame_GetCode(frame);
|
||||
executor = code->co_executors->executors[target->op.arg];
|
||||
Py_INCREF(executor);
|
||||
}
|
||||
else {
|
||||
_PyFrame_SetStackPointer(frame, stack_pointer);
|
||||
_PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit);
|
||||
stack_pointer = _PyFrame_GetStackPointer(frame);
|
||||
assert(tstate->current_executor == (PyObject *)previous_executor);
|
||||
int chain_depth = previous_executor->vm_data.chain_depth + 1;
|
||||
_PyFrame_SetStackPointer(frame, stack_pointer);
|
||||
int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth);
|
||||
stack_pointer = _PyFrame_GetStackPointer(frame);
|
||||
if (optimized <= 0) {
|
||||
exit->temperature = restart_backoff_counter(temperature);
|
||||
GOTO_TIER_ONE(optimized < 0 ? NULL : target);
|
||||
}
|
||||
exit->temperature = initial_temperature_backoff_counter();
|
||||
}
|
||||
assert(tstate->jit_exit == exit);
|
||||
exit->executor = executor;
|
||||
GOTO_TIER_TWO(exit->executor);
|
||||
break;
|
||||
}
|
||||
|
||||
#undef TIER_TWO
|
||||
|
|
|
|||
4
Python/generated_cases.c.h
generated
4
Python/generated_cases.c.h
generated
|
|
@ -5595,6 +5595,8 @@
|
|||
}
|
||||
DISPATCH_GOTO();
|
||||
}
|
||||
assert(executor != tstate->interp->cold_executor);
|
||||
tstate->jit_exit = NULL;
|
||||
GOTO_TIER_TWO(executor);
|
||||
#else
|
||||
Py_FatalError("ENTER_EXECUTOR is not supported in this build");
|
||||
|
|
@ -7793,6 +7795,8 @@
|
|||
this_instr[1].counter = initial_jump_backoff_counter();
|
||||
stack_pointer = _PyFrame_GetStackPointer(frame);
|
||||
assert(tstate->current_executor == NULL);
|
||||
assert(executor != tstate->interp->cold_executor);
|
||||
tstate->jit_exit = NULL;
|
||||
GOTO_TIER_TWO(executor);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -546,7 +546,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz
|
|||
group->emit(code, data, executor, NULL, &state);
|
||||
code += group->code_size;
|
||||
data += group->data_size;
|
||||
assert(trace[0].opcode == _START_EXECUTOR);
|
||||
assert(trace[0].opcode == _START_EXECUTOR || trace[0].opcode == _COLD_EXIT);
|
||||
for (size_t i = 0; i < length; i++) {
|
||||
const _PyUOpInstruction *instruction = &trace[i];
|
||||
group = &stencil_groups[instruction->opcode];
|
||||
|
|
|
|||
|
|
@ -205,8 +205,8 @@ static int executor_clear(PyObject *executor);
|
|||
static void unlink_executor(_PyExecutorObject *executor);
|
||||
|
||||
|
||||
static void
|
||||
free_executor(_PyExecutorObject *self)
|
||||
void
|
||||
_PyExecutor_Free(_PyExecutorObject *self)
|
||||
{
|
||||
#ifdef _Py_JIT
|
||||
_PyJIT_Free(self);
|
||||
|
|
@ -242,7 +242,7 @@ _Py_ClearExecutorDeletionList(PyInterpreterState *interp)
|
|||
}
|
||||
else {
|
||||
*prev_to_next_ptr = exec->vm_data.links.next;
|
||||
free_executor(exec);
|
||||
_PyExecutor_Free(exec);
|
||||
}
|
||||
exec = *prev_to_next_ptr;
|
||||
}
|
||||
|
|
@ -1129,7 +1129,7 @@ sanity_check(_PyExecutorObject *executor)
|
|||
}
|
||||
bool ended = false;
|
||||
uint32_t i = 0;
|
||||
CHECK(executor->trace[0].opcode == _START_EXECUTOR);
|
||||
CHECK(executor->trace[0].opcode == _START_EXECUTOR || executor->trace[0].opcode == _COLD_EXIT);
|
||||
for (; i < executor->code_size; i++) {
|
||||
const _PyUOpInstruction *inst = &executor->trace[i];
|
||||
uint16_t opcode = inst->opcode;
|
||||
|
|
@ -1182,9 +1182,11 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil
|
|||
}
|
||||
|
||||
/* Initialize exits */
|
||||
_PyExecutorObject *cold = _PyExecutor_GetColdExecutor();
|
||||
for (int i = 0; i < exit_count; i++) {
|
||||
executor->exits[i].executor = NULL;
|
||||
executor->exits[i].index = i;
|
||||
executor->exits[i].temperature = initial_temperature_backoff_counter();
|
||||
executor->exits[i].executor = cold;
|
||||
}
|
||||
int next_exit = exit_count-1;
|
||||
_PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length];
|
||||
|
|
@ -1462,6 +1464,46 @@ _Py_ExecutorInit(_PyExecutorObject *executor, const _PyBloomFilter *dependency_s
|
|||
link_executor(executor);
|
||||
}
|
||||
|
||||
_PyExecutorObject *
|
||||
_PyExecutor_GetColdExecutor(void)
|
||||
{
|
||||
PyInterpreterState *interp = _PyInterpreterState_GET();
|
||||
if (interp->cold_executor != NULL) {
|
||||
return interp->cold_executor;
|
||||
}
|
||||
_PyExecutorObject *cold = allocate_executor(0, 1);
|
||||
if (cold == NULL) {
|
||||
Py_FatalError("Cannot allocate core JIT code");
|
||||
}
|
||||
((_PyUOpInstruction *)cold->trace)->opcode = _COLD_EXIT;
|
||||
#ifdef _Py_JIT
|
||||
cold->jit_code = NULL;
|
||||
cold->jit_side_entry = NULL;
|
||||
cold->jit_size = 0;
|
||||
// This is initialized to true so we can prevent the executor
|
||||
// from being immediately detected as cold and invalidated.
|
||||
cold->vm_data.warm = true;
|
||||
if (_PyJIT_Compile(cold, cold->trace, 1)) {
|
||||
Py_DECREF(cold);
|
||||
Py_FatalError("Cannot allocate core JIT code");
|
||||
}
|
||||
#endif
|
||||
_Py_SetImmortal((PyObject *)cold);
|
||||
interp->cold_executor = cold;
|
||||
return cold;
|
||||
}
|
||||
|
||||
void
|
||||
_PyExecutor_ClearExit(_PyExitData *exit)
|
||||
{
|
||||
if (exit == NULL) {
|
||||
return;
|
||||
}
|
||||
_PyExecutorObject *old = exit->executor;
|
||||
exit->executor = _PyExecutor_GetColdExecutor();
|
||||
Py_DECREF(old);
|
||||
}
|
||||
|
||||
/* Detaches the executor from the code object (if any) that
|
||||
* holds a reference to it */
|
||||
void
|
||||
|
|
@ -1492,14 +1534,18 @@ executor_clear(PyObject *op)
|
|||
assert(executor->vm_data.valid == 1);
|
||||
unlink_executor(executor);
|
||||
executor->vm_data.valid = 0;
|
||||
|
||||
/* It is possible for an executor to form a reference
|
||||
* cycle with itself, so decref'ing a side exit could
|
||||
* free the executor unless we hold a strong reference to it
|
||||
*/
|
||||
_PyExecutorObject *cold = _PyExecutor_GetColdExecutor();
|
||||
Py_INCREF(executor);
|
||||
for (uint32_t i = 0; i < executor->exit_count; i++) {
|
||||
executor->exits[i].temperature = initial_unreachable_backoff_counter();
|
||||
Py_CLEAR(executor->exits[i].executor);
|
||||
_PyExecutorObject *e = executor->exits[i].executor;
|
||||
executor->exits[i].executor = cold;
|
||||
Py_DECREF(e);
|
||||
}
|
||||
_Py_ExecutorDetach(executor);
|
||||
Py_DECREF(executor);
|
||||
|
|
@ -1741,4 +1787,11 @@ _PyDumpExecutors(FILE *out)
|
|||
return -1;
|
||||
}
|
||||
|
||||
void
|
||||
_PyExecutor_Free(struct _PyExecutorObject *self)
|
||||
{
|
||||
/* This should never be called */
|
||||
Py_UNREACHABLE();
|
||||
}
|
||||
|
||||
#endif /* _Py_TIER2 */
|
||||
|
|
|
|||
4
Python/optimizer_cases.c.h
generated
4
Python/optimizer_cases.c.h
generated
|
|
@ -3206,3 +3206,7 @@
|
|||
break;
|
||||
}
|
||||
|
||||
case _COLD_EXIT: {
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -815,7 +815,13 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate)
|
|||
/* Last garbage collection on this interpreter */
|
||||
_PyGC_CollectNoFail(tstate);
|
||||
_PyGC_Fini(interp);
|
||||
|
||||
struct _PyExecutorObject *cold = interp->cold_executor;
|
||||
if (cold != NULL) {
|
||||
interp->cold_executor = NULL;
|
||||
assert(cold->vm_data.valid);
|
||||
assert(cold->vm_data.warm);
|
||||
_PyExecutor_Free(cold);
|
||||
}
|
||||
/* We don't clear sysdict and builtins until the end of this function.
|
||||
Because clearing other attributes can execute arbitrary Python code
|
||||
which requires sysdict and builtins. */
|
||||
|
|
@ -1469,6 +1475,7 @@ init_threadstate(_PyThreadStateImpl *_tstate,
|
|||
tstate->datastack_limit = NULL;
|
||||
tstate->what_event = -1;
|
||||
tstate->current_executor = NULL;
|
||||
tstate->jit_exit = NULL;
|
||||
tstate->dict_global_version = 0;
|
||||
|
||||
_tstate->c_stack_soft_limit = UINTPTR_MAX;
|
||||
|
|
|
|||
|
|
@ -51,7 +51,6 @@
|
|||
do { \
|
||||
OPT_STAT_INC(traces_executed); \
|
||||
_PyExecutorObject *_executor = (EXECUTOR); \
|
||||
tstate->current_executor = (PyObject *)_executor; \
|
||||
jit_func_preserve_none jitted = _executor->jit_side_entry; \
|
||||
__attribute__((musttail)) return jitted(frame, stack_pointer, tstate); \
|
||||
} while (0)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue