gh-143123: Protect against recursive tracer calls/finalization (GH-143126)

* Stronger check for recursive traces

* Add a stop_tracing field

* Stop early when tracing exceptions

Python/bytecodes.c

@@ -5620,6 +5620,9 @@ dummy_func(
 #else
             assert(_PyErr_Occurred(tstate));
 #endif
+            SAVE_STACK();
+            STOP_TRACING();
+            RELOAD_STACK();
 
             /* Log traceback info. */
             assert(frame->owner != FRAME_OWNED_BY_INTERPRETER);
@@ -5634,6 +5637,9 @@ dummy_func(
         }
 
         spilled label(exception_unwind) {
+            SAVE_STACK();
+            STOP_TRACING();
+            RELOAD_STACK();
             /* We can't use frame->instr_ptr here, as RERAISE may have set it */
             int offset = INSTR_OFFSET()-1;
             int level, handler, lasti;

Python/ceval.c

@@ -1460,32 +1460,7 @@ stop_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame)
     if (!_PyErr_Occurred(tstate) && !_is_sys_tracing) {
         err = _PyOptimizer_Optimize(frame, tstate);
     }
-    _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;
-    // Deal with backoffs
-    _PyJitTracerState *tracer = _tstate->jit_tracer_state;
-    assert(tracer != NULL);
-    _PyExitData *exit = tracer->initial_state.exit;
-    if (exit == NULL) {
-        // We hold a strong reference to the code object, so the instruction won't be freed.
-        if (err <= 0) {
-            _Py_BackoffCounter counter = tracer->initial_state.jump_backward_instr[1].counter;
-            tracer->initial_state.jump_backward_instr[1].counter = restart_backoff_counter(counter);
-        }
-        else {
-            tracer->initial_state.jump_backward_instr[1].counter = initial_jump_backoff_counter(&_tstate->policy);
-        }
-    }
-    else if (tracer->initial_state.executor->vm_data.valid) {
-        // Likewise, we hold a strong reference to the executor containing this exit, so the exit is guaranteed
-        // to be valid to access.
-        if (err <= 0) {
-            exit->temperature = restart_backoff_counter(exit->temperature);
-        }
-        else {
-            exit->temperature = initial_temperature_backoff_counter(&_tstate->policy);
-        }
-    }
-    _PyJit_FinalizeTracing(tstate);
+    _PyJit_FinalizeTracing(tstate, err);
     return err;
 }
 #endif

Python/ceval_macros.h

@@ -156,6 +156,19 @@
 #  define LEAVE_TRACING() tracing_mode = 0
 #endif
 
+#if _Py_TIER2
+#define STOP_TRACING() \
+    do { \
+        if (IS_JIT_TRACING()) { \
+            LEAVE_TRACING(); \
+            _PyJit_FinalizeTracing(tstate, 0); \
+        } \
+    } while (0);
+#else
+#define STOP_TRACING() ((void)(0));
+#endif
+
+
 /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */
 #ifdef Py_DEBUG
 #define PRE_DISPATCH_GOTO() if (frame->lltrace >= 5) { \

Python/generated_cases.c.h

@@ -12368,7 +12368,9 @@ JUMP_TO_LABEL(error);
             #else
             assert(_PyErr_Occurred(tstate));
             #endif
-
+            _PyFrame_SetStackPointer(frame, stack_pointer);
+            STOP_TRACING();
+            stack_pointer = _PyFrame_GetStackPointer(frame);
             assert(frame->owner != FRAME_OWNED_BY_INTERPRETER);
             if (!_PyFrame_IsIncomplete(frame)) {
                 _PyFrame_SetStackPointer(frame, stack_pointer);
@@ -12387,6 +12389,7 @@ JUMP_TO_LABEL(error);
 
         LABEL(exception_unwind)
         {
+            STOP_TRACING();
             int offset = INSTR_OFFSET()-1;
             int level, handler, lasti;
             int handled = get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti);

Python/optimizer.c

@@ -1030,11 +1030,11 @@ _PyJit_TryInitializeTracing(
             // Don't error, just go to next instruction.
             return 0;
         }
+        _tstate->jit_tracer_state->is_tracing = false;
     }
     _PyJitTracerState *tracer = _tstate->jit_tracer_state;
     // A recursive trace.
-    // Don't trace into the inner call because it will stomp on the previous trace, causing endless retraces.
-    if (tracer->prev_state.code_curr_size > CODE_SIZE_EMPTY) {
+    if (tracer->is_tracing) {
         return 0;
     }
     if (oparg > 0xFFFF) {
@@ -1086,20 +1086,45 @@ _PyJit_TryInitializeTracing(
         close_loop_instr[1].counter = trigger_backoff_counter();
     }
     _Py_BloomFilter_Init(&tracer->prev_state.dependencies);
+    tracer->is_tracing = true;
     return 1;
 }
 
 Py_NO_INLINE void
-_PyJit_FinalizeTracing(PyThreadState *tstate)
+_PyJit_FinalizeTracing(PyThreadState *tstate, int err)
 {
     _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;
     _PyJitTracerState *tracer = _tstate->jit_tracer_state;
+    // Deal with backoffs
+    assert(tracer != NULL);
+    _PyExitData *exit = tracer->initial_state.exit;
+    if (exit == NULL) {
+        // We hold a strong reference to the code object, so the instruction won't be freed.
+        if (err <= 0) {
+            _Py_BackoffCounter counter = tracer->initial_state.jump_backward_instr[1].counter;
+            tracer->initial_state.jump_backward_instr[1].counter = restart_backoff_counter(counter);
+        }
+        else {
+            tracer->initial_state.jump_backward_instr[1].counter = initial_jump_backoff_counter(&_tstate->policy);
+        }
+    }
+    else if (tracer->initial_state.executor->vm_data.valid) {
+        // Likewise, we hold a strong reference to the executor containing this exit, so the exit is guaranteed
+        // to be valid to access.
+        if (err <= 0) {
+            exit->temperature = restart_backoff_counter(exit->temperature);
+        }
+        else {
+            exit->temperature = initial_temperature_backoff_counter(&_tstate->policy);
+        }
+    }
     Py_CLEAR(tracer->initial_state.code);
     Py_CLEAR(tracer->initial_state.func);
     Py_CLEAR(tracer->initial_state.executor);
     Py_CLEAR(tracer->prev_state.instr_code);
     tracer->prev_state.code_curr_size = CODE_SIZE_EMPTY;
     tracer->prev_state.code_max_size = UOP_MAX_TRACE_LENGTH/2 - 1;
+    tracer->is_tracing = false;
 }
 
 void