| 
									
										
										
										
											2022-12-08 17:50:58 -07:00
										 |  |  | #ifndef Py_INTERNAL_PYTHREAD_H
 | 
					
						
							|  |  |  | #define Py_INTERNAL_PYTHREAD_H
 | 
					
						
							|  |  |  | #ifdef __cplusplus
 | 
					
						
							|  |  |  | extern "C" { | 
					
						
							|  |  |  | #endif
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | #ifndef Py_BUILD_CORE
 | 
					
						
							|  |  |  | #  error "this header requires Py_BUILD_CORE define"
 | 
					
						
							|  |  |  | #endif
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2023-10-18 00:33:50 +09:00
										 |  |  | #include "dynamic_annotations.h" // _Py_ANNOTATE_PURE_HAPPENS_BEFORE_MUTEX
 | 
					
						
							| 
									
										
										
										
											2024-02-06 14:45:04 -05:00
										 |  |  | #include "pycore_llist.h"        // struct llist_node
 | 
					
						
							| 
									
										
										
										
											2023-10-18 00:33:50 +09:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2023-09-30 19:25:54 +02:00
										 |  |  | // Get _POSIX_THREADS and _POSIX_SEMAPHORES macros if available
 | 
					
						
							|  |  |  | #if (defined(HAVE_UNISTD_H) && !defined(_POSIX_THREADS) \
 | 
					
						
							|  |  |  |                             && !defined(_POSIX_SEMAPHORES)) | 
					
						
							|  |  |  | #  include <unistd.h>             // _POSIX_THREADS, _POSIX_SEMAPHORES
 | 
					
						
							|  |  |  | #endif
 | 
					
						
							|  |  |  | #if (defined(HAVE_PTHREAD_H) && !defined(_POSIX_THREADS) \
 | 
					
						
							|  |  |  |                              && !defined(_POSIX_SEMAPHORES)) | 
					
						
							|  |  |  |    // This means pthreads are not implemented in libc headers, hence the macro
 | 
					
						
							|  |  |  |    // not present in <unistd.h>. But they still can be implemented as an
 | 
					
						
							|  |  |  |    // external library (e.g. gnu pth in pthread emulation)
 | 
					
						
							|  |  |  | #  include <pthread.h>            // _POSIX_THREADS, _POSIX_SEMAPHORES
 | 
					
						
							|  |  |  | #endif
 | 
					
						
							|  |  |  | #if !defined(_POSIX_THREADS) && defined(__hpux) && defined(_SC_THREADS)
 | 
					
						
							|  |  |  |    // Check if we're running on HP-UX and _SC_THREADS is defined. If so, then
 | 
					
						
							|  |  |  |    // enough of the POSIX threads package is implemented to support Python
 | 
					
						
							|  |  |  |    // threads.
 | 
					
						
							|  |  |  |    //
 | 
					
						
							|  |  |  |    // This is valid for HP-UX 11.23 running on an ia64 system. If needed, add
 | 
					
						
							|  |  |  |    // a check of __ia64 to verify that we're running on an ia64 system instead
 | 
					
						
							|  |  |  |    // of a pa-risc system.
 | 
					
						
							|  |  |  | #  define _POSIX_THREADS
 | 
					
						
							|  |  |  | #endif
 | 
					
						
							| 
									
										
										
										
											2022-12-08 17:50:58 -07:00
										 |  |  | 
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2022-12-09 10:17:54 -07:00
										 |  |  | #if defined(_POSIX_THREADS) || defined(HAVE_PTHREAD_STUBS)
 | 
					
						
							| 
									
										
										
										
											2023-08-21 20:05:59 +02:00
										 |  |  | #  define _USE_PTHREADS
 | 
					
						
							| 
									
										
										
										
											2022-12-08 17:50:58 -07:00
										 |  |  | #endif
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | #if defined(_USE_PTHREADS) && defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC)
 | 
					
						
							|  |  |  | // monotonic is supported statically.  It doesn't mean it works on runtime.
 | 
					
						
							| 
									
										
										
										
											2023-08-21 20:05:59 +02:00
										 |  |  | #  define CONDATTR_MONOTONIC
 | 
					
						
							| 
									
										
										
										
											2022-12-08 17:50:58 -07:00
										 |  |  | #endif
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | #if defined(HAVE_PTHREAD_STUBS)
 | 
					
						
							| 
									
										
										
										
											2024-03-01 15:22:31 -08:00
										 |  |  | #include "cpython/pthread_stubs.h"  // PTHREAD_KEYS_MAX
 | 
					
						
							|  |  |  | #include <stdbool.h>                // bool
 | 
					
						
							| 
									
										
										
										
											2023-09-30 22:06:45 +02:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2022-12-08 17:50:58 -07:00
										 |  |  | // pthread_key
 | 
					
						
							|  |  |  | struct py_stub_tls_entry { | 
					
						
							|  |  |  |     bool in_use; | 
					
						
							|  |  |  |     void *value; | 
					
						
							|  |  |  | }; | 
					
						
							|  |  |  | #endif
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | struct _pythread_runtime_state { | 
					
						
							|  |  |  |     int initialized; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | #ifdef _USE_PTHREADS
 | 
					
						
							|  |  |  |     // This matches when thread_pthread.h is used.
 | 
					
						
							|  |  |  |     struct { | 
					
						
							|  |  |  |         /* NULL when pthread_condattr_setclock(CLOCK_MONOTONIC) is not supported. */ | 
					
						
							|  |  |  |         pthread_condattr_t *ptr; | 
					
						
							|  |  |  | # ifdef CONDATTR_MONOTONIC
 | 
					
						
							|  |  |  |     /* The value to which condattr_monotonic is set. */ | 
					
						
							|  |  |  |         pthread_condattr_t val; | 
					
						
							|  |  |  | # endif
 | 
					
						
							|  |  |  |     } _condattr_monotonic; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | #endif  // USE_PTHREADS
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | #if defined(HAVE_PTHREAD_STUBS)
 | 
					
						
							|  |  |  |     struct { | 
					
						
							|  |  |  |         struct py_stub_tls_entry tls_entries[PTHREAD_KEYS_MAX]; | 
					
						
							|  |  |  |     } stubs; | 
					
						
							|  |  |  | #endif
 | 
					
						
							| 
									
										
										
										
											2024-02-06 14:45:04 -05:00
										 |  |  | 
 | 
					
						
							| 
									
										
											  
											
												gh-114271: Fix race in `Thread.join()` (#114839)
There is a race between when `Thread._tstate_lock` is released[^1] in `Thread._wait_for_tstate_lock()`
and when `Thread._stop()` asserts[^2] that it is unlocked. Consider the following execution
involving threads A, B, and C:
1. A starts.
2. B joins A, blocking on its `_tstate_lock`.
3. C joins A, blocking on its `_tstate_lock`.
4. A finishes and releases its `_tstate_lock`.
5. B acquires A's `_tstate_lock` in `_wait_for_tstate_lock()`, releases it, but is swapped
   out before calling `_stop()`.
6. C is scheduled, acquires A's `_tstate_lock` in `_wait_for_tstate_lock()` but is swapped
   out before releasing it.
7. B is scheduled, calls `_stop()`, which asserts that A's `_tstate_lock` is not held.
   However, C holds it, so the assertion fails.
The race can be reproduced[^3] by inserting sleeps at the appropriate points in
the threading code. To do so, run the `repro_join_race.py` from the linked repo.
There are two main parts to this PR:
1. `_tstate_lock` is replaced with an event that is attached to `PyThreadState`.
   The event is set by the runtime prior to the thread being cleared (in the same
   place that `_tstate_lock` was released). `Thread.join()` blocks waiting for the
   event to be set.
2. `_PyInterpreterState_WaitForThreads()` provides the ability to wait for all
   non-daemon threads to exit. To do so, an `is_daemon` predicate was added to
   `PyThreadState`. This field is set each time a thread is created. `threading._shutdown()`
   now calls into `_PyInterpreterState_WaitForThreads()` instead of waiting on
   `_tstate_lock`s.
[^1]: https://github.com/python/cpython/blob/441affc9e7f419ef0b68f734505fa2f79fe653c7/Lib/threading.py#L1201
[^2]: https://github.com/python/cpython/blob/441affc9e7f419ef0b68f734505fa2f79fe653c7/Lib/threading.py#L1115
[^3]: https://github.com/mpage/cpython/commit/81946532792f938cd6f6ab4c4ff92a4edf61314f
---------
Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com>
Co-authored-by: Antoine Pitrou <antoine@python.org>
											
										 
											2024-03-16 05:56:30 -07:00
										 |  |  |     // Linked list of ThreadHandles
 | 
					
						
							| 
									
										
										
										
											2024-02-06 14:45:04 -05:00
										 |  |  |     struct llist_node handles; | 
					
						
							| 
									
										
										
										
											2022-12-08 17:50:58 -07:00
										 |  |  | }; | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2024-02-06 14:45:04 -05:00
										 |  |  | #define _pythread_RUNTIME_INIT(pythread) \
 | 
					
						
							|  |  |  |     { \ | 
					
						
							|  |  |  |         .handles = LLIST_INIT(pythread.handles), \ | 
					
						
							|  |  |  |     } | 
					
						
							| 
									
										
										
										
											2022-12-08 17:50:58 -07:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2023-08-29 04:38:23 +02:00
										 |  |  | #ifdef HAVE_FORK
 | 
					
						
							|  |  |  | /* Private function to reinitialize a lock at fork in the child process.
 | 
					
						
							|  |  |  |    Reset the lock to the unlocked state. | 
					
						
							|  |  |  |    Return 0 on success, return -1 on error. */ | 
					
						
							|  |  |  | extern int _PyThread_at_fork_reinit(PyThread_type_lock *lock); | 
					
						
							| 
									
										
										
										
											2024-02-06 14:45:04 -05:00
										 |  |  | extern void _PyThread_AfterFork(struct _pythread_runtime_state *state); | 
					
						
							| 
									
										
										
										
											2023-08-29 04:38:23 +02:00
										 |  |  | #endif  /* HAVE_FORK */
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2023-10-17 16:32:00 -06:00
										 |  |  | // unset: -1 seconds, in nanoseconds
 | 
					
						
							| 
									
										
										
										
											2024-02-20 16:02:27 +01:00
										 |  |  | #define PyThread_UNSET_TIMEOUT ((PyTime_t)(-1 * 1000 * 1000 * 1000))
 | 
					
						
							| 
									
										
										
										
											2023-10-17 16:32:00 -06:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2023-10-17 17:05:49 -06:00
										 |  |  | // Exported for the _xxinterpchannels module.
 | 
					
						
							|  |  |  | PyAPI_FUNC(int) PyThread_ParseTimeoutArg( | 
					
						
							|  |  |  |     PyObject *arg, | 
					
						
							|  |  |  |     int blocking, | 
					
						
							|  |  |  |     PY_TIMEOUT_T *timeout); | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2023-10-17 16:32:00 -06:00
										 |  |  | /* Helper to acquire an interruptible lock with a timeout.  If the lock acquire
 | 
					
						
							|  |  |  |  * is interrupted, signal handlers are run, and if they raise an exception, | 
					
						
							|  |  |  |  * PY_LOCK_INTR is returned.  Otherwise, PY_LOCK_ACQUIRED or PY_LOCK_FAILURE | 
					
						
							|  |  |  |  * are returned, depending on whether the lock can be acquired within the | 
					
						
							|  |  |  |  * timeout. | 
					
						
							|  |  |  |  */ | 
					
						
							|  |  |  | // Exported for the _xxinterpchannels module.
 | 
					
						
							|  |  |  | PyAPI_FUNC(PyLockStatus) PyThread_acquire_lock_timed_with_retries( | 
					
						
							|  |  |  |     PyThread_type_lock, | 
					
						
							|  |  |  |     PY_TIMEOUT_T microseconds); | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2023-11-04 14:59:24 +01:00
										 |  |  | typedef unsigned long long PyThread_ident_t; | 
					
						
							|  |  |  | typedef Py_uintptr_t PyThread_handle_t; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | #define PY_FORMAT_THREAD_IDENT_T "llu"
 | 
					
						
							|  |  |  | #define Py_PARSE_THREAD_IDENT_T "K"
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | PyAPI_FUNC(PyThread_ident_t) PyThread_get_thread_ident_ex(void); | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | /* Thread joining APIs.
 | 
					
						
							|  |  |  |  * | 
					
						
							|  |  |  |  * These APIs have a strict contract: | 
					
						
							|  |  |  |  *  - Either PyThread_join_thread or PyThread_detach_thread must be called | 
					
						
							|  |  |  |  *    exactly once with the given handle. | 
					
						
							|  |  |  |  *  - Calling neither PyThread_join_thread nor PyThread_detach_thread results | 
					
						
							|  |  |  |  *    in a resource leak until the end of the process. | 
					
						
							|  |  |  |  *  - Any other usage, such as calling both PyThread_join_thread and | 
					
						
							|  |  |  |  *    PyThread_detach_thread, or calling them more than once (including | 
					
						
							|  |  |  |  *    simultaneously), results in undefined behavior. | 
					
						
							|  |  |  |  */ | 
					
						
							|  |  |  | PyAPI_FUNC(int) PyThread_start_joinable_thread(void (*func)(void *), | 
					
						
							|  |  |  |                                                void *arg, | 
					
						
							|  |  |  |                                                PyThread_ident_t* ident, | 
					
						
							|  |  |  |                                                PyThread_handle_t* handle); | 
					
						
							|  |  |  | /*
 | 
					
						
							|  |  |  |  * Join a thread started with `PyThread_start_joinable_thread`. | 
					
						
							|  |  |  |  * This function cannot be interrupted. It returns 0 on success, | 
					
						
							|  |  |  |  * a non-zero value on failure. | 
					
						
							|  |  |  |  */ | 
					
						
							|  |  |  | PyAPI_FUNC(int) PyThread_join_thread(PyThread_handle_t); | 
					
						
							|  |  |  | /*
 | 
					
						
							|  |  |  |  * Detach a thread started with `PyThread_start_joinable_thread`, such | 
					
						
							|  |  |  |  * that its resources are relased as soon as it exits. | 
					
						
							|  |  |  |  * This function cannot be interrupted. It returns 0 on success, | 
					
						
							|  |  |  |  * a non-zero value on failure. | 
					
						
							|  |  |  |  */ | 
					
						
							|  |  |  | PyAPI_FUNC(int) PyThread_detach_thread(PyThread_handle_t); | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2022-12-08 17:50:58 -07:00
										 |  |  | #ifdef __cplusplus
 | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | #endif
 | 
					
						
							|  |  |  | #endif /* !Py_INTERNAL_PYTHREAD_H */
 |