gh-138122: Add --subprocesses flag to profile child processes in tachyon (#142636)

This commit is contained in:
Pablo Galindo Salgado 2025-12-15 12:11:40 +00:00 committed by GitHub
parent 14e6052b43
commit 6658e2cb07
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 2702 additions and 562 deletions

View file

@ -8,23 +8,24 @@
#ifndef Py_REMOTE_DEBUGGING_H
#define Py_REMOTE_DEBUGGING_H
/* _GNU_SOURCE must be defined before any system headers */
#define _GNU_SOURCE
#ifdef __cplusplus
extern "C" {
#endif
#define _GNU_SOURCE
#ifndef Py_BUILD_CORE_BUILTIN
# define Py_BUILD_CORE_MODULE 1
#endif
#include "Python.h"
#include <internal/pycore_debug_offsets.h> // _Py_DebugOffsets
#include <internal/pycore_frame.h> // FRAME_SUSPENDED_YIELD_FROM
#include <internal/pycore_interpframe.h> // FRAME_OWNED_BY_INTERPRETER
#include <internal/pycore_llist.h> // struct llist_node
#include <internal/pycore_long.h> // _PyLong_GetZero
#include <internal/pycore_stackref.h> // Py_TAG_BITS
#include "internal/pycore_debug_offsets.h" // _Py_DebugOffsets
#include "internal/pycore_frame.h" // FRAME_SUSPENDED_YIELD_FROM
#include "internal/pycore_interpframe.h" // FRAME_OWNED_BY_INTERPRETER
#include "internal/pycore_llist.h" // struct llist_node
#include "internal/pycore_long.h" // _PyLong_GetZero
#include "internal/pycore_stackref.h" // Py_TAG_BITS
#include "../../Python/remote_debug.h"
#include <assert.h>
@ -40,10 +41,17 @@ extern "C" {
# define HAVE_PROCESS_VM_READV 0
#endif
#if defined(__APPLE__) && TARGET_OS_OSX
#include <libproc.h>
#include <sys/types.h>
#define MAX_NATIVE_THREADS 4096
#if defined(__APPLE__)
#include <TargetConditionals.h>
# if !defined(TARGET_OS_OSX)
/* Older macOS SDKs do not define TARGET_OS_OSX */
# define TARGET_OS_OSX 1
# endif
# if TARGET_OS_OSX
# include <libproc.h>
# include <sys/types.h>
# define MAX_NATIVE_THREADS 4096
# endif
#endif
#ifdef MS_WINDOWS
@ -581,6 +589,16 @@ extern int process_thread_for_async_stack_trace(
void *context
);
/* ============================================================================
* SUBPROCESS ENUMERATION FUNCTION DECLARATIONS
* ============================================================================ */
/* Get all child PIDs of a process.
* Returns a new Python list of PIDs, or NULL on error with exception set.
* If recursive is true, includes all descendants (children, grandchildren, etc.)
*/
extern PyObject *enumerate_child_pids(pid_t target_pid, int recursive);
#ifdef __cplusplus
}
#endif

View file

@ -433,4 +433,153 @@ _remote_debugging_RemoteUnwinder_get_stats(PyObject *self, PyObject *Py_UNUSED(i
return return_value;
}
/*[clinic end generated code: output=1943fb7a56197e39 input=a9049054013a1b77]*/
PyDoc_STRVAR(_remote_debugging_get_child_pids__doc__,
"get_child_pids($module, /, pid, *, recursive=True)\n"
"--\n"
"\n"
"Get all child process IDs of the given process.\n"
"\n"
" pid\n"
" Process ID of the parent process\n"
" recursive\n"
" If True, return all descendants (children, grandchildren, etc.).\n"
" If False, return only direct children.\n"
"\n"
"Returns a list of child process IDs. Returns an empty list if no children\n"
"are found.\n"
"\n"
"This function provides a snapshot of child processes at a moment in time.\n"
"Child processes may exit or new ones may be created after the list is returned.\n"
"\n"
"Raises:\n"
" OSError: If unable to enumerate processes\n"
" NotImplementedError: If not supported on this platform");
#define _REMOTE_DEBUGGING_GET_CHILD_PIDS_METHODDEF \
{"get_child_pids", _PyCFunction_CAST(_remote_debugging_get_child_pids), METH_FASTCALL|METH_KEYWORDS, _remote_debugging_get_child_pids__doc__},
static PyObject *
_remote_debugging_get_child_pids_impl(PyObject *module, int pid,
int recursive);
static PyObject *
_remote_debugging_get_child_pids(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
{
PyObject *return_value = NULL;
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
#define NUM_KEYWORDS 2
static struct {
PyGC_Head _this_is_not_used;
PyObject_VAR_HEAD
Py_hash_t ob_hash;
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_hash = -1,
.ob_item = { &_Py_ID(pid), &_Py_ID(recursive), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
#else // !Py_BUILD_CORE
# define KWTUPLE NULL
#endif // !Py_BUILD_CORE
static const char * const _keywords[] = {"pid", "recursive", NULL};
static _PyArg_Parser _parser = {
.keywords = _keywords,
.fname = "get_child_pids",
.kwtuple = KWTUPLE,
};
#undef KWTUPLE
PyObject *argsbuf[2];
Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1;
int pid;
int recursive = 1;
args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser,
/*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf);
if (!args) {
goto exit;
}
pid = PyLong_AsInt(args[0]);
if (pid == -1 && PyErr_Occurred()) {
goto exit;
}
if (!noptargs) {
goto skip_optional_kwonly;
}
recursive = PyObject_IsTrue(args[1]);
if (recursive < 0) {
goto exit;
}
skip_optional_kwonly:
return_value = _remote_debugging_get_child_pids_impl(module, pid, recursive);
exit:
return return_value;
}
PyDoc_STRVAR(_remote_debugging_is_python_process__doc__,
"is_python_process($module, /, pid)\n"
"--\n"
"\n"
"Check if a process is a Python process.");
#define _REMOTE_DEBUGGING_IS_PYTHON_PROCESS_METHODDEF \
{"is_python_process", _PyCFunction_CAST(_remote_debugging_is_python_process), METH_FASTCALL|METH_KEYWORDS, _remote_debugging_is_python_process__doc__},
static PyObject *
_remote_debugging_is_python_process_impl(PyObject *module, int pid);
static PyObject *
_remote_debugging_is_python_process(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
{
PyObject *return_value = NULL;
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
#define NUM_KEYWORDS 1
static struct {
PyGC_Head _this_is_not_used;
PyObject_VAR_HEAD
Py_hash_t ob_hash;
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_hash = -1,
.ob_item = { &_Py_ID(pid), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
#else // !Py_BUILD_CORE
# define KWTUPLE NULL
#endif // !Py_BUILD_CORE
static const char * const _keywords[] = {"pid", NULL};
static _PyArg_Parser _parser = {
.keywords = _keywords,
.fname = "is_python_process",
.kwtuple = KWTUPLE,
};
#undef KWTUPLE
PyObject *argsbuf[1];
int pid;
args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser,
/*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf);
if (!args) {
goto exit;
}
pid = PyLong_AsInt(args[0]);
if (pid == -1 && PyErr_Occurred()) {
goto exit;
}
return_value = _remote_debugging_is_python_process_impl(module, pid);
exit:
return return_value;
}
/*[clinic end generated code: output=dc0550ad3d6a409c input=a9049054013a1b77]*/

View file

@ -350,7 +350,7 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self,
}
// Validate that the debug offsets are valid
if(validate_debug_offsets(&self->debug_offsets) == -1) {
if (validate_debug_offsets(&self->debug_offsets) == -1) {
set_exception_cause(self, PyExc_RuntimeError, "Invalid debug offsets found");
return -1;
}
@ -933,7 +933,7 @@ RemoteUnwinder_dealloc(PyObject *op)
_Py_hashtable_destroy(self->code_object_cache);
}
#ifdef MS_WINDOWS
if(self->win_process_buffer != NULL) {
if (self->win_process_buffer != NULL) {
PyMem_Free(self->win_process_buffer);
}
#endif
@ -1122,7 +1122,74 @@ static PyModuleDef_Slot remote_debugging_slots[] = {
{0, NULL},
};
/* ============================================================================
* MODULE-LEVEL FUNCTIONS
* ============================================================================ */
/*[clinic input]
_remote_debugging.get_child_pids
pid: int
Process ID of the parent process
*
recursive: bool = True
If True, return all descendants (children, grandchildren, etc.).
If False, return only direct children.
Get all child process IDs of the given process.
Returns a list of child process IDs. Returns an empty list if no children
are found.
This function provides a snapshot of child processes at a moment in time.
Child processes may exit or new ones may be created after the list is returned.
Raises:
OSError: If unable to enumerate processes
NotImplementedError: If not supported on this platform
[clinic start generated code]*/
static PyObject *
_remote_debugging_get_child_pids_impl(PyObject *module, int pid,
int recursive)
/*[clinic end generated code: output=1ae2289c6b953e4b input=3395cbe7f17066c9]*/
{
return enumerate_child_pids((pid_t)pid, recursive);
}
/*[clinic input]
_remote_debugging.is_python_process
pid: int
Check if a process is a Python process.
[clinic start generated code]*/
static PyObject *
_remote_debugging_is_python_process_impl(PyObject *module, int pid)
/*[clinic end generated code: output=22947dc8afcac362 input=13488e28c7295d84]*/
{
proc_handle_t handle;
if (_Py_RemoteDebug_InitProcHandle(&handle, pid) < 0) {
PyErr_Clear();
Py_RETURN_FALSE;
}
uintptr_t runtime_start_address = _Py_RemoteDebug_GetPyRuntimeAddress(&handle);
_Py_RemoteDebug_CleanupProcHandle(&handle);
if (runtime_start_address == 0) {
PyErr_Clear();
Py_RETURN_FALSE;
}
Py_RETURN_TRUE;
}
static PyMethodDef remote_debugging_methods[] = {
_REMOTE_DEBUGGING_GET_CHILD_PIDS_METHODDEF
_REMOTE_DEBUGGING_IS_PYTHON_PROCESS_METHODDEF
{NULL, NULL, 0, NULL},
};

View file

@ -0,0 +1,459 @@
/******************************************************************************
* Remote Debugging Module - Subprocess Enumeration
*
* This file contains platform-specific functions for enumerating child
* processes of a given PID.
******************************************************************************/
#include "_remote_debugging.h"
#ifndef MS_WINDOWS
#include <unistd.h>
#include <dirent.h>
#endif
#ifdef MS_WINDOWS
#include <tlhelp32.h>
#endif
/* ============================================================================
* INTERNAL DATA STRUCTURES
* ============================================================================ */
/* Simple dynamic array for collecting PIDs */
typedef struct {
pid_t *pids;
size_t count;
size_t capacity;
} pid_array_t;
static int
pid_array_init(pid_array_t *arr)
{
arr->capacity = 64;
arr->count = 0;
arr->pids = (pid_t *)PyMem_Malloc(arr->capacity * sizeof(pid_t));
if (arr->pids == NULL) {
PyErr_NoMemory();
return -1;
}
return 0;
}
static void
pid_array_cleanup(pid_array_t *arr)
{
if (arr->pids != NULL) {
PyMem_Free(arr->pids);
arr->pids = NULL;
}
arr->count = 0;
arr->capacity = 0;
}
static int
pid_array_append(pid_array_t *arr, pid_t pid)
{
if (arr->count >= arr->capacity) {
/* Check for overflow before multiplication */
if (arr->capacity > SIZE_MAX / 2) {
PyErr_SetString(PyExc_OverflowError, "PID array capacity overflow");
return -1;
}
size_t new_capacity = arr->capacity * 2;
/* Check allocation size won't overflow */
if (new_capacity > SIZE_MAX / sizeof(pid_t)) {
PyErr_SetString(PyExc_OverflowError, "PID array size overflow");
return -1;
}
pid_t *new_pids = (pid_t *)PyMem_Realloc(arr->pids, new_capacity * sizeof(pid_t));
if (new_pids == NULL) {
PyErr_NoMemory();
return -1;
}
arr->pids = new_pids;
arr->capacity = new_capacity;
}
arr->pids[arr->count++] = pid;
return 0;
}
static int
pid_array_contains(pid_array_t *arr, pid_t pid)
{
for (size_t i = 0; i < arr->count; i++) {
if (arr->pids[i] == pid) {
return 1;
}
}
return 0;
}
/* ============================================================================
* SHARED BFS HELPER
* ============================================================================ */
/* Find child PIDs using BFS traversal of the pid->ppid mapping.
* all_pids and ppids must have the same count (parallel arrays).
* Returns 0 on success, -1 on error. */
static int
find_children_bfs(pid_t target_pid, int recursive,
pid_t *all_pids, pid_t *ppids, size_t pid_count,
pid_array_t *result)
{
int retval = -1;
pid_array_t to_process = {0};
if (pid_array_init(&to_process) < 0) {
goto done;
}
if (pid_array_append(&to_process, target_pid) < 0) {
goto done;
}
size_t process_idx = 0;
while (process_idx < to_process.count) {
pid_t current_pid = to_process.pids[process_idx++];
for (size_t i = 0; i < pid_count; i++) {
if (ppids[i] != current_pid) {
continue;
}
pid_t child_pid = all_pids[i];
if (pid_array_contains(result, child_pid)) {
continue;
}
if (pid_array_append(result, child_pid) < 0) {
goto done;
}
if (recursive && pid_array_append(&to_process, child_pid) < 0) {
goto done;
}
}
if (!recursive) {
break;
}
}
retval = 0;
done:
pid_array_cleanup(&to_process);
return retval;
}
/* ============================================================================
* LINUX IMPLEMENTATION
* ============================================================================ */
#if defined(__linux__)
/* Parse /proc/{pid}/stat to get parent PID */
static pid_t
get_ppid_linux(pid_t pid)
{
char stat_path[64];
char buffer[2048];
snprintf(stat_path, sizeof(stat_path), "/proc/%d/stat", (int)pid);
int fd = open(stat_path, O_RDONLY);
if (fd == -1) {
return -1;
}
ssize_t n = read(fd, buffer, sizeof(buffer) - 1);
close(fd);
if (n <= 0) {
return -1;
}
buffer[n] = '\0';
/* Find closing paren of comm field - stat format: pid (comm) state ppid ... */
char *p = strrchr(buffer, ')');
if (!p) {
return -1;
}
/* Skip ") " with bounds checking */
char *end = buffer + n;
p += 2;
if (p >= end) {
return -1;
}
if (*p == ' ') {
p++;
if (p >= end) {
return -1;
}
}
/* Parse: state ppid */
char state;
int ppid;
if (sscanf(p, "%c %d", &state, &ppid) != 2) {
return -1;
}
return (pid_t)ppid;
}
static int
get_child_pids_platform(pid_t target_pid, int recursive, pid_array_t *result)
{
int retval = -1;
pid_array_t all_pids = {0};
pid_array_t ppids = {0};
DIR *proc_dir = NULL;
if (pid_array_init(&all_pids) < 0) {
goto done;
}
if (pid_array_init(&ppids) < 0) {
goto done;
}
proc_dir = opendir("/proc");
if (!proc_dir) {
PyErr_SetFromErrnoWithFilename(PyExc_OSError, "/proc");
goto done;
}
/* Single pass: collect PIDs and their PPIDs together */
struct dirent *entry;
while ((entry = readdir(proc_dir)) != NULL) {
/* Skip non-numeric entries (also skips . and ..) */
if (entry->d_name[0] < '1' || entry->d_name[0] > '9') {
continue;
}
char *endptr;
long pid_long = strtol(entry->d_name, &endptr, 10);
if (*endptr != '\0' || pid_long <= 0) {
continue;
}
pid_t pid = (pid_t)pid_long;
pid_t ppid = get_ppid_linux(pid);
if (ppid < 0) {
continue;
}
if (pid_array_append(&all_pids, pid) < 0 ||
pid_array_append(&ppids, ppid) < 0) {
goto done;
}
}
closedir(proc_dir);
proc_dir = NULL;
if (find_children_bfs(target_pid, recursive,
all_pids.pids, ppids.pids, all_pids.count,
result) < 0) {
goto done;
}
retval = 0;
done:
if (proc_dir) {
closedir(proc_dir);
}
pid_array_cleanup(&all_pids);
pid_array_cleanup(&ppids);
return retval;
}
#endif /* __linux__ */
/* ============================================================================
* MACOS IMPLEMENTATION
* ============================================================================ */
#if defined(__APPLE__) && TARGET_OS_OSX
#include <sys/proc_info.h>
static int
get_child_pids_platform(pid_t target_pid, int recursive, pid_array_t *result)
{
int retval = -1;
pid_t *pid_list = NULL;
pid_t *ppids = NULL;
/* Get count of all PIDs */
int n_pids = proc_listallpids(NULL, 0);
if (n_pids <= 0) {
PyErr_SetString(PyExc_OSError, "Failed to get process count");
goto done;
}
/* Allocate buffer for PIDs (add some slack for new processes) */
int buffer_size = n_pids + 64;
pid_list = (pid_t *)PyMem_Malloc(buffer_size * sizeof(pid_t));
if (!pid_list) {
PyErr_NoMemory();
goto done;
}
/* Get actual PIDs */
int actual = proc_listallpids(pid_list, buffer_size * sizeof(pid_t));
if (actual <= 0) {
PyErr_SetString(PyExc_OSError, "Failed to list PIDs");
goto done;
}
/* Build pid -> ppid mapping */
ppids = (pid_t *)PyMem_Malloc(actual * sizeof(pid_t));
if (!ppids) {
PyErr_NoMemory();
goto done;
}
/* Get parent PIDs for each process */
int valid_count = 0;
for (int i = 0; i < actual; i++) {
struct proc_bsdinfo proc_info;
int ret = proc_pidinfo(pid_list[i], PROC_PIDTBSDINFO, 0,
&proc_info, sizeof(proc_info));
if (ret != sizeof(proc_info)) {
continue;
}
pid_list[valid_count] = pid_list[i];
ppids[valid_count] = proc_info.pbi_ppid;
valid_count++;
}
if (find_children_bfs(target_pid, recursive,
pid_list, ppids, valid_count,
result) < 0) {
goto done;
}
retval = 0;
done:
PyMem_Free(pid_list);
PyMem_Free(ppids);
return retval;
}
#endif /* __APPLE__ && TARGET_OS_OSX */
/* ============================================================================
* WINDOWS IMPLEMENTATION
* ============================================================================ */
#ifdef MS_WINDOWS
static int
get_child_pids_platform(pid_t target_pid, int recursive, pid_array_t *result)
{
int retval = -1;
pid_array_t all_pids = {0};
pid_array_t ppids = {0};
HANDLE snapshot = INVALID_HANDLE_VALUE;
snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0);
if (snapshot == INVALID_HANDLE_VALUE) {
PyErr_SetFromWindowsErr(0);
goto done;
}
if (pid_array_init(&all_pids) < 0) {
goto done;
}
if (pid_array_init(&ppids) < 0) {
goto done;
}
/* Single pass: collect PIDs and PPIDs together */
PROCESSENTRY32 pe;
pe.dwSize = sizeof(PROCESSENTRY32);
if (Process32First(snapshot, &pe)) {
do {
if (pid_array_append(&all_pids, (pid_t)pe.th32ProcessID) < 0 ||
pid_array_append(&ppids, (pid_t)pe.th32ParentProcessID) < 0) {
goto done;
}
} while (Process32Next(snapshot, &pe));
}
CloseHandle(snapshot);
snapshot = INVALID_HANDLE_VALUE;
if (find_children_bfs(target_pid, recursive,
all_pids.pids, ppids.pids, all_pids.count,
result) < 0) {
goto done;
}
retval = 0;
done:
if (snapshot != INVALID_HANDLE_VALUE) {
CloseHandle(snapshot);
}
pid_array_cleanup(&all_pids);
pid_array_cleanup(&ppids);
return retval;
}
#endif /* MS_WINDOWS */
/* ============================================================================
* UNSUPPORTED PLATFORM STUB
* ============================================================================ */
#if !defined(__linux__) && !(defined(__APPLE__) && TARGET_OS_OSX) && !defined(MS_WINDOWS)
static int
get_child_pids_platform(pid_t target_pid, int recursive, pid_array_t *result)
{
PyErr_SetString(PyExc_NotImplementedError,
"Subprocess enumeration not supported on this platform");
return -1;
}
#endif
/* ============================================================================
* PUBLIC API
* ============================================================================ */
PyObject *
enumerate_child_pids(pid_t target_pid, int recursive)
{
pid_array_t result;
if (pid_array_init(&result) < 0) {
return NULL;
}
if (get_child_pids_platform(target_pid, recursive, &result) < 0) {
pid_array_cleanup(&result);
return NULL;
}
/* Convert to Python list */
PyObject *list = PyList_New(result.count);
if (list == NULL) {
pid_array_cleanup(&result);
return NULL;
}
for (size_t i = 0; i < result.count; i++) {
PyObject *pid_obj = PyLong_FromLong((long)result.pids[i]);
if (pid_obj == NULL) {
Py_DECREF(list);
pid_array_cleanup(&result);
return NULL;
}
PyList_SET_ITEM(list, i, pid_obj);
}
pid_array_cleanup(&result);
return list;
}