[3.14] gh-132775: Use _PyCode GetScriptXIData() (gh-134515)

(cherry picked from commit 09e72cf091, AKA gh-134511)

Co-authored-by: Eric Snow <ericsnowcurrently@gmail.com>

Lib/test/support/interpreters/channels.py

@@ -69,7 +69,7 @@ def list_all():
         if not hasattr(send, '_unboundop'):
             send._set_unbound(unboundop)
         else:
-            assert send._unbound[0] == op
+            assert send._unbound[0] == unboundop
         channels.append(chan)
     return channels
 

Lib/test/test__interpreters.py

@@ -474,13 +474,15 @@ def setUp(self):
 
     def test_signatures(self):
         # See https://github.com/python/cpython/issues/126654
-        msg = "expected 'shared' to be a dict"
+        msg = r'_interpreters.exec\(\) argument 3 must be dict, not int'
         with self.assertRaisesRegex(TypeError, msg):
             _interpreters.exec(self.id, 'a', 1)
         with self.assertRaisesRegex(TypeError, msg):
             _interpreters.exec(self.id, 'a', shared=1)
+        msg = r'_interpreters.run_string\(\) argument 3 must be dict, not int'
         with self.assertRaisesRegex(TypeError, msg):
             _interpreters.run_string(self.id, 'a', shared=1)
+        msg = r'_interpreters.run_func\(\) argument 3 must be dict, not int'
         with self.assertRaisesRegex(TypeError, msg):
             _interpreters.run_func(self.id, lambda: None, shared=1)
 
@@ -952,7 +954,8 @@ def test_invalid_syntax(self):
             """)
 
         with self.subTest('script'):
-            self.assert_run_failed(SyntaxError, script)
+            with self.assertRaises(SyntaxError):
+                _interpreters.run_string(self.id, script)
 
         with self.subTest('module'):
             modname = 'spam_spam_spam'
@@ -1019,12 +1022,19 @@ def script():
             with open(w, 'w', encoding="utf-8") as spipe:
                 with contextlib.redirect_stdout(spipe):
                     print('it worked!', end='')
+        failed = None
         def f():
-            _interpreters.set___main___attrs(self.id, dict(w=w))
+            nonlocal failed
-            _interpreters.run_func(self.id, script)
+            try:
+                _interpreters.set___main___attrs(self.id, dict(w=w))
+                _interpreters.run_func(self.id, script)
+            except Exception as exc:
+                failed = exc
         t = threading.Thread(target=f)
         t.start()
         t.join()
+        if failed:
+            raise Exception from failed
 
         with open(r, encoding="utf-8") as outfile:
             out = outfile.read()
@@ -1053,19 +1063,16 @@ def test_closure(self):
         spam = True
         def script():
             assert spam
-
+        with self.assertRaises(ValueError):
-        with self.assertRaises(TypeError):
             _interpreters.run_func(self.id, script)
 
-    # XXX This hasn't been fixed yet.
-    @unittest.expectedFailure
     def test_return_value(self):
         def script():
             return 'spam'
         with self.assertRaises(ValueError):
             _interpreters.run_func(self.id, script)
 
-    @unittest.skip("we're not quite there yet")
+#    @unittest.skip("we're not quite there yet")
     def test_args(self):
         with self.subTest('args'):
             def script(a, b=0):

Lib/test/test_interpreters/test_api.py

@@ -839,9 +839,16 @@ def test_bad_script(self):
             interp.exec(10)
 
     def test_bytes_for_script(self):
+        r, w = self.pipe()
+        RAN = b'R'
+        DONE = b'D'
         interp = interpreters.create()
-        with self.assertRaises(TypeError):
+        interp.exec(f"""if True:
-            interp.exec(b'print("spam")')
+            import os
+            os.write({w}, {RAN!r})
+            """)
+        os.write(w, DONE)
+        self.assertEqual(os.read(r, 1), RAN)
 
     def test_with_background_threads_still_running(self):
         r_interp, w_interp = self.pipe()
@@ -1010,8 +1017,6 @@ def test_call(self):
 
         for i, (callable, args, kwargs) in enumerate([
             (call_func_noop, (), {}),
-            (call_func_return_shareable, (), {}),
-            (call_func_return_not_shareable, (), {}),
             (Spam.noop, (), {}),
         ]):
             with self.subTest(f'success case #{i+1}'):
@@ -1036,6 +1041,8 @@ def test_call(self):
             (call_func_complex, ('custom', 'spam!'), {}),
             (call_func_complex, ('custom-inner', 'eggs!'), {}),
             (call_func_complex, ('???',), {'exc': ValueError('spam')}),
+            (call_func_return_shareable, (), {}),
+            (call_func_return_not_shareable, (), {}),
         ]):
             with self.subTest(f'invalid case #{i+1}'):
                 with self.assertRaises(Exception):
@@ -1051,8 +1058,6 @@ def test_call_in_thread(self):
 
         for i, (callable, args, kwargs) in enumerate([
             (call_func_noop, (), {}),
-            (call_func_return_shareable, (), {}),
-            (call_func_return_not_shareable, (), {}),
             (Spam.noop, (), {}),
         ]):
             with self.subTest(f'success case #{i+1}'):
@@ -1079,6 +1084,8 @@ def test_call_in_thread(self):
             (call_func_complex, ('custom', 'spam!'), {}),
             (call_func_complex, ('custom-inner', 'eggs!'), {}),
             (call_func_complex, ('???',), {'exc': ValueError('spam')}),
+            (call_func_return_shareable, (), {}),
+            (call_func_return_not_shareable, (), {}),
         ]):
             with self.subTest(f'invalid case #{i+1}'):
                 if args or kwargs:
@@ -1618,8 +1625,8 @@ def test_exec(self):
     def test_call(self):
         with self.subTest('no args'):
             interpid = _interpreters.create()
-            exc = _interpreters.call(interpid, call_func_return_shareable)
+            with self.assertRaises(ValueError):
-            self.assertIs(exc, None)
+                _interpreters.call(interpid, call_func_return_shareable)
 
         with self.subTest('uncaught exception'):
             interpid = _interpreters.create()

Modules/_interpretersmodule.c

@@ -9,7 +9,6 @@
 #include "pycore_code.h"          // _PyCode_HAS_EXECUTORS()
 #include "pycore_crossinterp.h"   // _PyXIData_t
 #include "pycore_pyerrors.h"      // _PyErr_GetRaisedException()
-#include "pycore_function.h"      // _PyFunction_VerifyStateless()
 #include "pycore_interp.h"        // _PyInterpreterState_IDIncref()
 #include "pycore_modsupport.h"    // _PyArg_BadArgument()
 #include "pycore_namespace.h"     // _PyNamespace_New()
@@ -361,81 +360,6 @@ _get_current_xibufferview_type(void)
 }
 
 
-/* Python code **************************************************************/
-
-static const char *
-check_code_str(PyUnicodeObject *text)
-{
-    assert(text != NULL);
-    if (PyUnicode_GET_LENGTH(text) == 0) {
-        return "too short";
-    }
-
-    // XXX Verify that it parses?
-
-    return NULL;
-}
-
-#ifndef NDEBUG
-static int
-code_has_args(PyCodeObject *code)
-{
-    assert(code != NULL);
-    return (code->co_argcount > 0
-        || code->co_posonlyargcount > 0
-        || code->co_kwonlyargcount > 0
-        || code->co_flags & (CO_VARARGS | CO_VARKEYWORDS));
-}
-#endif
-
-#define RUN_TEXT 1
-#define RUN_CODE 2
-
-static const char *
-get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p)
-{
-    const char *codestr = NULL;
-    Py_ssize_t len = -1;
-    PyObject *bytes_obj = NULL;
-    int flags = 0;
-
-    if (PyUnicode_Check(arg)) {
-        assert(PyUnicode_Check(arg)
-               && (check_code_str((PyUnicodeObject *)arg) == NULL));
-        codestr = PyUnicode_AsUTF8AndSize(arg, &len);
-        if (codestr == NULL) {
-            return NULL;
-        }
-        if (strlen(codestr) != (size_t)len) {
-            PyErr_SetString(PyExc_ValueError,
-                            "source code string cannot contain null bytes");
-            return NULL;
-        }
-        flags = RUN_TEXT;
-    }
-    else {
-        assert(PyCode_Check(arg));
-        assert(_PyCode_VerifyStateless(
-            PyThreadState_Get(), (PyCodeObject *)arg, NULL, NULL, NULL) == 0);
-        assert(!code_has_args((PyCodeObject *)arg));
-        flags = RUN_CODE;
-
-        // Serialize the code object.
-        bytes_obj = PyMarshal_WriteObjectToString(arg, Py_MARSHAL_VERSION);
-        if (bytes_obj == NULL) {
-            return NULL;
-        }
-        codestr = PyBytes_AS_STRING(bytes_obj);
-        len = PyBytes_GET_SIZE(bytes_obj);
-    }
-
-    *flags_p = flags;
-    *bytes_p = bytes_obj;
-    *len_p = len;
-    return codestr;
-}
-
-
 /* interpreter-specific code ************************************************/
 
 static int
@@ -499,22 +423,14 @@ config_from_object(PyObject *configobj, PyInterpreterConfig *config)
 
 
 static int
-_run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags)
+_run_script(_PyXIData_t *script, PyObject *ns)
 {
-    PyObject *result = NULL;
+    PyObject *code = _PyXIData_NewObject(script);
-    if (flags & RUN_TEXT) {
+    if (code == NULL) {
-        result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL);
+        return -1;
-    }
-    else if (flags & RUN_CODE) {
-        PyObject *code = PyMarshal_ReadObjectFromString(codestr, codestrlen);
-        if (code != NULL) {
-            result = PyEval_EvalCode(code, ns, ns);
-            Py_DECREF(code);
-        }
-    }
-    else {
-        Py_UNREACHABLE();
     }
+    PyObject *result = PyEval_EvalCode(code, ns, ns);
+    Py_DECREF(code);
     if (result == NULL) {
         return -1;
     }
@@ -523,17 +439,16 @@ _run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags)
 }
 
 static int
-_run_in_interpreter(PyInterpreterState *interp,
+_exec_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp,
-                    const char *codestr, Py_ssize_t codestrlen,
+                    _PyXIData_t *script, PyObject *shareables,
-                    PyObject *shareables, int flags,
                     PyObject **p_excinfo)
 {
-    assert(!PyErr_Occurred());
+    assert(!_PyErr_Occurred(tstate));
     _PyXI_session session = {0};
 
     // Prep and switch interpreters.
     if (_PyXI_Enter(&session, interp, shareables) < 0) {
-        if (PyErr_Occurred()) {
+        if (_PyErr_Occurred(tstate)) {
             // If an error occured at this step, it means that interp
             // was not prepared and switched.
             return -1;
@@ -548,7 +463,7 @@ _run_in_interpreter(PyInterpreterState *interp,
     }
 
     // Run the script.
-    int res = _run_script(session.main_ns, codestr, codestrlen, flags);
+    int res = _run_script(script, session.main_ns);
 
     // Clean up and switch back.
     _PyXI_Exit(&session);
@@ -935,104 +850,23 @@ PyDoc_STRVAR(set___main___attrs_doc,
 Bind the given attributes in the interpreter's __main__ module.");
 
 
-static PyUnicodeObject *
+static void
-convert_script_arg(PyThreadState *tstate,
+unwrap_not_shareable(PyThreadState *tstate)
-                   PyObject *arg, const char *fname, const char *displayname,
-                   const char *expected)
 {
-    PyUnicodeObject *str = NULL;
+    PyObject *exctype = _PyXIData_GetNotShareableErrorType(tstate);
-    if (PyUnicode_CheckExact(arg)) {
+    if (!_PyErr_ExceptionMatches(tstate, exctype)) {
-        str = (PyUnicodeObject *)Py_NewRef(arg);
+        return;
     }
-    else if (PyUnicode_Check(arg)) {
-        // XXX str = PyUnicode_FromObject(arg);
-        str = (PyUnicodeObject *)Py_NewRef(arg);
-    }
-    else {
-        _PyArg_BadArgument(fname, displayname, expected, arg);
-        return NULL;
-    }
-
-    const char *err = check_code_str(str);
-    if (err != NULL) {
-        Py_DECREF(str);
-        _PyErr_Format(tstate, PyExc_ValueError,
-                      "%.200s(): bad script text (%s)", fname, err);
-        return NULL;
-    }
-
-    return str;
-}
-
-static PyCodeObject *
-convert_code_arg(PyThreadState *tstate,
-                 PyObject *arg, const char *fname, const char *displayname,
-                 const char *expected)
-{
-    PyObject *cause;
-    PyCodeObject *code = NULL;
-    if (PyFunction_Check(arg)) {
-        // For now we allow globals, so we can't use
-        // _PyFunction_VerifyStateless().
-        PyObject *codeobj = PyFunction_GetCode(arg);
-        if (_PyCode_VerifyStateless(
-                    tstate, (PyCodeObject *)codeobj, NULL, NULL, NULL) < 0) {
-            goto chained;
-        }
-        code = (PyCodeObject *)Py_NewRef(codeobj);
-    }
-    else if (PyCode_Check(arg)) {
-        if (_PyCode_VerifyStateless(
-                    tstate, (PyCodeObject *)arg, NULL, NULL, NULL) < 0) {
-            goto chained;
-        }
-        code = (PyCodeObject *)Py_NewRef(arg);
-    }
-    else {
-        _PyArg_BadArgument(fname, displayname, expected, arg);
-        return NULL;
-    }
-
-    return code;
-
-chained:
-    cause = _PyErr_GetRaisedException(tstate);
-    assert(cause != NULL);
-    _PyArg_BadArgument(fname, displayname, expected, arg);
     PyObject *exc = _PyErr_GetRaisedException(tstate);
-    PyException_SetCause(exc, cause);
+    PyObject *cause = PyException_GetCause(exc);
+    if (cause != NULL) {
+        Py_DECREF(exc);
+        exc = cause;
+    }
+    else {
+        assert(PyException_GetContext(exc) == NULL);
+    }
     _PyErr_SetRaisedException(tstate, exc);
-    return NULL;
-}
-
-static int
-_interp_exec(PyObject *self, PyInterpreterState *interp,
-             PyObject *code_arg, PyObject *shared_arg, PyObject **p_excinfo)
-{
-    if (shared_arg != NULL && !PyDict_CheckExact(shared_arg)) {
-        PyErr_SetString(PyExc_TypeError, "expected 'shared' to be a dict");
-        return -1;
-    }
-
-    // Extract code.
-    Py_ssize_t codestrlen = -1;
-    PyObject *bytes_obj = NULL;
-    int flags = 0;
-    const char *codestr = get_code_str(code_arg,
-                                       &codestrlen, &bytes_obj, &flags);
-    if (codestr == NULL) {
-        return -1;
-    }
-
-    // Run the code in the interpreter.
-    int res = _run_in_interpreter(interp, codestr, codestrlen,
-                                  shared_arg, flags, p_excinfo);
-    Py_XDECREF(bytes_obj);
-    if (res < 0) {
-        return -1;
-    }
-
-    return 0;
 }
 
 static PyObject *
@@ -1045,8 +879,9 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds)
     PyObject *shared = NULL;
     int restricted = 0;
     if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "OO|O$p:" FUNCNAME, kwlist,
+                                     "OO|O!$p:" FUNCNAME, kwlist,
-                                     &id, &code, &shared, &restricted))
+                                     &id, &code, &PyDict_Type, &shared,
+                                     &restricted))
     {
         return NULL;
     }
@@ -1058,22 +893,17 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds)
         return NULL;
     }
 
-    const char *expected = "a string, a function, or a code object";
+    // We don't need the script to be "pure", which means it can use
-    if (PyUnicode_Check(code)) {
+    // global variables.  They will be resolved against __main__.
-        code = (PyObject *)convert_script_arg(tstate, code, FUNCNAME,
+    _PyXIData_t xidata = {0};
-                                              "argument 2", expected);
+    if (_PyCode_GetScriptXIData(tstate, code, &xidata) < 0) {
-    }
+        unwrap_not_shareable(tstate);
-    else {
-        code = (PyObject *)convert_code_arg(tstate, code, FUNCNAME,
-                                            "argument 2", expected);
-    }
-    if (code == NULL) {
         return NULL;
     }
 
     PyObject *excinfo = NULL;
-    int res = _interp_exec(self, interp, code, shared, &excinfo);
+    int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &excinfo);
-    Py_DECREF(code);
+    _PyXIData_Release(&xidata);
     if (res < 0) {
         assert((excinfo == NULL) != (PyErr_Occurred() == NULL));
         return excinfo;
@@ -1109,8 +939,9 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds)
     PyObject *shared = NULL;
     int restricted = 0;
     if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "OU|O$p:" FUNCNAME, kwlist,
+                                     "OU|O!$p:" FUNCNAME, kwlist,
-                                     &id, &script, &shared, &restricted))
+                                     &id, &script, &PyDict_Type, &shared,
+                                     &restricted))
     {
         return NULL;
     }
@@ -1122,15 +953,20 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds)
         return NULL;
     }
 
-    script = (PyObject *)convert_script_arg(tstate, script, FUNCNAME,
+    if (PyFunction_Check(script) || PyCode_Check(script)) {
-                                            "argument 2", "a string");
+        _PyArg_BadArgument(FUNCNAME, "argument 2", "a string", script);
-    if (script == NULL) {
+        return NULL;
+    }
+
+    _PyXIData_t xidata = {0};
+    if (_PyCode_GetScriptXIData(tstate, script, &xidata) < 0) {
+        unwrap_not_shareable(tstate);
         return NULL;
     }
 
     PyObject *excinfo = NULL;
-    int res = _interp_exec(self, interp, script, shared, &excinfo);
+    int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &excinfo);
-    Py_DECREF(script);
+    _PyXIData_Release(&xidata);
     if (res < 0) {
         assert((excinfo == NULL) != (PyErr_Occurred() == NULL));
         return excinfo;
@@ -1156,8 +992,9 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds)
     PyObject *shared = NULL;
     int restricted = 0;
     if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "OO|O$p:" FUNCNAME, kwlist,
+                                     "OO|O!$p:" FUNCNAME, kwlist,
-                                     &id, &func, &shared, &restricted))
+                                     &id, &func, &PyDict_Type, &shared,
+                                     &restricted))
     {
         return NULL;
     }
@@ -1169,16 +1006,29 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds)
         return NULL;
     }
 
-    PyCodeObject *code = convert_code_arg(tstate, func, FUNCNAME,
+    // We don't worry about checking globals.  They will be resolved
-                                          "argument 2",
+    // against __main__.
-                                          "a function or a code object");
+    PyObject *code;
-    if (code == NULL) {
+    if (PyFunction_Check(func)) {
+        code = PyFunction_GET_CODE(func);
+    }
+    else if (PyCode_Check(func)) {
+        code = func;
+    }
+    else {
+        _PyArg_BadArgument(FUNCNAME, "argument 2", "a function", func);
+        return NULL;
+    }
+
+    _PyXIData_t xidata = {0};
+    if (_PyCode_GetScriptXIData(tstate, code, &xidata) < 0) {
+        unwrap_not_shareable(tstate);
         return NULL;
     }
 
     PyObject *excinfo = NULL;
-    int res = _interp_exec(self, interp, (PyObject *)code, shared, &excinfo);
+    int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &excinfo);
-    Py_DECREF(code);
+    _PyXIData_Release(&xidata);
     if (res < 0) {
         assert((excinfo == NULL) != (PyErr_Occurred() == NULL));
         return excinfo;
@@ -1231,15 +1081,15 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds)
         return NULL;
     }
 
-    PyObject *code = (PyObject *)convert_code_arg(tstate, callable, FUNCNAME,
+    _PyXIData_t xidata = {0};
-                                                  "argument 2", "a function");
+    if (_PyCode_GetPureScriptXIData(tstate, callable, &xidata) < 0) {
-    if (code == NULL) {
+        unwrap_not_shareable(tstate);
         return NULL;
     }
 
     PyObject *excinfo = NULL;
-    int res = _interp_exec(self, interp, code, NULL, &excinfo);
+    int res = _exec_in_interpreter(tstate, interp, &xidata, NULL, &excinfo);
-    Py_DECREF(code);
+    _PyXIData_Release(&xidata);
     if (res < 0) {
         assert((excinfo == NULL) != (PyErr_Occurred() == NULL));
         return excinfo;