[3.11] gh-101072: support default and kw default in PyEval_EvalCodeEx for 3.11+ (GH-101127) (#101636)

Co-authored-by: Łukasz Langa <lukasz@langa.pl>
Co-authored-by: Matthieu Dartiailh <m.dartiailh@gmail.com>

Lib/test/test_capi/test_eval_code_ex.py

@@ -0,0 +1,56 @@
+import unittest
+
+from test.support import import_helper
+
+
+# Skip this test if the _testcapi module isn't available.
+_testcapi = import_helper.import_module('_testcapi')
+
+
+class PyEval_EvalCodeExTests(unittest.TestCase):
+
+    def test_simple(self):
+        def f():
+            return a
+
+        self.assertEqual(_testcapi.eval_code_ex(f.__code__, dict(a=1)), 1)
+
+    # Need to force the compiler to use LOAD_NAME
+    # def test_custom_locals(self):
+    #     def f():
+    #         return
+
+    def test_with_args(self):
+        def f(a, b, c):
+            return a
+
+        self.assertEqual(_testcapi.eval_code_ex(f.__code__, {}, {}, (1, 2, 3)), 1)
+
+    def test_with_kwargs(self):
+        def f(a, b, c):
+            return a
+
+        self.assertEqual(_testcapi.eval_code_ex(f.__code__, {}, {}, (), dict(a=1, b=2, c=3)), 1)
+
+    def test_with_default(self):
+        def f(a):
+            return a
+
+        self.assertEqual(_testcapi.eval_code_ex(f.__code__, {}, {}, (), {}, (1,)), 1)
+
+    def test_with_kwarg_default(self):
+        def f(*, a):
+            return a
+
+        self.assertEqual(_testcapi.eval_code_ex(f.__code__, {}, {}, (), {}, (), dict(a=1)), 1)
+
+    def test_with_closure(self):
+        a = 1
+        def f():
+            return a
+
+        self.assertEqual(_testcapi.eval_code_ex(f.__code__, {}, {}, (), {}, (), {}, f.__closure__), 1)
+
+
+if __name__ == "__main__":
+    unittest.main()

Misc/NEWS.d/next/Core and Builtins/2023-02-06-20-13-36.gh-issue-92173.RQE0mk.rst

@@ -0,0 +1,2 @@
+Fix the ``defs`` and ``kwdefs`` arguments to :c:func:`PyEval_EvalCodeEx`
+and a reference leak in that function.

Modules/_testcapimodule.c

@@ -6058,6 +6058,144 @@ eval_get_func_desc(PyObject *self, PyObject *func)
     return PyUnicode_FromString(PyEval_GetFuncDesc(func));
 }
 
+static PyObject *
+eval_eval_code_ex(PyObject *mod, PyObject *pos_args)
+{
+    PyObject *result = NULL;
+    PyObject *code;
+    PyObject *globals;
+    PyObject *locals = NULL;
+    PyObject *args = NULL;
+    PyObject *kwargs = NULL;
+    PyObject *defaults = NULL;
+    PyObject *kw_defaults = NULL;
+    PyObject *closure = NULL;
+
+    PyObject **c_kwargs = NULL;
+
+    if (!PyArg_UnpackTuple(pos_args,
+                           "eval_code_ex",
+                           2,
+                           8,
+                           &code,
+                           &globals,
+                           &locals,
+                           &args,
+                           &kwargs,
+                           &defaults,
+                           &kw_defaults,
+                           &closure))
+    {
+        goto exit;
+    }
+
+    if (!PyCode_Check(code)) {
+        PyErr_SetString(PyExc_TypeError,
+                        "code must be a Python code object");
+        goto exit;
+    }
+
+    if (!PyDict_Check(globals)) {
+        PyErr_SetString(PyExc_TypeError, "globals must be a dict");
+        goto exit;
+    }
+
+    if (locals && !PyMapping_Check(locals)) {
+        PyErr_SetString(PyExc_TypeError, "locals must be a mapping");
+        goto exit;
+    }
+    if (locals == Py_None) {
+        locals = NULL;
+    }
+
+    PyObject **c_args = NULL;
+    Py_ssize_t c_args_len = 0;
+
+    if (args)
+    {
+        if (!PyTuple_Check(args)) {
+            PyErr_SetString(PyExc_TypeError, "args must be a tuple");
+            goto exit;
+        } else {
+            c_args = &PyTuple_GET_ITEM(args, 0);
+            c_args_len = PyTuple_Size(args);
+        }
+    }
+
+    Py_ssize_t c_kwargs_len = 0;
+
+    if (kwargs)
+    {
+        if (!PyDict_Check(kwargs)) {
+            PyErr_SetString(PyExc_TypeError, "keywords must be a dict");
+            goto exit;
+        } else {
+            c_kwargs_len = PyDict_Size(kwargs);
+            if (c_kwargs_len > 0) {
+                c_kwargs = PyMem_NEW(PyObject*, 2 * c_kwargs_len);
+                if (!c_kwargs) {
+                    PyErr_NoMemory();
+                    goto exit;
+                }
+
+                Py_ssize_t i = 0;
+                Py_ssize_t pos = 0;
+
+                while (PyDict_Next(kwargs,
+                                   &pos,
+                                   &c_kwargs[i],
+                                   &c_kwargs[i + 1]))
+                {
+                    i += 2;
+                }
+                c_kwargs_len = i / 2;
+                /* XXX This is broken if the caller deletes dict items! */
+            }
+        }
+    }
+
+
+    PyObject **c_defaults = NULL;
+    Py_ssize_t c_defaults_len = 0;
+
+    if (defaults && PyTuple_Check(defaults)) {
+        c_defaults = &PyTuple_GET_ITEM(defaults, 0);
+        c_defaults_len = PyTuple_Size(defaults);
+    }
+
+    if (kw_defaults && !PyDict_Check(kw_defaults)) {
+        PyErr_SetString(PyExc_TypeError, "kw_defaults must be a dict");
+        goto exit;
+    }
+
+    if (closure && !PyTuple_Check(closure)) {
+        PyErr_SetString(PyExc_TypeError, "closure must be a tuple of cells");
+        goto exit;
+    }
+
+
+    result = PyEval_EvalCodeEx(
+        code,
+        globals,
+        locals,
+        c_args,
+        c_args_len,
+        c_kwargs,
+        c_kwargs_len,
+        c_defaults,
+        c_defaults_len,
+        kw_defaults,
+        closure
+    );
+
+exit:
+    if (c_kwargs) {
+        PyMem_DEL(c_kwargs);
+    }
+
+    return result;
+}
+
 static PyObject *
 get_feature_macros(PyObject *self, PyObject *Py_UNUSED(args))
 {
@@ -6402,6 +6540,7 @@ static PyMethodDef TestMethods[] = {
     {"set_exc_info",            test_set_exc_info,               METH_VARARGS},
     {"argparsing",              argparsing,                      METH_VARARGS},
     {"code_newempty",           code_newempty,                   METH_VARARGS},
+    {"eval_code_ex",            eval_eval_code_ex,               METH_VARARGS},
     {"make_exception_with_doc", _PyCFunction_CAST(make_exception_with_doc),
      METH_VARARGS | METH_KEYWORDS},
     {"make_memoryview_from_NULL_pointer", make_memoryview_from_NULL_pointer,

Objects/funcobject.c

@@ -27,8 +27,10 @@ _PyFunction_FromConstructor(PyFrameConstructor *constr)
     op->func_qualname = constr->fc_qualname;
     Py_INCREF(constr->fc_code);
     op->func_code = constr->fc_code;
-    op->func_defaults = NULL;
-    op->func_kwdefaults = NULL;
+    Py_XINCREF(constr->fc_defaults);
+    op->func_defaults = constr->fc_defaults;
+    Py_XINCREF(constr->fc_kwdefaults);
+    op->func_kwdefaults = constr->fc_kwdefaults;
     Py_XINCREF(constr->fc_closure);
     op->func_closure = constr->fc_closure;
     Py_INCREF(Py_None);

Python/ceval.c

@@ -6489,10 +6489,6 @@ PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals,
         }
         allargs = newargs;
     }
-    for (int i = 0; i < kwcount; i++) {
-        Py_INCREF(kws[2*i]);
-        PyTuple_SET_ITEM(kwnames, i, kws[2*i]);
-    }
     PyFrameConstructor constr = {
         .fc_globals = globals,
         .fc_builtins = builtins,