Add _PyObject_FastCallKeywords()

Issue #27830: Similar to _PyObject_FastCallDict(), but keyword arguments are
also passed in the same C array than positional arguments, rather than being
passed as a Python dict.

Include/abstract.h

@@ -292,6 +292,23 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
 #define _PyObject_CallArg1(func, arg) \
     _PyObject_FastCall((func), &(arg), 1)
 
+     /* Call the callable object func with the "fast call" calling convention:
+        args is a C array for positional arguments followed by (key, value)
+        pairs for keyword arguments.
+
+        nargs is the number of positional parameters at the beginning of stack.
+        nkwargs is the number of (key, value) pairs at the end of stack.
+
+        If nargs and nkwargs are equal to zero, stack can be NULL.
+
+        Return the result on success. Raise an exception and return NULL on
+        error. */
+     PyAPI_FUNC(PyObject *) _PyObject_FastCallKeywords(
+                                PyObject *func,
+                                PyObject **stack,
+                                Py_ssize_t nargs,
+                                Py_ssize_t nkwargs);
+
      PyAPI_FUNC(PyObject *) _Py_CheckFunctionResult(PyObject *func,
                                                     PyObject *result,
                                                     const char *where);

Include/funcobject.h

@@ -64,6 +64,12 @@ PyAPI_FUNC(PyObject *) _PyFunction_FastCallDict(
     PyObject **args,
     Py_ssize_t nargs,
     PyObject *kwargs);
+
+PyAPI_FUNC(PyObject *) _PyFunction_FastCallKeywords(
+    PyObject *func,
+    PyObject **stack,
+    Py_ssize_t nargs,
+    Py_ssize_t nkwargs);
 #endif
 
 /* Macros for direct access to these values. Type checks are *not*

Objects/abstract.c

@@ -2309,6 +2309,85 @@ _PyObject_FastCallDict(PyObject *func, PyObject **args, Py_ssize_t nargs,
     return result;
 }
 
+static PyObject *
+_PyStack_AsDict(PyObject **stack, Py_ssize_t nkwargs, PyObject *func)
+{
+    PyObject *kwdict;
+
+    kwdict = PyDict_New();
+    if (kwdict == NULL) {
+        return NULL;
+    }
+
+    while (--nkwargs >= 0) {
+        int err;
+        PyObject *key = *stack++;
+        PyObject *value = *stack++;
+        if (PyDict_GetItem(kwdict, key) != NULL) {
+            PyErr_Format(PyExc_TypeError,
+                         "%.200s%s got multiple values "
+                         "for keyword argument '%U'",
+                         PyEval_GetFuncName(func),
+                         PyEval_GetFuncDesc(func),
+                         key);
+            Py_DECREF(kwdict);
+            return NULL;
+        }
+
+        err = PyDict_SetItem(kwdict, key, value);
+        if (err) {
+            Py_DECREF(kwdict);
+            return NULL;
+        }
+    }
+    return kwdict;
+}
+
+PyObject *
+_PyObject_FastCallKeywords(PyObject *func, PyObject **stack, Py_ssize_t nargs,
+                           Py_ssize_t nkwargs)
+{
+    PyObject *args, *kwdict, *result;
+
+    /* _PyObject_FastCallKeywords() must not be called with an exception set,
+       because it may clear it (directly or indirectly) and so the
+       caller loses its exception */
+    assert(!PyErr_Occurred());
+
+    assert(func != NULL);
+    assert(nargs >= 0);
+    assert(nkwargs >= 0);
+    assert((nargs == 0 && nkwargs == 0) || stack != NULL);
+
+    if (PyFunction_Check(func)) {
+        /* Fast-path: avoid temporary tuple or dict */
+        return _PyFunction_FastCallKeywords(func, stack, nargs, nkwargs);
+    }
+
+    if (PyCFunction_Check(func) && nkwargs == 0) {
+        return _PyCFunction_FastCallDict(func, args, nargs, NULL);
+    }
+
+    /* Slow-path: build temporary tuple and/or dict */
+    args = _PyStack_AsTuple(stack, nargs);
+
+    if (nkwargs > 0) {
+        kwdict = _PyStack_AsDict(stack + nargs, nkwargs, func);
+        if (kwdict == NULL) {
+            Py_DECREF(args);
+            return NULL;
+        }
+    }
+    else {
+        kwdict = NULL;
+    }
+
+    result = PyObject_Call(func, args, kwdict);
+    Py_DECREF(args);
+    Py_XDECREF(kwdict);
+    return result;
+}
+
 static PyObject*
 call_function_tail(PyObject *callable, PyObject *args)
 {

Python/ceval.c

@@ -113,7 +113,6 @@ static PyObject * call_function(PyObject ***, int, uint64*, uint64*);
 #else
 static PyObject * call_function(PyObject ***, int);
 #endif
-static PyObject * fast_function(PyObject *, PyObject **, Py_ssize_t, Py_ssize_t);
 static PyObject * do_call(PyObject *, PyObject ***, Py_ssize_t, Py_ssize_t);
 static PyObject * ext_do_call(PyObject *, PyObject ***, int, Py_ssize_t, Py_ssize_t);
 static PyObject * update_keyword_args(PyObject *, Py_ssize_t, PyObject ***,
@@ -4767,7 +4766,7 @@ call_function(PyObject ***pp_stack, int oparg
         }
         READ_TIMESTAMP(*pintr0);
         if (PyFunction_Check(func)) {
-            x = fast_function(func, (*pp_stack) - n, nargs, nkwargs);
+            x = _PyFunction_FastCallKeywords(func, (*pp_stack) - n, nargs, nkwargs);
         }
         else {
             x = do_call(func, pp_stack, nargs, nkwargs);
@@ -4780,7 +4779,7 @@ call_function(PyObject ***pp_stack, int oparg
 
     /* Clear the stack of the function object.  Also removes
        the arguments in case they weren't consumed already
-       (fast_function() and err_args() leave them on the stack).
+       (_PyFunction_FastCallKeywords() and err_args() leave them on the stack).
      */
     while ((*pp_stack) > pfunc) {
         w = EXT_POP(*pp_stack);
@@ -4792,7 +4791,7 @@ call_function(PyObject ***pp_stack, int oparg
     return x;
 }
 
-/* The fast_function() function optimize calls for which no argument
+/* The _PyFunction_FastCallKeywords() function optimize calls for which no argument
    tuple is necessary; the objects are passed directly from the stack.
    For the simplest case -- a function that takes only positional
    arguments and is called with only positional arguments -- it
@@ -4840,8 +4839,9 @@ _PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t nargs,
 
 /* Similar to _PyFunction_FastCall() but keywords are passed a (key, value)
    pairs in stack */
-static PyObject *
-fast_function(PyObject *func, PyObject **stack, Py_ssize_t nargs, Py_ssize_t nkwargs)
+PyObject *
+_PyFunction_FastCallKeywords(PyObject *func, PyObject **stack,
+                             Py_ssize_t nargs, Py_ssize_t nkwargs)
 {
     PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func);
     PyObject *globals = PyFunction_GET_GLOBALS(func);
@@ -4850,6 +4850,11 @@ fast_function(PyObject *func, PyObject **stack, Py_ssize_t nargs, Py_ssize_t nkw
     PyObject **d;
     int nd;
 
+    assert(func != NULL);
+    assert(nargs >= 0);
+    assert(nkwargs >= 0);
+    assert((nargs == 0 && nkwargs == 0) || stack != NULL);
+
     PCALL(PCALL_FUNCTION);
     PCALL(PCALL_FAST_FUNCTION);
 
@@ -4902,14 +4907,14 @@ _PyFunction_FastCallDict(PyObject *func, PyObject **args, Py_ssize_t nargs,
     Py_ssize_t nd, nk;
     PyObject *result;
 
-    PCALL(PCALL_FUNCTION);
-    PCALL(PCALL_FAST_FUNCTION);
-
     assert(func != NULL);
     assert(nargs >= 0);
     assert(nargs == 0 || args != NULL);
     assert(kwargs == NULL || PyDict_Check(kwargs));
 
+    PCALL(PCALL_FUNCTION);
+    PCALL(PCALL_FAST_FUNCTION);
+
     if (co->co_kwonlyargcount == 0 &&
         (kwargs == NULL || PyDict_Size(kwargs) == 0) &&
         co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE))