bpo-43693: Un-revert commit f3fa63e. (#26609)

This was reverted in GH-26596 (commit 6d518bb) due to some bad memory accesses.

* Add the MAKE_CELL opcode. (gh-26396)

The memory accesses have been fixed.

https://bugs.python.org/issue43693

Doc/library/dis.rst

@@ -1056,6 +1056,14 @@ All of the following opcodes use their arguments.
    Deletes local ``co_varnames[var_num]``.
 
 
+.. opcode:: MAKE_CELL (i)
+
+   Creates a new cell in slot ``i``.  If that slot is empty then
+   that value is stored into the new cell.
+
+   .. versionadded:: 3.11
+
+
 .. opcode:: LOAD_CLOSURE (i)
 
    Pushes a reference to the cell contained in slot ``i`` of the "fast locals"

Include/cpython/code.h

@@ -2,9 +2,16 @@
 #  error "this header file must not be included directly"
 #endif
 
+/* Each instruction in a code object is a fixed-width value,
+ * currently 2 bytes: 1-byte opcode + 1-byte oparg.  The EXTENDED_ARG
+ * opcode allows for larger values but the current limit is 3 uses
+ * of EXTENDED_ARG (see Python/wordcode_helpers.h), for a maximum
+ * 32-bit value.  This aligns with the note in Python/compile.c
+ * (compiler_addop_i_line) indicating that the max oparg value is
+ * 2**32 - 1, rather than INT_MAX.
+ */
+
 typedef uint16_t _Py_CODEUNIT;
-// Each oparg must fit in the second half of _Py_CODEUNIT, hence 8 bits.
-#define _Py_MAX_OPARG 255
 
 #ifdef WORDS_BIGENDIAN
 #  define _Py_OPCODE(word) ((word) >> 8)

Include/internal/pycore_frame.h

@@ -32,6 +32,8 @@ _PyFrame_GetBuiltins(PyFrameObject *f)
 
 int _PyFrame_TakeLocals(PyFrameObject *f);
 
+PyAPI_FUNC(int) _PyFrame_OpAlreadyRan(PyFrameObject *f, int opcode, int oparg);
+
 #ifdef __cplusplus
 }
 #endif

Include/opcode.h

@@ -113,10 +113,11 @@ extern "C" {
 #define CALL_FUNCTION           131
 #define MAKE_FUNCTION           132
 #define BUILD_SLICE             133
-#define LOAD_CLOSURE            135
+#define MAKE_CELL               135
-#define LOAD_DEREF              136
+#define LOAD_CLOSURE            136
-#define STORE_DEREF             137
+#define LOAD_DEREF              137
-#define DELETE_DEREF            138
+#define STORE_DEREF             138
+#define DELETE_DEREF            139
 #define CALL_FUNCTION_KW        141
 #define CALL_FUNCTION_EX        142
 #define EXTENDED_ARG            144

Lib/importlib/_bootstrap_external.py

@@ -357,6 +357,7 @@ def _write_atomic(path, data, mode=0o666):
 #     Python 3.11a1 3452 (drop nlocals from marshaled code objects)
 #     Python 3.11a1 3453 (add co_fastlocalnames and co_fastlocalkinds)
 #     Python 3.11a1 3454 (compute cell offsets relative to locals bpo-43693)
+#     Python 3.11a1 3455 (add MAKE_CELL bpo-43693)
 
 #
 # MAGIC must change whenever the bytecode emitted by the compiler may no
@@ -366,7 +367,7 @@ def _write_atomic(path, data, mode=0o666):
 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
 # in PC/launcher.c must also be updated.
 
-MAGIC_NUMBER = (3454).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3455).to_bytes(2, 'little') + b'\r\n'
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 
 _PYCACHE = '__pycache__'

Lib/opcode.py

@@ -181,14 +181,16 @@ def jabs_op(name, op):
 def_op('MAKE_FUNCTION', 132)    # Flags
 def_op('BUILD_SLICE', 133)      # Number of items
 
-def_op('LOAD_CLOSURE', 135)
+def_op('MAKE_CELL', 135)
 hasfree.append(135)
-def_op('LOAD_DEREF', 136)
+def_op('LOAD_CLOSURE', 136)
 hasfree.append(136)
-def_op('STORE_DEREF', 137)
+def_op('LOAD_DEREF', 137)
 hasfree.append(137)
-def_op('DELETE_DEREF', 138)
+def_op('STORE_DEREF', 138)
 hasfree.append(138)
+def_op('DELETE_DEREF', 139)
+hasfree.append(139)
 
 def_op('CALL_FUNCTION_KW', 141)  # #args + #kwargs
 def_op('CALL_FUNCTION_EX', 142)  # Flags

Lib/test/test_dis.py

@@ -427,15 +427,17 @@ def foo(x):
     return foo
 
 dis_nested_0 = """\
-%3d           0 LOAD_CLOSURE             2 (y)
+              0 MAKE_CELL                2 (y)
-              2 BUILD_TUPLE              1
-              4 LOAD_CONST               1 (<code object foo at 0x..., file "%s", line %d>)
-              6 LOAD_CONST               2 ('_h.<locals>.foo')
-              8 MAKE_FUNCTION            8 (closure)
-             10 STORE_FAST               1 (foo)
 
-%3d          12 LOAD_FAST                1 (foo)
+%3d           2 LOAD_CLOSURE             2 (y)
-             14 RETURN_VALUE
+              4 BUILD_TUPLE              1
+              6 LOAD_CONST               1 (<code object foo at 0x..., file "%s", line %d>)
+              8 LOAD_CONST               2 ('_h.<locals>.foo')
+             10 MAKE_FUNCTION            8 (closure)
+             12 STORE_FAST               1 (foo)
+
+%3d          14 LOAD_FAST                1 (foo)
+             16 RETURN_VALUE
 """ % (_h.__code__.co_firstlineno + 1,
        __file__,
        _h.__code__.co_firstlineno + 1,
@@ -444,15 +446,17 @@ def foo(x):
 
 dis_nested_1 = """%s
 Disassembly of <code object foo at 0x..., file "%s", line %d>:
-%3d           0 LOAD_CLOSURE             1 (x)
+              0 MAKE_CELL                1 (x)
-              2 BUILD_TUPLE              1
+
-              4 LOAD_CONST               1 (<code object <listcomp> at 0x..., file "%s", line %d>)
+%3d           2 LOAD_CLOSURE             1 (x)
-              6 LOAD_CONST               2 ('_h.<locals>.foo.<locals>.<listcomp>')
+              4 BUILD_TUPLE              1
-              8 MAKE_FUNCTION            8 (closure)
+              6 LOAD_CONST               1 (<code object <listcomp> at 0x..., file "%s", line %d>)
-             10 LOAD_DEREF               2 (y)
+              8 LOAD_CONST               2 ('_h.<locals>.foo.<locals>.<listcomp>')
-             12 GET_ITER
+             10 MAKE_FUNCTION            8 (closure)
-             14 CALL_FUNCTION            1
+             12 LOAD_DEREF               2 (y)
-             16 RETURN_VALUE
+             14 GET_ITER
+             16 CALL_FUNCTION            1
+             18 RETURN_VALUE
 """ % (dis_nested_0,
        __file__,
        _h.__code__.co_firstlineno + 1,
@@ -958,59 +962,64 @@ def jumpy():
 #print('expected_opinfo_jumpy = [\n  ',
       #',\n  '.join(map(str, _instructions)), ',\n]', sep='')
 
+#dis.dis(outer)
 
 Instruction = dis.Instruction
 expected_opinfo_outer = [
-  Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval=(3, 4), argrepr='(3, 4)', offset=0, starts_line=2, is_jump_target=False),
+  Instruction(opname='MAKE_CELL', opcode=135, arg=3, argval='a', argrepr='a', offset=0, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CLOSURE', opcode=135, arg=3, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
+  Instruction(opname='MAKE_CELL', opcode=135, arg=4, argval='b', argrepr='b', offset=2, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CLOSURE', opcode=135, arg=4, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval=(3, 4), argrepr='(3, 4)', offset=4, starts_line=2, is_jump_target=False),
-  Instruction(opname='BUILD_TUPLE', opcode=102, arg=2, argval=2, argrepr='', offset=6, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CLOSURE', opcode=136, arg=3, argval='a', argrepr='a', offset=6, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_f, argrepr=repr(code_object_f), offset=8, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CLOSURE', opcode=136, arg=4, argval='b', argrepr='b', offset=8, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f', argrepr="'outer.<locals>.f'", offset=10, starts_line=None, is_jump_target=False),
+  Instruction(opname='BUILD_TUPLE', opcode=102, arg=2, argval=2, argrepr='', offset=10, starts_line=None, is_jump_target=False),
-  Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=12, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_f, argrepr=repr(code_object_f), offset=12, starts_line=None, is_jump_target=False),
-  Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=14, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f', argrepr="'outer.<locals>.f'", offset=14, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=16, starts_line=7, is_jump_target=False),
+  Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=16, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='a', argrepr='a', offset=18, starts_line=None, is_jump_target=False),
+  Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=18, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_DEREF', opcode=136, arg=4, argval='b', argrepr='b', offset=20, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=20, starts_line=7, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval='', argrepr="''", offset=22, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='a', argrepr='a', offset=22, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval=1, argrepr='1', offset=24, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='b', argrepr='b', offset=24, starts_line=None, is_jump_target=False),
-  Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=26, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval='', argrepr="''", offset=26, starts_line=None, is_jump_target=False),
-  Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=28, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval=1, argrepr='1', offset=28, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval='Hello world!', argrepr="'Hello world!'", offset=30, starts_line=None, is_jump_target=False),
+  Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=30, starts_line=None, is_jump_target=False),
-  Instruction(opname='CALL_FUNCTION', opcode=131, arg=7, argval=7, argrepr='', offset=32, starts_line=None, is_jump_target=False),
+  Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=32, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=34, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval='Hello world!', argrepr="'Hello world!'", offset=34, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=36, starts_line=8, is_jump_target=False),
+  Instruction(opname='CALL_FUNCTION', opcode=131, arg=7, argval=7, argrepr='', offset=36, starts_line=None, is_jump_target=False),
-  Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=38, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=38, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=40, starts_line=8, is_jump_target=False),
+  Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=42, starts_line=None, is_jump_target=False),
 ]
 
 expected_opinfo_f = [
-  Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=(5, 6), argrepr='(5, 6)', offset=0, starts_line=3, is_jump_target=False),
+  Instruction(opname='MAKE_CELL', opcode=135, arg=3, argval='c', argrepr='c', offset=0, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CLOSURE', opcode=135, arg=5, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
+  Instruction(opname='MAKE_CELL', opcode=135, arg=4, argval='d', argrepr='d', offset=2, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CLOSURE', opcode=135, arg=6, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=(5, 6), argrepr='(5, 6)', offset=4, starts_line=3, is_jump_target=False),
-  Instruction(opname='LOAD_CLOSURE', opcode=135, arg=3, argval='c', argrepr='c', offset=6, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CLOSURE', opcode=136, arg=5, argval='a', argrepr='a', offset=6, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CLOSURE', opcode=135, arg=4, argval='d', argrepr='d', offset=8, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CLOSURE', opcode=136, arg=6, argval='b', argrepr='b', offset=8, starts_line=None, is_jump_target=False),
-  Instruction(opname='BUILD_TUPLE', opcode=102, arg=4, argval=4, argrepr='', offset=10, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CLOSURE', opcode=136, arg=3, argval='c', argrepr='c', offset=10, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_inner, argrepr=repr(code_object_inner), offset=12, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CLOSURE', opcode=136, arg=4, argval='d', argrepr='d', offset=12, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f.<locals>.inner', argrepr="'outer.<locals>.f.<locals>.inner'", offset=14, starts_line=None, is_jump_target=False),
+  Instruction(opname='BUILD_TUPLE', opcode=102, arg=4, argval=4, argrepr='', offset=14, starts_line=None, is_jump_target=False),
-  Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=16, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_inner, argrepr=repr(code_object_inner), offset=16, starts_line=None, is_jump_target=False),
-  Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=18, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f.<locals>.inner', argrepr="'outer.<locals>.f.<locals>.inner'", offset=18, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=20, starts_line=5, is_jump_target=False),
+  Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=20, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_DEREF', opcode=136, arg=5, argval='a', argrepr='a', offset=22, starts_line=None, is_jump_target=False),
+  Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=22, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_DEREF', opcode=136, arg=6, argval='b', argrepr='b', offset=24, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=24, starts_line=5, is_jump_target=False),
-  Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='c', argrepr='c', offset=26, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_DEREF', opcode=137, arg=5, argval='a', argrepr='a', offset=26, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_DEREF', opcode=136, arg=4, argval='d', argrepr='d', offset=28, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_DEREF', opcode=137, arg=6, argval='b', argrepr='b', offset=28, starts_line=None, is_jump_target=False),
-  Instruction(opname='CALL_FUNCTION', opcode=131, arg=4, argval=4, argrepr='', offset=30, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='c', argrepr='c', offset=30, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=32, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='d', argrepr='d', offset=32, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='inner', argrepr='inner', offset=34, starts_line=6, is_jump_target=False),
+  Instruction(opname='CALL_FUNCTION', opcode=131, arg=4, argval=4, argrepr='', offset=34, starts_line=None, is_jump_target=False),
-  Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=36, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=36, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='inner', argrepr='inner', offset=38, starts_line=6, is_jump_target=False),
+  Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=40, starts_line=None, is_jump_target=False),
 ]
 
 expected_opinfo_inner = [
   Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=0, starts_line=4, is_jump_target=False),
-  Instruction(opname='LOAD_DEREF', opcode=136, arg=2, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_DEREF', opcode=137, arg=2, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_DEREF', opcode=136, arg=4, argval='c', argrepr='c', offset=6, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='c', argrepr='c', offset=6, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_DEREF', opcode=136, arg=5, argval='d', argrepr='d', offset=8, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_DEREF', opcode=137, arg=5, argval='d', argrepr='d', offset=8, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='e', argrepr='e', offset=10, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_FAST', opcode=124, arg=1, argval='f', argrepr='f', offset=12, starts_line=None, is_jump_target=False),
   Instruction(opname='CALL_FUNCTION', opcode=131, arg=6, argval=6, argrepr='', offset=14, starts_line=None, is_jump_target=False),

Lib/test/test_scope.py

@@ -176,6 +176,57 @@ def bar():
         self.assertEqual(foo(a=42), 50)
         self.assertEqual(foo(), 25)
 
+    def testCellIsArgAndEscapes(self):
+        # We need to be sure that a cell passed in as an arg still
+        # gets wrapped in a new cell if the arg escapes into an
+        # inner function (closure).
+
+        def external():
+            value = 42
+            def inner():
+                return value
+            cell, = inner.__closure__
+            return cell
+        cell_ext = external()
+
+        def spam(arg):
+            def eggs():
+                return arg
+            return eggs
+
+        eggs = spam(cell_ext)
+        cell_closure, = eggs.__closure__
+        cell_eggs = eggs()
+
+        self.assertIs(cell_eggs, cell_ext)
+        self.assertIsNot(cell_eggs, cell_closure)
+
+    def testCellIsLocalAndEscapes(self):
+        # We need to be sure that a cell bound to a local still
+        # gets wrapped in a new cell if the local escapes into an
+        # inner function (closure).
+
+        def external():
+            value = 42
+            def inner():
+                return value
+            cell, = inner.__closure__
+            return cell
+        cell_ext = external()
+
+        def spam(arg):
+            cell = arg
+            def eggs():
+                return cell
+            return eggs
+
+        eggs = spam(cell_ext)
+        cell_closure, = eggs.__closure__
+        cell_eggs = eggs()
+
+        self.assertIs(cell_eggs, cell_ext)
+        self.assertIsNot(cell_eggs, cell_closure)
+
     def testRecursion(self):
 
         def f(x):

Misc/NEWS.d/next/Core and Builtins/2021-05-26-19-10-47.bpo-43693.1KSG9u.rst

@@ -0,0 +1,5 @@
+A new opcode MAKE_CELL has been added that effectively moves some of
+the work done on function entry into the compiler and into the eval
+loop.  In addition to creating the required cell objects, the new
+opcode converts relevant arguments (and other locals) to cell
+variables on function entry.

Objects/frameobject.c

@@ -918,6 +918,19 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code,
     return f;
 }
 
+int
+_PyFrame_OpAlreadyRan(PyFrameObject *f, int opcode, int oparg)
+{
+    const _Py_CODEUNIT *code =
+        (const _Py_CODEUNIT *)PyBytes_AS_STRING(f->f_code->co_code);
+    for (int i = 0; i < f->f_lasti; i++) {
+        if (_Py_OPCODE(code[i]) == opcode && _Py_OPARG(code[i]) == oparg) {
+            return 1;
+        }
+    }
+    return 0;
+}
+
 int
 PyFrame_FastToLocalsWithError(PyFrameObject *f)
 {
@@ -961,16 +974,53 @@ PyFrame_FastToLocalsWithError(PyFrameObject *f)
            former here and will later use the cell for the variable.
         */
         if (kind & CO_FAST_LOCAL && kind & CO_FAST_CELL) {
-            assert(fast[i] == NULL);
             continue;
         }
 
         PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i);
         PyObject *value = fast[i];
-        if (kind & (CO_FAST_CELL | CO_FAST_FREE) && value != NULL) {
+        if (f->f_state != FRAME_CLEARED) {
-            assert(PyCell_Check(value));
+            int cellargoffset = CO_CELL_NOT_AN_ARG;
+            if (co->co_cell2arg != NULL) {
+                cellargoffset = co->co_cell2arg[i - co->co_nlocals];
+            }
+            if (kind & CO_FAST_FREE) {
+                // The cell was set by _PyEval_MakeFrameVector() from
+                // the function's closure.
+                assert(value != NULL && PyCell_Check(value));
                 value = PyCell_GET(value);
             }
+            else if (kind & CO_FAST_CELL) {
+                // Note that no *_DEREF ops can happen before MAKE_CELL
+                // executes.  So there's no need to duplicate the work
+                // that MAKE_CELL would otherwise do later, if it hasn't
+                // run yet.
+                if (value != NULL) {
+                    if (PyCell_Check(value) &&
+                            _PyFrame_OpAlreadyRan(f, MAKE_CELL, i)) {
+                        // (likely) MAKE_CELL must have executed already.
+                        value = PyCell_GET(value);
+                    }
+                    // (unlikely) Otherwise it must be an initial value set
+                    // by an earlier call to PyFrame_FastToLocals().
+                }
+                else {
+                    // (unlikely) MAKE_CELL hasn't executed yet.
+                    if (cellargoffset != CO_CELL_NOT_AN_ARG) {
+                        // It is an arg that escapes into an inner
+                        // function so we use the initial value that
+                        // was already set by _PyEval_MakeFrameVector().
+                        // Normally the arg value would always be set.
+                        // However, it can be NULL if it was deleted via
+                        // PyFrame_LocalsToFast().
+                        value = fast[cellargoffset];
+                    }
+                }
+            }
+        }
+        else {
+            assert(value == NULL);
+        }
         if (value == NULL) {
             if (PyObject_DelItem(locals, name) != 0) {
                 if (PyErr_ExceptionMatches(PyExc_KeyError)) {
@@ -1010,8 +1060,9 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear)
     PyObject **fast;
     PyObject *error_type, *error_value, *error_traceback;
     PyCodeObject *co;
-    if (f == NULL)
+    if (f == NULL || f->f_state == FRAME_CLEARED) {
         return;
+    }
     locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET];
     if (locals == NULL)
         return;
@@ -1039,16 +1090,68 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear)
                 continue;
             }
         }
-        if (kind & (CO_FAST_CELL | CO_FAST_FREE)) {
+        PyObject *oldvalue = fast[i];
-            assert(PyCell_Check(fast[i]));
+        int cellargoffset = CO_CELL_NOT_AN_ARG;
-            if (PyCell_GET(fast[i]) != value) {
+        if (kind & CO_FAST_CELL && co->co_cell2arg != NULL) {
-                if (PyCell_Set(fast[i], value) < 0) {
+            assert(i >= co->co_nlocals);
-                    PyErr_Clear();
+            cellargoffset = co->co_cell2arg[i - co->co_nlocals];
+        }
+        PyObject *cell = NULL;
+        if (kind == CO_FAST_FREE) {
+            // The cell was set by _PyEval_MakeFrameVector() from
+            // the function's closure.
+            assert(oldvalue != NULL && PyCell_Check(oldvalue));
+            cell = oldvalue;
+        }
+        else if (kind & CO_FAST_CELL && oldvalue != NULL) {
+            if (cellargoffset != CO_CELL_NOT_AN_ARG) {
+                // (likely) MAKE_CELL must have executed already.
+                // It's the cell for an arg.
+                assert(PyCell_Check(oldvalue));
+                cell = oldvalue;
+            }
+            else {
+                if (PyCell_Check(oldvalue) &&
+                        _PyFrame_OpAlreadyRan(f, MAKE_CELL, i)) {
+                    // (likely) MAKE_CELL must have executed already.
+                    cell = oldvalue;
+                }
+                // (unlikely) Otherwise, it must have been set to some
+                // initial value by an earlier call to PyFrame_LocalsToFast().
             }
         }
-        } else if (fast[i] != value) {
+        if (cell != NULL) {
+            oldvalue = PyCell_GET(cell);
+            if (value != oldvalue) {
+                Py_XDECREF(oldvalue);
                 Py_XINCREF(value);
-            Py_XSETREF(fast[i], value);
+                PyCell_SET(cell, value);
+            }
+        }
+        else {
+            int offset = i;
+            if (kind & CO_FAST_CELL) {
+                // (unlikely) MAKE_CELL hasn't executed yet.
+                // Note that there is no need to create the cell that
+                // MAKE_CELL would otherwise create later, since no
+                // *_DEREF ops can happen before MAKE_CELL has run.
+                if (cellargoffset != CO_CELL_NOT_AN_ARG) {
+                    // It's the cell for an arg.
+                    // Replace the initial value that was set by
+                    // _PyEval_MakeFrameVector().
+                    // Normally the arg value would always be set.
+                    // However, it can be NULL if it was deleted
+                    // via an earlier PyFrame_LocalsToFast() call.
+                    offset = cellargoffset;
+                    oldvalue = fast[offset];
+                }
+                // Otherwise set an initial value for MAKE_CELL to use
+                // when it runs later.
+            }
+            if (value != oldvalue) {
+                Py_XINCREF(value);
+                Py_XSETREF(fast[offset], value);
+            }
         }
         Py_XDECREF(value);
     }

Objects/typeobject.c

@@ -11,6 +11,8 @@
 #include "pycore_pystate.h"       // _PyThreadState_GET()
 #include "pycore_unionobject.h"   // _Py_Union(), _Py_union_type_or
 #include "frameobject.h"
+#include "pycore_frame.h"         // _PyFrame_OpAlreadyRan
+#include "opcode.h"               // MAKE_CELL
 #include "structmember.h"         // PyMemberDef
 
 #include <ctype.h>
@@ -8877,14 +8879,17 @@ super_init_without_args(PyFrameObject *f, PyCodeObject *co,
     }
 
     PyObject *obj = f->f_localsptr[0];
-    Py_ssize_t i;
+    int i;
     if (obj == NULL && co->co_cell2arg) {
         /* The first argument might be a cell. */
         for (i = 0; i < co->co_ncellvars; i++) {
             if (co->co_cell2arg[i] == 0) {
-                PyObject *cell = f->f_localsptr[co->co_nlocals + i];
+                int celloffset = co->co_nlocals + i;
-                assert(PyCell_Check(cell));
+                PyObject *cell = f->f_localsptr[celloffset];
+                if (PyCell_Check(cell) &&
+                        _PyFrame_OpAlreadyRan(f, MAKE_CELL, celloffset)) {
                     obj = PyCell_GET(cell);
+                }
                 break;
             }
         }

Python/ceval.c

@@ -3076,6 +3076,34 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
             goto error;
         }
 
+        case TARGET(MAKE_CELL): {
+            PyObject *initial = GETLOCAL(oparg);
+            // Normally initial would be NULL.  However, it
+            // might have been set to an initial value during
+            // a call to PyFrame_LocalsToFast().
+            PyObject *cell = PyCell_New(initial);
+            if (cell == NULL) {
+                goto error;
+            }
+            /* If it is an arg then copy the arg into the cell. */
+            if (initial == NULL && co->co_cell2arg != NULL) {
+                int argoffset = co->co_cell2arg[oparg - co->co_nlocals];
+                if (argoffset != CO_CELL_NOT_AN_ARG) {
+                    PyObject *arg = GETLOCAL(argoffset);
+                    // It will have been set in initialize_locals() but
+                    // may have been deleted PyFrame_LocalsToFast().
+                    if (arg != NULL) {;
+                        Py_INCREF(arg);
+                        PyCell_SET(cell, arg);
+                        /* Clear the local copy. */
+                        SETLOCAL(argoffset, NULL);
+                    }
+                }
+            }
+            SETLOCAL(oparg, cell);
+            DISPATCH();
+        }
+
         case TARGET(DELETE_DEREF): {
             PyObject *cell = GETLOCAL(oparg);
             PyObject *oldobj = PyCell_GET(cell);
@@ -5067,27 +5095,6 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con,
         }
     }
 
-
-    /* Allocate and initialize storage for cell vars, and copy free
-       vars into frame. */
-    for (i = 0; i < co->co_ncellvars; ++i) {
-        PyObject *c;
-        Py_ssize_t arg;
-        /* Possibly account for the cell variable being an argument. */
-        if (co->co_cell2arg != NULL &&
-            (arg = co->co_cell2arg[i]) != CO_CELL_NOT_AN_ARG) {
-            c = PyCell_New(GETLOCAL(arg));
-            /* Clear the local copy. */
-            SETLOCAL(arg, NULL);
-        }
-        else {
-            c = PyCell_New(NULL);
-        }
-        if (c == NULL)
-            goto fail;
-        SETLOCAL(co->co_nlocals + i, c);
-    }
-
     /* Copy closure variables to free variables */
     for (i = 0; i < co->co_nfreevars; ++i) {
         PyObject *o = PyTuple_GET_ITEM(con->fc_closure, i);

Python/compile.c

@@ -1185,6 +1185,8 @@ stack_effect(int opcode, int oparg, int jump)
                 return -1;
 
         /* Closures */
+        case MAKE_CELL:
+            return 0;
         case LOAD_CLOSURE:
             return 1;
         case LOAD_DEREF:
@@ -7374,15 +7376,47 @@ optimize_cfg(struct compiler *c, struct assembler *a, PyObject *consts);
 static int
 ensure_exits_have_lineno(struct compiler *c);
 
+static inline int
+insert_instruction(basicblock *block, int pos, struct instr *instr) {
+    if (compiler_next_instr(block) < 0) {
+        return -1;
+    }
+    for (int i = block->b_iused-1; i > pos; i--) {
+        block->b_instr[i] = block->b_instr[i-1];
+    }
+    block->b_instr[pos] = *instr;
+    return 0;
+}
+
 static int
-insert_generator_prefix(struct compiler *c, basicblock *entryblock) {
+insert_prefix_instructions(struct compiler *c, basicblock *entryblock) {
 
     int flags = compute_code_flags(c);
     if (flags < 0) {
         return -1;
     }
-    int kind;
+
+    /* Set up cells for any variable that escapes, to be put in a closure. */
+    PyObject *k, *v;
+    Py_ssize_t pos = 0;
+    while (PyDict_Next(c->u->u_cellvars, &pos, &k, &v)) {
+        assert(PyLong_AS_LONG(v) < INT_MAX);
+        int cellindex = (int)PyLong_AS_LONG(v);
+        struct instr make_cell = {
+            .i_opcode = MAKE_CELL,
+            // This will get fixed in offset_derefs().
+            .i_oparg = cellindex,
+            .i_lineno = -1,
+            .i_target = NULL,
+        };
+        if (insert_instruction(entryblock, (int)(pos - 1), &make_cell) < 0) {
+            return -1;
+        }
+    }
+
+    /* Add the generator prefix instructions. */
     if (flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) {
+        int kind;
         if (flags & CO_COROUTINE) {
             kind = 1;
         }
@@ -7392,20 +7426,18 @@ insert_generator_prefix(struct compiler *c, basicblock *entryblock) {
         else {
             kind = 0;
         }
-    }
+
-    else {
+        struct instr gen_start = {
-        return 0;
+            .i_opcode = GEN_START,
-    }
+            .i_oparg = kind,
-    if (compiler_next_instr(entryblock) < 0) {
+            .i_lineno = -1,
+            .i_target = NULL,
+        };
+        if (insert_instruction(entryblock, 0, &gen_start) < 0) {
             return -1;
         }
-    for (int i = entryblock->b_iused-1; i > 0; i--) {
-        entryblock->b_instr[i] = entryblock->b_instr[i-1];
     }
-    entryblock->b_instr[0].i_opcode = GEN_START;
+
-    entryblock->b_instr[0].i_oparg = kind;
-    entryblock->b_instr[0].i_lineno = -1;
-    entryblock->b_instr[0].i_target = NULL;
     return 0;
 }
 
@@ -7438,12 +7470,15 @@ guarantee_lineno_for_exits(struct assembler *a, int firstlineno) {
 }
 
 static void
-offset_derefs(basicblock *entryblock, int nlocals)
+fix_cell_offsets(struct compiler *c, basicblock *entryblock)
 {
+    assert(PyDict_GET_SIZE(c->u->u_varnames) < INT_MAX);
+    int nlocals = (int)PyDict_GET_SIZE(c->u->u_varnames);
     for (basicblock *b = entryblock; b != NULL; b = b->b_next) {
         for (int i = 0; i < b->b_iused; i++) {
             struct instr *inst = &b->b_instr[i];
             switch(inst->i_opcode) {
+                case MAKE_CELL:
                 case LOAD_CLOSURE:
                 case LOAD_DEREF:
                 case STORE_DEREF:
@@ -7493,7 +7528,7 @@ assemble(struct compiler *c, int addNone)
     }
     assert(entryblock != NULL);
 
-    if (insert_generator_prefix(c, entryblock)) {
+    if (insert_prefix_instructions(c, entryblock)) {
         goto error;
     }
 
@@ -7510,8 +7545,7 @@ assemble(struct compiler *c, int addNone)
     a.a_entry = entryblock;
     a.a_nblocks = nblocks;
 
-    assert(PyDict_GET_SIZE(c->u->u_varnames) < INT_MAX);
+    fix_cell_offsets(c, entryblock);
-    offset_derefs(entryblock, (int)PyDict_GET_SIZE(c->u->u_varnames));
 
     consts = consts_dict_keys_inorder(c->u->u_consts);
     if (consts == NULL) {

Python/opcode_targets.h

@@ -134,12 +134,12 @@ static void *opcode_targets[256] = {
     &&TARGET_MAKE_FUNCTION,
     &&TARGET_BUILD_SLICE,
     &&_unknown_opcode,
+    &&TARGET_MAKE_CELL,
     &&TARGET_LOAD_CLOSURE,
     &&TARGET_LOAD_DEREF,
     &&TARGET_STORE_DEREF,
     &&TARGET_DELETE_DEREF,
     &&_unknown_opcode,
-    &&_unknown_opcode,
     &&TARGET_CALL_FUNCTION_KW,
     &&TARGET_CALL_FUNCTION_EX,
     &&_unknown_opcode,