gh-119180: PEP 649 compiler changes (#119361)

Include/internal/pycore_global_objects_fini_generated.h

@@ -559,6 +559,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot_locals));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(empty));
+    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(format));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(generic_base));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(json_decoder));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(kwdefaults));
@@ -745,7 +746,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abstract_));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_active));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_align_));
-    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_annotation));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_anonymous_));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_argtypes_));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_as_parameter_));

Include/internal/pycore_global_strings.h

@@ -45,6 +45,7 @@ struct _Py_global_strings {
         STRUCT_FOR_STR(dot, ".")
         STRUCT_FOR_STR(dot_locals, ".<locals>")
         STRUCT_FOR_STR(empty, "")
+        STRUCT_FOR_STR(format, ".format")
         STRUCT_FOR_STR(generic_base, ".generic_base")
         STRUCT_FOR_STR(json_decoder, "json.decoder")
         STRUCT_FOR_STR(kwdefaults, ".kwdefaults")
@@ -234,7 +235,6 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(_abstract_)
         STRUCT_FOR_ID(_active)
         STRUCT_FOR_ID(_align_)
-        STRUCT_FOR_ID(_annotation)
         STRUCT_FOR_ID(_anonymous_)
         STRUCT_FOR_ID(_argtypes_)
         STRUCT_FOR_ID(_as_parameter_)

Include/internal/pycore_opcode_utils.h

@@ -57,6 +57,7 @@ extern "C" {
 #define MAKE_FUNCTION_KWDEFAULTS  0x02
 #define MAKE_FUNCTION_ANNOTATIONS 0x04
 #define MAKE_FUNCTION_CLOSURE     0x08
+#define MAKE_FUNCTION_ANNOTATE    0x10
 
 /* Values used as the oparg for LOAD_COMMON_CONSTANT */
 #define CONSTANT_ASSERTIONERROR 0

Include/internal/pycore_runtime_init_generated.h

@@ -554,6 +554,7 @@ extern "C" {
     INIT_STR(dot, "."), \
     INIT_STR(dot_locals, ".<locals>"), \
     INIT_STR(empty, ""), \
+    INIT_STR(format, ".format"), \
     INIT_STR(generic_base, ".generic_base"), \
     INIT_STR(json_decoder, "json.decoder"), \
     INIT_STR(kwdefaults, ".kwdefaults"), \
@@ -743,7 +744,6 @@ extern "C" {
     INIT_ID(_abstract_), \
     INIT_ID(_active), \
     INIT_ID(_align_), \
-    INIT_ID(_annotation), \
     INIT_ID(_anonymous_), \
     INIT_ID(_argtypes_), \
     INIT_ID(_as_parameter_), \

Include/internal/pycore_symtable.h

@@ -12,8 +12,9 @@ struct _mod;   // Type defined in pycore_ast.h
 
 typedef enum _block_type {
     FunctionBlock, ClassBlock, ModuleBlock,
-    // Used for annotations if 'from __future__ import annotations' is active.
+    // Used for annotations. If 'from __future__ import annotations' is active,
-    // Annotation blocks cannot bind names and are not evaluated.
+    // annotation blocks cannot bind names and are not evaluated. Otherwise, they
+    // are lazily evaluated (see PEP 649).
     AnnotationBlock,
     // Used for generics and type aliases. These work mostly like functions
     // (see PEP 695 for details). The three different blocks function identically;
@@ -89,6 +90,7 @@ typedef struct _symtable_entry {
                                      including free refs to globals */
     unsigned ste_generator : 1;   /* true if namespace is a generator */
     unsigned ste_coroutine : 1;   /* true if namespace is a coroutine */
+    unsigned ste_annotations_used : 1;  /* true if there are any annotations in this scope */
     _Py_comprehension_ty ste_comprehension;  /* Kind of comprehension (if any) */
     unsigned ste_varargs : 1;     /* true if block has varargs */
     unsigned ste_varkeywords : 1; /* true if block has varkeywords */
@@ -110,6 +112,7 @@ typedef struct _symtable_entry {
     int ste_end_col_offset;  /* end offset of first line of block */
     int ste_opt_lineno;      /* lineno of last exec or import * */
     int ste_opt_col_offset;  /* offset of last exec or import * */
+    struct _symtable_entry *ste_annotation_block; /* symbol table entry for this entry's annotations */
     struct symtable *ste_table;
 } PySTEntryObject;
 
@@ -126,6 +129,7 @@ extern struct symtable* _PySymtable_Build(
     PyObject *filename,
     _PyFutureFeatures *future);
 extern PySTEntryObject* _PySymtable_Lookup(struct symtable *, void *);
+extern int _PySymtable_LookupOptional(struct symtable *, void *, PySTEntryObject **);
 
 extern void _PySymtable_Free(struct symtable *);
 

Include/internal/pycore_unicodeobject_generated.h

@@ -543,9 +543,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
     string = &_Py_ID(_align_);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
-    string = &_Py_ID(_annotation);
-    assert(_PyUnicode_CheckConsistency(string, 1));
-    _PyUnicode_InternInPlace(interp, &string);
     string = &_Py_ID(_anonymous_);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);

Lib/inspect.py

@@ -220,13 +220,7 @@ def get_annotations(obj, *, globals=None, locals=None, eval_str=False):
     """
     if isinstance(obj, type):
         # class
-        obj_dict = getattr(obj, '__dict__', None)
+        ann = obj.__annotations__
-        if obj_dict and hasattr(obj_dict, 'get'):
-            ann = obj_dict.get('__annotations__', None)
-            if isinstance(ann, types.GetSetDescriptorType):
-                ann = None
-        else:
-            ann = None
 
         obj_globals = None
         module_name = getattr(obj, '__module__', None)

Lib/symtable.py

@@ -222,6 +222,8 @@ def get_methods(self):
         if self.__methods is None:
             d = {}
             for st in self._table.children:
+                if st.type == _symtable.TYPE_ANNOTATION:
+                    continue
                 d[st.name] = 1
             self.__methods = tuple(d)
         return self.__methods

Lib/test/test_dis.py

@@ -352,32 +352,21 @@ def wrap_func_w_kwargs():
 dis_annot_stmt_str = """\
   0           RESUME                   0
 
-  2           SETUP_ANNOTATIONS
+  2           LOAD_CONST               0 (1)
-              LOAD_CONST               0 (1)
               STORE_NAME               0 (x)
-              LOAD_NAME                1 (int)
-              LOAD_NAME                2 (__annotations__)
-              LOAD_CONST               1 ('x')
-              STORE_SUBSCR
-
-  3           LOAD_NAME                3 (fun)
-              PUSH_NULL
-              LOAD_CONST               0 (1)
-              CALL                     1
-              LOAD_NAME                2 (__annotations__)
-              LOAD_CONST               2 ('y')
-              STORE_SUBSCR
 
   4           LOAD_CONST               0 (1)
-              LOAD_NAME                4 (lst)
+              LOAD_NAME                1 (lst)
-              LOAD_NAME                3 (fun)
+              LOAD_NAME                2 (fun)
               PUSH_NULL
-              LOAD_CONST               3 (0)
+              LOAD_CONST               1 (0)
               CALL                     1
               STORE_SUBSCR
-              LOAD_NAME                1 (int)
+
-              POP_TOP
+  2           LOAD_CONST               2 (<code object __annotate__ at 0x..., file "<dis>", line 2>)
-              RETURN_CONST             4 (None)
+              MAKE_FUNCTION
+              STORE_NAME               3 (__annotate__)
+              RETURN_CONST             3 (None)
 """
 
 compound_stmt_str = """\

Lib/test/test_grammar.py

@@ -306,16 +306,6 @@ def test_eof_error(self):
 
 var_annot_global: int # a global annotated is necessary for test_var_annot
 
-# custom namespace for testing __annotations__
-
-class CNS:
-    def __init__(self):
-        self._dct = {}
-    def __setitem__(self, item, value):
-        self._dct[item.lower()] = value
-    def __getitem__(self, item):
-        return self._dct[item]
-
 
 class GrammarTests(unittest.TestCase):
 
@@ -446,22 +436,12 @@ class F(C, A):
         self.assertEqual(E.__annotations__, {})
         self.assertEqual(F.__annotations__, {})
 
-
-    def test_var_annot_metaclass_semantics(self):
-        class CMeta(type):
-            @classmethod
-            def __prepare__(metacls, name, bases, **kwds):
-                return {'__annotations__': CNS()}
-        class CC(metaclass=CMeta):
-            XX: 'ANNOT'
-        self.assertEqual(CC.__annotations__['xx'], 'ANNOT')
-
     def test_var_annot_module_semantics(self):
         self.assertEqual(test.__annotations__, {})
         self.assertEqual(ann_module.__annotations__,
-                     {1: 2, 'x': int, 'y': str, 'f': typing.Tuple[int, int], 'u': int | float})
+                         {'x': int, 'y': str, 'f': typing.Tuple[int, int], 'u': int | float})
         self.assertEqual(ann_module.M.__annotations__,
-                              {'123': 123, 'o': type})
+                         {'o': type})
         self.assertEqual(ann_module2.__annotations__, {})
 
     def test_var_annot_in_module(self):
@@ -476,51 +456,12 @@ def test_var_annot_in_module(self):
             ann_module3.D_bad_ann(5)
 
     def test_var_annot_simple_exec(self):
-        gns = {}; lns= {}
+        gns = {}; lns = {}
         exec("'docstring'\n"
-             "__annotations__[1] = 2\n"
              "x: int = 5\n", gns, lns)
-        self.assertEqual(lns["__annotations__"], {1: 2, 'x': int})
+        self.assertEqual(lns["__annotate__"](1), {'x': int})
         with self.assertRaises(KeyError):
-            gns['__annotations__']
+            gns['__annotate__']
-
-    def test_var_annot_custom_maps(self):
-        # tests with custom locals() and __annotations__
-        ns = {'__annotations__': CNS()}
-        exec('X: int; Z: str = "Z"; (w): complex = 1j', ns)
-        self.assertEqual(ns['__annotations__']['x'], int)
-        self.assertEqual(ns['__annotations__']['z'], str)
-        with self.assertRaises(KeyError):
-            ns['__annotations__']['w']
-        nonloc_ns = {}
-        class CNS2:
-            def __init__(self):
-                self._dct = {}
-            def __setitem__(self, item, value):
-                nonlocal nonloc_ns
-                self._dct[item] = value
-                nonloc_ns[item] = value
-            def __getitem__(self, item):
-                return self._dct[item]
-        exec('x: int = 1', {}, CNS2())
-        self.assertEqual(nonloc_ns['__annotations__']['x'], int)
-
-    def test_var_annot_refleak(self):
-        # complex case: custom locals plus custom __annotations__
-        # this was causing refleak
-        cns = CNS()
-        nonloc_ns = {'__annotations__': cns}
-        class CNS2:
-            def __init__(self):
-                self._dct = {'__annotations__': cns}
-            def __setitem__(self, item, value):
-                nonlocal nonloc_ns
-                self._dct[item] = value
-                nonloc_ns[item] = value
-            def __getitem__(self, item):
-                return self._dct[item]
-        exec('X: str', {}, CNS2())
-        self.assertEqual(nonloc_ns['__annotations__']['x'], str)
 
     def test_var_annot_rhs(self):
         ns = {}

Lib/test/test_module/__init__.py

@@ -360,6 +360,8 @@ def test_annotations_are_created_correctly(self):
         ann_module4 = import_helper.import_fresh_module(
             'test.typinganndata.ann_module4',
         )
+        self.assertFalse("__annotations__" in ann_module4.__dict__)
+        self.assertEqual(ann_module4.__annotations__, {"a": int, "b": str})
         self.assertTrue("__annotations__" in ann_module4.__dict__)
         del ann_module4.__annotations__
         self.assertFalse("__annotations__" in ann_module4.__dict__)

Lib/test/test_opcodes.py

@@ -39,16 +39,19 @@ class C: pass
     def test_use_existing_annotations(self):
         ns = {'__annotations__': {1: 2}}
         exec('x: int', ns)
-        self.assertEqual(ns['__annotations__'], {'x': int, 1: 2})
+        self.assertEqual(ns['__annotations__'], {1: 2})
 
     def test_do_not_recreate_annotations(self):
         # Don't rely on the existence of the '__annotations__' global.
         with support.swap_item(globals(), '__annotations__', {}):
-            del globals()['__annotations__']
+            globals().pop('__annotations__', None)
             class C:
-                del __annotations__
+                try:
-                with self.assertRaises(NameError):
+                    del __annotations__
-                    x: int
+                except NameError:
+                    pass
+                x: int
+            self.assertEqual(C.__annotations__, {"x": int})
 
     def test_raise_class_exceptions(self):
 

Lib/test/test_positional_only_arg.py

@@ -2,6 +2,7 @@
 
 import dis
 import pickle
+import types
 import unittest
 
 from test.support import check_syntax_error
@@ -440,7 +441,9 @@ def f(x: not (int is int), /): ...
         # without constant folding we end up with
         # COMPARE_OP(is), IS_OP (0)
         # with constant folding we should expect a IS_OP (1)
-        codes = [(i.opname, i.argval) for i in dis.get_instructions(g)]
+        code_obj = next(const for const in g.__code__.co_consts
+                        if isinstance(const, types.CodeType) and const.co_name == "__annotate__")
+        codes = [(i.opname, i.argval) for i in dis.get_instructions(code_obj)]
         self.assertNotIn(('UNARY_NOT', None), codes)
         self.assertIn(('IS_OP', 1), codes)
 

Lib/test/test_pyclbr.py

@@ -109,6 +109,8 @@ def ismethod(oclass, obj, name):
 
                 actualMethods = []
                 for m in py_item.__dict__.keys():
+                    if m == "__annotate__":
+                        continue
                     if ismethod(py_item, getattr(py_item, m), m):
                         actualMethods.append(m)
                 foundMethods = []

Lib/test/test_pydoc/test_pydoc.py

@@ -77,6 +77,11 @@ class A(builtins.object)
      |  __weakref__%s
 
     class B(builtins.object)
+     |  Methods defined here:
+     |
+     |  __annotate__(...)
+     |
+     |  ----------------------------------------------------------------------
      |  Data descriptors defined here:
      |
      |  __dict__%s
@@ -87,8 +92,6 @@ class B(builtins.object)
      |  Data and other attributes defined here:
      |
      |  NO_MEANING = 'eggs'
-     |
-     |  __annotations__ = {'NO_MEANING': <class 'str'>}
 
     class C(builtins.object)
      |  Methods defined here:
@@ -176,6 +179,9 @@ class A(builtins.object)
             list of weak references to the object
 
 class B(builtins.object)
+    Methods defined here:
+        __annotate__(...)
+    ----------------------------------------------------------------------
     Data descriptors defined here:
         __dict__
             dictionary for instance variables
@@ -184,7 +190,6 @@ class B(builtins.object)
     ----------------------------------------------------------------------
     Data and other attributes defined here:
         NO_MEANING = 'eggs'
-        __annotations__ = {'NO_MEANING': <class 'str'>}
 
 
 class C(builtins.object)

Lib/test/test_pyrepl/test_interact.py

@@ -105,7 +105,7 @@ def test_runsource_shows_syntax_error_for_failed_compilation(self):
 
     def test_no_active_future(self):
         console = InteractiveColoredConsole()
-        source = "x: int = 1; print(__annotations__)"
+        source = "x: int = 1; print(__annotate__(1))"
         f = io.StringIO()
         with contextlib.redirect_stdout(f):
             result = console.runsource(source)

Lib/test/test_symtable.py

@@ -205,12 +205,14 @@ def test_assigned(self):
 
     def test_annotated(self):
         st1 = symtable.symtable('def f():\n    x: int\n', 'test', 'exec')
-        st2 = st1.get_children()[0]
+        st2 = st1.get_children()[1]
+        self.assertEqual(st2.get_type(), "function")
         self.assertTrue(st2.lookup('x').is_local())
         self.assertTrue(st2.lookup('x').is_annotated())
         self.assertFalse(st2.lookup('x').is_global())
         st3 = symtable.symtable('def f():\n    x = 1\n', 'test', 'exec')
-        st4 = st3.get_children()[0]
+        st4 = st3.get_children()[1]
+        self.assertEqual(st4.get_type(), "function")
         self.assertTrue(st4.lookup('x').is_local())
         self.assertFalse(st4.lookup('x').is_annotated())
 

Lib/test/test_traceback.py

@@ -622,6 +622,7 @@ def test_caret_in_type_annotation(self):
         def f_with_type():
             def foo(a: THIS_DOES_NOT_EXIST ) -> int:
                 return 0
+            foo.__annotations__
 
         lineno_f = f_with_type.__code__.co_firstlineno
         expected_f = (
@@ -629,7 +630,9 @@ def foo(a: THIS_DOES_NOT_EXIST ) -> int:
             f'  File "{__file__}", line {self.callable_line}, in get_exception\n'
             '    callable()\n'
             '    ~~~~~~~~^^\n'
-            f'  File "{__file__}", line {lineno_f+1}, in f_with_type\n'
+            f'  File "{__file__}", line {lineno_f+3}, in f_with_type\n'
+            '    foo.__annotations__\n'
+            f'  File "{__file__}", line {lineno_f+1}, in __annotate__\n'
             '    def foo(a: THIS_DOES_NOT_EXIST ) -> int:\n'
             '               ^^^^^^^^^^^^^^^^^^^\n'
         )

Lib/test/test_type_annotations.py

@@ -1,7 +1,12 @@
 import textwrap
 import types
 import unittest
-from test.support import run_code
+from test.support import run_code, check_syntax_error
+
+VALUE = 1
+FORWARDREF = 2
+SOURCE = 3
+
 
 class TypeAnnotationTests(unittest.TestCase):
 
@@ -49,6 +54,7 @@ def test_annotations_are_created_correctly(self):
         class C:
             a:int=3
             b:str=4
+        self.assertEqual(C.__annotations__, {"a": int, "b": str})
         self.assertTrue("__annotations__" in C.__dict__)
         del C.__annotations__
         self.assertFalse("__annotations__" in C.__dict__)
@@ -106,6 +112,13 @@ class D(metaclass=C):
         self.assertEqual(D.__annotations__, {})
 
 
+def build_module(code: str, name: str = "top") -> types.ModuleType:
+    ns = run_code(code)
+    mod = types.ModuleType(name)
+    mod.__dict__.update(ns)
+    return mod
+
+
 class TestSetupAnnotations(unittest.TestCase):
     def check(self, code: str):
         code = textwrap.dedent(code)
@@ -113,11 +126,10 @@ def check(self, code: str):
             with self.subTest(scope=scope):
                 if scope == "class":
                     code = f"class C:\n{textwrap.indent(code, '    ')}"
-                ns = run_code(code)
+                    ns = run_code(code)
-                if scope == "class":
                     annotations = ns["C"].__annotations__
                 else:
-                    annotations = ns["__annotations__"]
+                    annotations = build_module(code).__annotations__
                 self.assertEqual(annotations, {"x": int})
 
     def test_top_level(self):
@@ -256,3 +268,146 @@ def check_annotations(self, f):
         # Setting f.__annotations__ also clears __annotate__
         f.__annotations__ = {"z": 43}
         self.assertIs(f.__annotate__, None)
+
+
+class DeferredEvaluationTests(unittest.TestCase):
+    def test_function(self):
+        def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined:
+            pass
+
+        with self.assertRaises(NameError):
+            func.__annotations__
+
+        undefined = 1
+        self.assertEqual(func.__annotations__, {
+            "x": 1,
+            "y": 1,
+            "args": 1,
+            "z": 1,
+            "kwargs": 1,
+            "return": 1,
+        })
+
+    def test_async_function(self):
+        async def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined:
+            pass
+
+        with self.assertRaises(NameError):
+            func.__annotations__
+
+        undefined = 1
+        self.assertEqual(func.__annotations__, {
+            "x": 1,
+            "y": 1,
+            "args": 1,
+            "z": 1,
+            "kwargs": 1,
+            "return": 1,
+        })
+
+    def test_class(self):
+        class X:
+            a: undefined
+
+        with self.assertRaises(NameError):
+            X.__annotations__
+
+        undefined = 1
+        self.assertEqual(X.__annotations__, {"a": 1})
+
+    def test_module(self):
+        ns = run_code("x: undefined = 1")
+        anno = ns["__annotate__"]
+        with self.assertRaises(NotImplementedError):
+            anno(2)
+
+        with self.assertRaises(NameError):
+            anno(1)
+
+        ns["undefined"] = 1
+        self.assertEqual(anno(1), {"x": 1})
+
+    def test_class_scoping(self):
+        class Outer:
+            def meth(self, x: Nested): ...
+            x: Nested
+            class Nested: ...
+
+        self.assertEqual(Outer.meth.__annotations__, {"x": Outer.Nested})
+        self.assertEqual(Outer.__annotations__, {"x": Outer.Nested})
+
+    def test_no_exotic_expressions(self):
+        check_syntax_error(self, "def func(x: (yield)): ...", "yield expression cannot be used within an annotation")
+        check_syntax_error(self, "def func(x: (yield from x)): ...", "yield expression cannot be used within an annotation")
+        check_syntax_error(self, "def func(x: (y := 3)): ...", "named expression cannot be used within an annotation")
+        check_syntax_error(self, "def func(x: (await 42)): ...", "await expression cannot be used within an annotation")
+
+    def test_no_exotic_expressions_in_unevaluated_annotations(self):
+        preludes = [
+            "",
+            "class X: ",
+            "def f(): ",
+            "async def f(): ",
+        ]
+        for prelude in preludes:
+            with self.subTest(prelude=prelude):
+                check_syntax_error(self, prelude + "(x): (yield)", "yield expression cannot be used within an annotation")
+                check_syntax_error(self, prelude + "(x): (yield from x)", "yield expression cannot be used within an annotation")
+                check_syntax_error(self, prelude + "(x): (y := 3)", "named expression cannot be used within an annotation")
+                check_syntax_error(self, prelude + "(x): (await 42)", "await expression cannot be used within an annotation")
+
+    def test_ignore_non_simple_annotations(self):
+        ns = run_code("class X: (y): int")
+        self.assertEqual(ns["X"].__annotations__, {})
+        ns = run_code("class X: int.b: int")
+        self.assertEqual(ns["X"].__annotations__, {})
+        ns = run_code("class X: int[str]: int")
+        self.assertEqual(ns["X"].__annotations__, {})
+
+    def test_generated_annotate(self):
+        def func(x: int):
+            pass
+        class X:
+            x: int
+        mod = build_module("x: int")
+        for obj in (func, X, mod):
+            with self.subTest(obj=obj):
+                annotate = obj.__annotate__
+                self.assertIsInstance(annotate, types.FunctionType)
+                self.assertEqual(annotate.__name__, "__annotate__")
+                with self.assertRaises(NotImplementedError):
+                    annotate(FORWARDREF)
+                with self.assertRaises(NotImplementedError):
+                    annotate(SOURCE)
+                with self.assertRaises(NotImplementedError):
+                    annotate(None)
+                self.assertEqual(annotate(VALUE), {"x": int})
+
+    def test_comprehension_in_annotation(self):
+        # This crashed in an earlier version of the code
+        ns = run_code("x: [y for y in range(10)]")
+        self.assertEqual(ns["__annotate__"](1), {"x": list(range(10))})
+
+    def test_future_annotations(self):
+        code = """
+        from __future__ import annotations
+
+        def f(x: int) -> int: pass
+        """
+        ns = run_code(code)
+        f = ns["f"]
+        self.assertIsInstance(f.__annotate__, types.FunctionType)
+        annos = {"x": "int", "return": "int"}
+        self.assertEqual(f.__annotate__(VALUE), annos)
+        self.assertEqual(f.__annotations__, annos)
+
+    def test_name_clash_with_format(self):
+        # this test would fail if __annotate__'s parameter was called "format"
+        code = """
+        class format: pass
+
+        def f(x: format): pass
+        """
+        ns = run_code(code)
+        f = ns["f"]
+        self.assertEqual(f.__annotations__, {"x": ns["format"]})

Lib/test/test_typing.py

@@ -6634,7 +6634,7 @@ def test_get_type_hints_from_various_objects(self):
             gth(None)
 
     def test_get_type_hints_modules(self):
-        ann_module_type_hints = {1: 2, 'f': Tuple[int, int], 'x': int, 'y': str, 'u': int | float}
+        ann_module_type_hints = {'f': Tuple[int, int], 'x': int, 'y': str, 'u': int | float}
         self.assertEqual(gth(ann_module), ann_module_type_hints)
         self.assertEqual(gth(ann_module2), {})
         self.assertEqual(gth(ann_module3), {})
@@ -6652,7 +6652,7 @@ def test_get_type_hints_classes(self):
         self.assertEqual(gth(ann_module.C),  # gth will find the right globalns
                          {'y': Optional[ann_module.C]})
         self.assertIsInstance(gth(ann_module.j_class), dict)
-        self.assertEqual(gth(ann_module.M), {'123': 123, 'o': type})
+        self.assertEqual(gth(ann_module.M), {'o': type})
         self.assertEqual(gth(ann_module.D),
                          {'j': str, 'k': str, 'y': Optional[ann_module.C]})
         self.assertEqual(gth(ann_module.Y), {'z': int})

Lib/test/typinganndata/ann_module.py

@@ -8,8 +8,6 @@
 from typing import Optional
 from functools import wraps
 
-__annotations__[1] = 2
-
 class C:
 
     x = 5; y: Optional['C'] = None
@@ -18,8 +16,6 @@ class C:
 x: int = 5; y: str = x; f: Tuple[int, int]
 
 class M(type):
-
-    __annotations__['123'] = 123
     o: type = object
 
 (pars): bool = True

Lib/typing.py

@@ -2412,7 +2412,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False):
                 base_globals = getattr(sys.modules.get(base.__module__, None), '__dict__', {})
             else:
                 base_globals = globalns
-            ann = base.__dict__.get('__annotations__', {})
+            ann = getattr(base, '__annotations__', {})
             if isinstance(ann, types.GetSetDescriptorType):
                 ann = {}
             base_locals = dict(vars(base)) if localns is None else localns
@@ -2970,7 +2970,12 @@ def __new__(cls, typename, bases, ns):
                 raise TypeError(
                     'can only inherit from a NamedTuple type and Generic')
         bases = tuple(tuple if base is _NamedTuple else base for base in bases)
-        types = ns.get('__annotations__', {})
+        if "__annotations__" in ns:
+            types = ns["__annotations__"]
+        elif "__annotate__" in ns:
+            types = ns["__annotate__"](1)  # VALUE
+        else:
+            types = {}
         default_names = []
         for field_name in types:
             if field_name in ns:
@@ -3131,7 +3136,12 @@ def __new__(cls, name, bases, ns, total=True):
             tp_dict.__orig_bases__ = bases
 
         annotations = {}
-        own_annotations = ns.get('__annotations__', {})
+        if "__annotations__" in ns:
+            own_annotations = ns["__annotations__"]
+        elif "__annotate__" in ns:
+            own_annotations = ns["__annotate__"](1)  # VALUE
+        else:
+            own_annotations = {}
         msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type"
         own_annotations = {
             n: _type_check(tp, msg, module=tp_dict.__module__)
@@ -3143,7 +3153,12 @@ def __new__(cls, name, bases, ns, total=True):
         mutable_keys = set()
 
         for base in bases:
-            annotations.update(base.__dict__.get('__annotations__', {}))
+            # TODO: Avoid eagerly evaluating annotations in VALUE format.
+            # Instead, evaluate in FORWARDREF format to figure out which
+            # keys have Required/NotRequired/ReadOnly qualifiers, and create
+            # a new __annotate__ function for the resulting TypedDict that
+            # combines the annotations from this class and its parents.
+            annotations.update(base.__annotations__)
 
             base_required = base.__dict__.get('__required_keys__', set())
             required_keys |= base_required

Misc/NEWS.d/next/Core and Builtins/2024-05-22-06-22-47.gh-issue-119180.vZMiXm.rst

@@ -0,0 +1 @@
+Evaluation of annotations is now deferred. See :pep:`649` for details.

Python/bytecodes.c

@@ -3975,6 +3975,11 @@ dummy_func(
                     assert(func_obj->func_defaults == NULL);
                     func_obj->func_defaults = attr;
                     break;
+                case MAKE_FUNCTION_ANNOTATE:
+                    assert(PyCallable_Check(attr));
+                    assert(func_obj->func_annotate == NULL);
+                    func_obj->func_annotate = attr;
+                    break;
                 default:
                     Py_UNREACHABLE();
             }

Python/compile.c

@@ -132,7 +132,7 @@ enum {
     COMPILER_SCOPE_ASYNC_FUNCTION,
     COMPILER_SCOPE_LAMBDA,
     COMPILER_SCOPE_COMPREHENSION,
-    COMPILER_SCOPE_TYPEPARAMS,
+    COMPILER_SCOPE_ANNOTATIONS,
 };
 
 
@@ -142,6 +142,15 @@ typedef _PyInstructionSequence instr_sequence;
 #define INITIAL_INSTR_SEQUENCE_SIZE 100
 #define INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE 10
 
+static const int compare_masks[] = {
+    [Py_LT] = COMPARISON_LESS_THAN,
+    [Py_LE] = COMPARISON_LESS_THAN | COMPARISON_EQUALS,
+    [Py_EQ] = COMPARISON_EQUALS,
+    [Py_NE] = COMPARISON_NOT_EQUALS,
+    [Py_GT] = COMPARISON_GREATER_THAN,
+    [Py_GE] = COMPARISON_GREATER_THAN | COMPARISON_EQUALS,
+};
+
 /*
  * Resize the array if index is out of range.
  *
@@ -208,6 +217,7 @@ struct compiler_unit {
 
     PyObject *u_private;            /* for private name mangling */
     PyObject *u_static_attributes;  /* for class: attributes accessed via self.X */
+    PyObject *u_deferred_annotations; /* AnnAssign nodes deferred to the end of compilation */
 
     instr_sequence *u_instr_sequence; /* codegen output */
 
@@ -330,6 +340,8 @@ static int compiler_pattern(struct compiler *, pattern_ty, pattern_context *);
 static int compiler_match(struct compiler *, stmt_ty);
 static int compiler_pattern_subpattern(struct compiler *,
                                        pattern_ty, pattern_context *);
+static int compiler_make_closure(struct compiler *c, location loc,
+                                 PyCodeObject *co, Py_ssize_t flags);
 
 static PyCodeObject *optimize_and_assemble(struct compiler *, int addNone);
 
@@ -545,6 +557,7 @@ compiler_unit_free(struct compiler_unit *u)
     Py_CLEAR(u->u_metadata.u_fasthidden);
     Py_CLEAR(u->u_private);
     Py_CLEAR(u->u_static_attributes);
+    Py_CLEAR(u->u_deferred_annotations);
     PyMem_Free(u);
 }
 
@@ -582,8 +595,8 @@ compiler_set_qualname(struct compiler *c)
         capsule = PyList_GET_ITEM(c->c_stack, stack_size - 1);
         parent = (struct compiler_unit *)PyCapsule_GetPointer(capsule, CAPSULE_NAME);
         assert(parent);
-        if (parent->u_scope_type == COMPILER_SCOPE_TYPEPARAMS) {
+        if (parent->u_scope_type == COMPILER_SCOPE_ANNOTATIONS) {
-            /* The parent is a type parameter scope, so we need to
+            /* The parent is an annotation scope, so we need to
                look at the grandparent. */
             if (stack_size == 2) {
                 // If we're immediately within the module, we can skip
@@ -1128,6 +1141,7 @@ compiler_enter_scope(struct compiler *c, identifier name,
     }
 
     u->u_private = NULL;
+    u->u_deferred_annotations = NULL;
     if (scope_type == COMPILER_SCOPE_CLASS) {
         u->u_static_attributes = PySet_New(0);
         if (!u->u_static_attributes) {
@@ -1209,85 +1223,6 @@ compiler_exit_scope(struct compiler *c)
     PyErr_SetRaisedException(exc);
 }
 
-/* Search if variable annotations are present statically in a block. */
-
-static bool
-find_ann(asdl_stmt_seq *stmts)
-{
-    int i, j, res = 0;
-    stmt_ty st;
-
-    for (i = 0; i < asdl_seq_LEN(stmts); i++) {
-        st = (stmt_ty)asdl_seq_GET(stmts, i);
-        switch (st->kind) {
-        case AnnAssign_kind:
-            return true;
-        case For_kind:
-            res = find_ann(st->v.For.body) ||
-                  find_ann(st->v.For.orelse);
-            break;
-        case AsyncFor_kind:
-            res = find_ann(st->v.AsyncFor.body) ||
-                  find_ann(st->v.AsyncFor.orelse);
-            break;
-        case While_kind:
-            res = find_ann(st->v.While.body) ||
-                  find_ann(st->v.While.orelse);
-            break;
-        case If_kind:
-            res = find_ann(st->v.If.body) ||
-                  find_ann(st->v.If.orelse);
-            break;
-        case With_kind:
-            res = find_ann(st->v.With.body);
-            break;
-        case AsyncWith_kind:
-            res = find_ann(st->v.AsyncWith.body);
-            break;
-        case Try_kind:
-            for (j = 0; j < asdl_seq_LEN(st->v.Try.handlers); j++) {
-                excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET(
-                    st->v.Try.handlers, j);
-                if (find_ann(handler->v.ExceptHandler.body)) {
-                    return true;
-                }
-            }
-            res = find_ann(st->v.Try.body) ||
-                  find_ann(st->v.Try.finalbody) ||
-                  find_ann(st->v.Try.orelse);
-            break;
-        case TryStar_kind:
-            for (j = 0; j < asdl_seq_LEN(st->v.TryStar.handlers); j++) {
-                excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET(
-                    st->v.TryStar.handlers, j);
-                if (find_ann(handler->v.ExceptHandler.body)) {
-                    return true;
-                }
-            }
-            res = find_ann(st->v.TryStar.body) ||
-                  find_ann(st->v.TryStar.finalbody) ||
-                  find_ann(st->v.TryStar.orelse);
-            break;
-        case Match_kind:
-            for (j = 0; j < asdl_seq_LEN(st->v.Match.cases); j++) {
-                match_case_ty match_case = (match_case_ty)asdl_seq_GET(
-                    st->v.Match.cases, j);
-                if (find_ann(match_case->body)) {
-                    return true;
-                }
-            }
-            break;
-        default:
-            res = false;
-            break;
-        }
-        if (res) {
-            break;
-        }
-    }
-    return res;
-}
-
 /*
  * Frame block handling functions
  */
@@ -1502,6 +1437,47 @@ compiler_unwind_fblock_stack(struct compiler *c, location *ploc,
     return SUCCESS;
 }
 
+static int
+compiler_setup_annotations_scope(struct compiler *c, location loc,
+                                 void *key, PyObject *name)
+{
+    if (compiler_enter_scope(c, name, COMPILER_SCOPE_ANNOTATIONS,
+                             key, loc.lineno) == -1) {
+        return ERROR;
+    }
+    c->u->u_metadata.u_posonlyargcount = 1;
+    // if .format != 1: raise NotImplementedError
+    _Py_DECLARE_STR(format, ".format");
+    ADDOP_I(c, loc, LOAD_FAST, 0);
+    ADDOP_LOAD_CONST(c, loc, _PyLong_GetOne());
+    ADDOP_I(c, loc, COMPARE_OP, (Py_NE << 5) | compare_masks[Py_NE]);
+    NEW_JUMP_TARGET_LABEL(c, body);
+    ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, body);
+    ADDOP_I(c, loc, LOAD_COMMON_CONSTANT, CONSTANT_NOTIMPLEMENTEDERROR);
+    ADDOP_I(c, loc, RAISE_VARARGS, 1);
+    USE_LABEL(c, body);
+    return 0;
+}
+
+static int
+compiler_leave_annotations_scope(struct compiler *c, location loc,
+                                 Py_ssize_t annotations_len)
+{
+    ADDOP_I(c, loc, BUILD_MAP, annotations_len);
+    ADDOP_IN_SCOPE(c, loc, RETURN_VALUE);
+    PyCodeObject *co = optimize_and_assemble(c, 1);
+    compiler_exit_scope(c);
+    if (co == NULL) {
+        return ERROR;
+    }
+    if (compiler_make_closure(c, loc, co, 0) < 0) {
+        Py_DECREF(co);
+        return ERROR;
+    }
+    Py_DECREF(co);
+    return 0;
+}
+
 /* Compile a sequence of statements, checking for a docstring
    and for annotations. */
 
@@ -1517,34 +1493,79 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts)
         stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0);
         loc = LOC(st);
     }
-    /* Every annotated class and module should have __annotations__. */
+    /* If from __future__ import annotations is active,
-    if (find_ann(stmts)) {
+     * every annotated class and module should have __annotations__.
+     * Else __annotate__ is created when necessary. */
+    if ((c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) && c->u->u_ste->ste_annotations_used) {
         ADDOP(c, loc, SETUP_ANNOTATIONS);
     }
     if (!asdl_seq_LEN(stmts)) {
         return SUCCESS;
     }
     Py_ssize_t first_instr = 0;
-    PyObject *docstring = _PyAST_GetDocString(stmts);
+    if (!c->c_interactive) {
-    if (docstring) {
+        PyObject *docstring = _PyAST_GetDocString(stmts);
-        first_instr = 1;
+        if (docstring) {
-        /* if not -OO mode, set docstring */
+            first_instr = 1;
-        if (c->c_optimize < 2) {
+            /* if not -OO mode, set docstring */
-            PyObject *cleandoc = _PyCompile_CleanDoc(docstring);
+            if (c->c_optimize < 2) {
-            if (cleandoc == NULL) {
+                PyObject *cleandoc = _PyCompile_CleanDoc(docstring);
-                return ERROR;
+                if (cleandoc == NULL) {
+                    return ERROR;
+                }
+                stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0);
+                assert(st->kind == Expr_kind);
+                location loc = LOC(st->v.Expr.value);
+                ADDOP_LOAD_CONST(c, loc, cleandoc);
+                Py_DECREF(cleandoc);
+                RETURN_IF_ERROR(compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store));
             }
-            stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0);
-            assert(st->kind == Expr_kind);
-            location loc = LOC(st->v.Expr.value);
-            ADDOP_LOAD_CONST(c, loc, cleandoc);
-            Py_DECREF(cleandoc);
-            RETURN_IF_ERROR(compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store));
         }
     }
     for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(stmts); i++) {
         VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i));
     }
+    // If there are annotations and the future import is not on, we
+    // collect the annotations in a separate pass and generate an
+    // __annotate__ function. See PEP 649.
+    if (!(c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) &&
+         c->u->u_deferred_annotations != NULL) {
+
+        // It's possible that ste_annotations_block is set but
+        // u_deferred_annotations is not, because the former is still
+        // set if there are only non-simple annotations (i.e., annotations
+        // for attributes, subscripts, or parenthesized names). However, the
+        // reverse should not be possible.
+        assert(c->u->u_ste->ste_annotation_block != NULL);
+        PyObject *deferred_anno = Py_NewRef(c->u->u_deferred_annotations);
+        void *key = (void *)((uintptr_t)c->u->u_ste->ste_id + 1);
+        if (compiler_setup_annotations_scope(c, loc, key,
+                                             c->u->u_ste->ste_annotation_block->ste_name) == -1) {
+            Py_DECREF(deferred_anno);
+            return ERROR;
+        }
+        Py_ssize_t annotations_len = PyList_Size(deferred_anno);
+        for (Py_ssize_t i = 0; i < annotations_len; i++) {
+            PyObject *ptr = PyList_GET_ITEM(deferred_anno, i);
+            stmt_ty st = (stmt_ty)PyLong_AsVoidPtr(ptr);
+            if (st == NULL) {
+                compiler_exit_scope(c);
+                Py_DECREF(deferred_anno);
+                return ERROR;
+            }
+            PyObject *mangled = _Py_Mangle(c->u->u_private, st->v.AnnAssign.target->v.Name.id);
+            ADDOP_LOAD_CONST_NEW(c, LOC(st), mangled);
+            VISIT(c, expr, st->v.AnnAssign.annotation);
+        }
+        Py_DECREF(deferred_anno);
+
+        RETURN_IF_ERROR(
+            compiler_leave_annotations_scope(c, loc, annotations_len)
+        );
+        RETURN_IF_ERROR(
+            compiler_nameop(c, loc, &_Py_ID(__annotate__), Store)
+        );
+    }
     return SUCCESS;
 }
 
@@ -1559,11 +1580,10 @@ compiler_codegen(struct compiler *c, mod_ty mod)
         }
         break;
     case Interactive_kind:
-        if (find_ann(mod->v.Interactive.body)) {
-            ADDOP(c, loc, SETUP_ANNOTATIONS);
-        }
         c->c_interactive = 1;
-        VISIT_SEQ(c, stmt, mod->v.Interactive.body);
+        if (compiler_body(c, loc, mod->v.Interactive.body) < 0) {
+            return ERROR;
+        }
         break;
     case Expression_kind:
         VISIT(c, expr, mod->v.Expression.body);
@@ -1702,6 +1722,9 @@ compiler_make_closure(struct compiler *c, location loc,
     if (flags & MAKE_FUNCTION_ANNOTATIONS) {
         ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATIONS);
     }
+    if (flags & MAKE_FUNCTION_ANNOTATE) {
+        ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATE);
+    }
     if (flags & MAKE_FUNCTION_KWDEFAULTS) {
         ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_KWDEFAULTS);
     }
@@ -1833,7 +1856,7 @@ compiler_visit_argannotation(struct compiler *c, identifier id,
             VISIT(c, expr, annotation);
         }
     }
-    *annotations_len += 2;
+    *annotations_len += 1;
     return SUCCESS;
 }
 
@@ -1856,43 +1879,76 @@ compiler_visit_argannotations(struct compiler *c, asdl_arg_seq* args,
 }
 
 static int
-compiler_visit_annotations(struct compiler *c, location loc,
+compiler_visit_annotations_in_scope(struct compiler *c, location loc,
-                           arguments_ty args, expr_ty returns)
+                                    arguments_ty args, expr_ty returns,
+                                    Py_ssize_t *annotations_len)
 {
-    /* Push arg annotation names and values.
+    RETURN_IF_ERROR(
-       The expressions are evaluated out-of-order wrt the source code.
+        compiler_visit_argannotations(c, args->args, annotations_len, loc));
-
-       Return -1 on error, 0 if no annotations pushed, 1 if a annotations is pushed.
-       */
-    Py_ssize_t annotations_len = 0;
 
     RETURN_IF_ERROR(
-        compiler_visit_argannotations(c, args->args, &annotations_len, loc));
+        compiler_visit_argannotations(c, args->posonlyargs, annotations_len, loc));
-
-    RETURN_IF_ERROR(
-        compiler_visit_argannotations(c, args->posonlyargs, &annotations_len, loc));
 
     if (args->vararg && args->vararg->annotation) {
         RETURN_IF_ERROR(
             compiler_visit_argannotation(c, args->vararg->arg,
-                                         args->vararg->annotation, &annotations_len, loc));
+                                         args->vararg->annotation, annotations_len, loc));
     }
 
     RETURN_IF_ERROR(
-        compiler_visit_argannotations(c, args->kwonlyargs, &annotations_len, loc));
+        compiler_visit_argannotations(c, args->kwonlyargs, annotations_len, loc));
 
     if (args->kwarg && args->kwarg->annotation) {
         RETURN_IF_ERROR(
             compiler_visit_argannotation(c, args->kwarg->arg,
-                                         args->kwarg->annotation, &annotations_len, loc));
+                                         args->kwarg->annotation, annotations_len, loc));
     }
 
     RETURN_IF_ERROR(
-        compiler_visit_argannotation(c, &_Py_ID(return), returns, &annotations_len, loc));
+        compiler_visit_argannotation(c, &_Py_ID(return), returns, annotations_len, loc));
 
-    if (annotations_len) {
+    return 0;
-        ADDOP_I(c, loc, BUILD_TUPLE, annotations_len);
+}
-        return 1;
+
+static int
+compiler_visit_annotations(struct compiler *c, location loc,
+                           arguments_ty args, expr_ty returns)
+{
+    /* Push arg annotation names and values.
+       The expressions are evaluated separately from the rest of the source code.
+
+       Return -1 on error, or a combination of flags to add to the function.
+       */
+    Py_ssize_t annotations_len = 0;
+
+    PySTEntryObject *ste;
+    if (_PySymtable_LookupOptional(c->c_st, args, &ste) < 0) {
+        return ERROR;
+    }
+    assert(ste != NULL);
+    bool annotations_used = ste->ste_annotations_used;
+
+    if (annotations_used) {
+        if (compiler_setup_annotations_scope(c, loc, (void *)args,
+                                             ste->ste_name) < 0) {
+            Py_DECREF(ste);
+            return ERROR;
+        }
+    }
+    Py_DECREF(ste);
+
+    if (compiler_visit_annotations_in_scope(c, loc, args, returns, &annotations_len) < 0) {
+        if (annotations_used) {
+            compiler_exit_scope(c);
+        }
+        return ERROR;
+    }
+
+    if (annotations_used) {
+        RETURN_IF_ERROR(
+            compiler_leave_annotations_scope(c, loc, annotations_len)
+        );
+        return MAKE_FUNCTION_ANNOTATE;
     }
 
     return 0;
@@ -2001,7 +2057,7 @@ compiler_type_param_bound_or_default(struct compiler *c, expr_ty e,
                                      identifier name, void *key,
                                      bool allow_starred)
 {
-    if (compiler_enter_scope(c, name, COMPILER_SCOPE_TYPEPARAMS,
+    if (compiler_enter_scope(c, name, COMPILER_SCOPE_ANNOTATIONS,
                              key, e->lineno) == -1) {
         return ERROR;
     }
@@ -2220,7 +2276,6 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async)
     asdl_expr_seq *decos;
     asdl_type_param_seq *type_params;
     Py_ssize_t funcflags;
-    int annotations;
     int firstlineno;
 
     if (is_async) {
@@ -2274,7 +2329,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async)
         if (!type_params_name) {
             return ERROR;
         }
-        if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS,
+        if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS,
                                  (void *)type_params, firstlineno) == -1) {
             Py_DECREF(type_params_name);
             return ERROR;
@@ -2286,16 +2341,14 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async)
         }
     }
 
-    annotations = compiler_visit_annotations(c, loc, args, returns);
+    int annotations_flag = compiler_visit_annotations(c, loc, args, returns);
-    if (annotations < 0) {
+    if (annotations_flag < 0) {
         if (is_generic) {
             compiler_exit_scope(c);
         }
         return ERROR;
     }
-    if (annotations > 0) {
+    funcflags |= annotations_flag;
-        funcflags |= MAKE_FUNCTION_ANNOTATIONS;
-    }
 
     if (compiler_function_body(c, s, is_async, funcflags, firstlineno) < 0) {
         if (is_generic) {
@@ -2510,7 +2563,7 @@ compiler_class(struct compiler *c, stmt_ty s)
         if (!type_params_name) {
             return ERROR;
         }
-        if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS,
+        if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS,
                                  (void *)type_params, firstlineno) == -1) {
             Py_DECREF(type_params_name);
             return ERROR;
@@ -2630,7 +2683,7 @@ compiler_typealias(struct compiler *c, stmt_ty s)
         if (!type_params_name) {
             return ERROR;
         }
-        if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS,
+        if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS,
                                  (void *)type_params, loc.lineno) == -1) {
             Py_DECREF(type_params_name);
             return ERROR;
@@ -2719,15 +2772,6 @@ check_compare(struct compiler *c, expr_ty e)
     return SUCCESS;
 }
 
-static const int compare_masks[] = {
-    [Py_LT] = COMPARISON_LESS_THAN,
-    [Py_LE] = COMPARISON_LESS_THAN | COMPARISON_EQUALS,
-    [Py_EQ] = COMPARISON_EQUALS,
-    [Py_NE] = COMPARISON_NOT_EQUALS,
-    [Py_GT] = COMPARISON_GREATER_THAN,
-    [Py_GE] = COMPARISON_GREATER_THAN | COMPARISON_EQUALS,
-};
-
 static int compiler_addcompare(struct compiler *c, location loc,
                                cmpop_ty op)
 {
@@ -6366,7 +6410,8 @@ compiler_annassign(struct compiler *c, stmt_ty s)
 {
     location loc = LOC(s);
     expr_ty targ = s->v.AnnAssign.target;
-    PyObject* mangled;
+    bool future_annotations = c->c_future.ff_features & CO_FUTURE_ANNOTATIONS;
+    PyObject *mangled;
 
     assert(s->kind == AnnAssign_kind);
 
@@ -6384,16 +6429,30 @@ compiler_annassign(struct compiler *c, stmt_ty s)
         if (s->v.AnnAssign.simple &&
             (c->u->u_scope_type == COMPILER_SCOPE_MODULE ||
              c->u->u_scope_type == COMPILER_SCOPE_CLASS)) {
-            if (c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) {
+            if (future_annotations) {
-                VISIT(c, annexpr, s->v.AnnAssign.annotation)
+                VISIT(c, annexpr, s->v.AnnAssign.annotation);
+                ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names);
+                mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, targ->v.Name.id);
+                ADDOP_LOAD_CONST_NEW(c, loc, mangled);
+                ADDOP(c, loc, STORE_SUBSCR);
             }
             else {
-                VISIT(c, expr, s->v.AnnAssign.annotation);
+                if (c->u->u_deferred_annotations == NULL) {
+                    c->u->u_deferred_annotations = PyList_New(0);
+                    if (c->u->u_deferred_annotations == NULL) {
+                        return ERROR;
+                    }
+                }
+                PyObject *ptr = PyLong_FromVoidPtr((void *)s);
+                if (ptr == NULL) {
+                    return ERROR;
+                }
+                if (PyList_Append(c->u->u_deferred_annotations, ptr) < 0) {
+                    Py_DECREF(ptr);
+                    return ERROR;
+                }
+                Py_DECREF(ptr);
             }
-            ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names);
-            mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, targ->v.Name.id);
-            ADDOP_LOAD_CONST_NEW(c, loc, mangled);
-            ADDOP(c, loc, STORE_SUBSCR);
         }
         break;
     case Attribute_kind:
@@ -6419,7 +6478,7 @@ compiler_annassign(struct compiler *c, stmt_ty s)
         return ERROR;
     }
     /* Annotation is evaluated last. */
-    if (!s->v.AnnAssign.simple && check_annotation(c, s) < 0) {
+    if (future_annotations && !s->v.AnnAssign.simple && check_annotation(c, s) < 0) {
         return ERROR;
     }
     return SUCCESS;

Python/executor_cases.c.h

@@ -4061,6 +4061,11 @@
                 assert(func_obj->func_defaults == NULL);
                 func_obj->func_defaults = attr;
                 break;
+                case MAKE_FUNCTION_ANNOTATE:
+                assert(PyCallable_Check(attr));
+                assert(func_obj->func_annotate == NULL);
+                func_obj->func_annotate = attr;
+                break;
                 default:
                 Py_UNREACHABLE();
             }

Python/generated_cases.c.h

@@ -5450,6 +5450,11 @@
                 assert(func_obj->func_defaults == NULL);
                 func_obj->func_defaults = attr;
                 break;
+                case MAKE_FUNCTION_ANNOTATE:
+                assert(PyCallable_Check(attr));
+                assert(func_obj->func_annotate == NULL);
+                func_obj->func_annotate = attr;
+                break;
                 default:
                 Py_UNREACHABLE();
             }

Python/symtable.c

@@ -112,6 +112,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block,
     ste->ste_varkeywords = 0;
     ste->ste_opt_lineno = 0;
     ste->ste_opt_col_offset = 0;
+    ste->ste_annotations_used = 0;
     ste->ste_lineno = lineno;
     ste->ste_col_offset = col_offset;
     ste->ste_end_lineno = end_lineno;
@@ -132,6 +133,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block,
     ste->ste_can_see_class_scope = 0;
     ste->ste_comp_iter_expr = 0;
     ste->ste_needs_classdict = 0;
+    ste->ste_annotation_block = NULL;
 
     ste->ste_symbols = PyDict_New();
     ste->ste_varnames = PyList_New(0);
@@ -167,6 +169,7 @@ ste_dealloc(PySTEntryObject *ste)
     Py_XDECREF(ste->ste_varnames);
     Py_XDECREF(ste->ste_children);
     Py_XDECREF(ste->ste_directives);
+    Py_XDECREF(ste->ste_annotation_block);
     Py_XDECREF(ste->ste_mangled_names);
     PyObject_Free(ste);
 }
@@ -245,10 +248,11 @@ static int symtable_visit_alias(struct symtable *st, alias_ty);
 static int symtable_visit_comprehension(struct symtable *st, comprehension_ty);
 static int symtable_visit_keyword(struct symtable *st, keyword_ty);
 static int symtable_visit_params(struct symtable *st, asdl_arg_seq *args);
-static int symtable_visit_annotation(struct symtable *st, expr_ty annotation);
+static int symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key);
 static int symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args);
 static int symtable_implicit_arg(struct symtable *st, int pos);
-static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty);
+static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty,
+                                      struct _symtable_entry *parent_ste);
 static int symtable_visit_withitem(struct symtable *st, withitem_ty item);
 static int symtable_visit_match_case(struct symtable *st, match_case_ty m);
 static int symtable_visit_pattern(struct symtable *st, pattern_ty s);
@@ -504,6 +508,21 @@ _PySymtable_Lookup(struct symtable *st, void *key)
     return (PySTEntryObject *)v;
 }
 
+int
+_PySymtable_LookupOptional(struct symtable *st, void *key,
+                           PySTEntryObject **out)
+{
+    PyObject *k = PyLong_FromVoidPtr(key);
+    if (k == NULL) {
+        *out = NULL;
+        return -1;
+    }
+    int result = PyDict_GetItemRef(st->st_blocks, k, (PyObject **)out);
+    Py_DECREF(k);
+    assert(*out == NULL || PySTEntry_Check(*out));
+    return result;
+}
+
 long
 _PyST_GetSymbol(PySTEntryObject *ste, PyObject *name)
 {
@@ -525,6 +544,7 @@ int
 _PyST_IsFunctionLike(PySTEntryObject *ste)
 {
     return ste->ste_type == FunctionBlock
+        || ste->ste_type == AnnotationBlock
         || ste->ste_type == TypeVarBoundBlock
         || ste->ste_type == TypeAliasBlock
         || ste->ste_type == TypeParamBlock;
@@ -1317,20 +1337,12 @@ symtable_exit_block(struct symtable *st)
 }
 
 static int
-symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
+symtable_enter_existing_block(struct symtable *st, PySTEntryObject* ste)
-                     void *ast, int lineno, int col_offset,
-                     int end_lineno, int end_col_offset)
 {
-    PySTEntryObject *prev = NULL, *ste;
-
-    ste = ste_new(st, name, block, ast, lineno, col_offset, end_lineno, end_col_offset);
-    if (ste == NULL)
-        return 0;
     if (PyList_Append(st->st_stack, (PyObject *)ste) < 0) {
-        Py_DECREF(ste);
         return 0;
     }
-    prev = st->st_cur;
+    PySTEntryObject *prev = st->st_cur;
     /* bpo-37757: For now, disallow *all* assignment expressions in the
      * outermost iterator expression of a comprehension, even those inside
      * a nested comprehension or a lambda expression.
@@ -1340,21 +1352,20 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
     }
     /* No need to inherit ste_mangled_names in classes, where all names
      * are mangled. */
-    if (prev && prev->ste_mangled_names != NULL && block != ClassBlock) {
+    if (prev && prev->ste_mangled_names != NULL && ste->ste_type != ClassBlock) {
         ste->ste_mangled_names = Py_NewRef(prev->ste_mangled_names);
     }
     /* The entry is owned by the stack. Borrow it for st_cur. */
-    Py_DECREF(ste);
     st->st_cur = ste;
 
-    /* Annotation blocks shouldn't have any affect on the symbol table since in
+    /* If "from __future__ import annotations" is active,
-     * the compilation stage, they will all be transformed to strings. They are
+     * annotation blocks shouldn't have any affect on the symbol table since in
-     * only created if future 'annotations' feature is activated. */
+     * the compilation stage, they will all be transformed to strings. */
-    if (block == AnnotationBlock) {
+    if (st->st_future->ff_features & CO_FUTURE_ANNOTATIONS && ste->ste_type == AnnotationBlock) {
         return 1;
     }
 
-    if (block == ModuleBlock)
+    if (ste->ste_type == ModuleBlock)
         st->st_global = st->st_cur->ste_symbols;
 
     if (prev) {
@@ -1365,6 +1376,20 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
     return 1;
 }
 
+static int
+symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
+                     void *ast, int lineno, int col_offset,
+                     int end_lineno, int end_col_offset)
+{
+    PySTEntryObject *ste = ste_new(st, name, block, ast,
+                                   lineno, col_offset, end_lineno, end_col_offset);
+    if (ste == NULL)
+        return 0;
+    int result = symtable_enter_existing_block(st, ste);
+    Py_DECREF(ste);
+    return result;
+}
+
 static long
 symtable_lookup_entry(struct symtable *st, PySTEntryObject *ste, PyObject *name)
 {
@@ -1643,7 +1668,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
         VISIT_QUIT(st, 0);
     }
     switch (s->kind) {
-    case FunctionDef_kind:
+    case FunctionDef_kind: {
         if (!symtable_add_def(st, s->v.FunctionDef.name, DEF_LOCAL, LOCATION(s)))
             VISIT_QUIT(st, 0);
         if (s->v.FunctionDef.args->defaults)
@@ -1665,13 +1690,22 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
             }
             VISIT_SEQ(st, type_param, s->v.FunctionDef.type_params);
         }
+        PySTEntryObject *new_ste = ste_new(st, s->v.FunctionDef.name, FunctionBlock, (void *)s,
+                                           LOCATION(s));
+        if (!new_ste) {
+            VISIT_QUIT(st, 0);
+        }
+
         if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args,
-                                        s->v.FunctionDef.returns))
+                                        s->v.FunctionDef.returns, new_ste)) {
+            Py_DECREF(new_ste);
             VISIT_QUIT(st, 0);
-        if (!symtable_enter_block(st, s->v.FunctionDef.name,
+        }
-                                  FunctionBlock, (void *)s,
+        if (!symtable_enter_existing_block(st, new_ste)) {
-                                  LOCATION(s)))
+            Py_DECREF(new_ste);
             VISIT_QUIT(st, 0);
+        }
+        Py_DECREF(new_ste);
         VISIT(st, arguments, s->v.FunctionDef.args);
         VISIT_SEQ(st, stmt, s->v.FunctionDef.body);
         if (!symtable_exit_block(st))
@@ -1681,6 +1715,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
                 VISIT_QUIT(st, 0);
         }
         break;
+    }
     case ClassDef_kind: {
         PyObject *tmp;
         if (!symtable_add_def(st, s->v.ClassDef.name, DEF_LOCAL, LOCATION(s)))
@@ -1776,6 +1811,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
         VISIT(st, expr, s->v.Assign.value);
         break;
     case AnnAssign_kind:
+        st->st_cur->ste_annotations_used = 1;
         if (s->v.AnnAssign.target->kind == Name_kind) {
             expr_ty e_name = s->v.AnnAssign.target;
             long cur = symtable_lookup(st, e_name->v.Name.id);
@@ -1810,7 +1846,8 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
         else {
             VISIT(st, expr, s->v.AnnAssign.target);
         }
-        if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation)) {
+        if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation,
+                                       (void *)((uintptr_t)st->st_cur->ste_id + 1))) {
             VISIT_QUIT(st, 0);
         }
 
@@ -1960,7 +1997,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
         VISIT_SEQ(st, withitem, s->v.With.items);
         VISIT_SEQ(st, stmt, s->v.With.body);
         break;
-    case AsyncFunctionDef_kind:
+    case AsyncFunctionDef_kind: {
         if (!symtable_add_def(st, s->v.AsyncFunctionDef.name, DEF_LOCAL, LOCATION(s)))
             VISIT_QUIT(st, 0);
         if (s->v.AsyncFunctionDef.args->defaults)
@@ -1983,14 +2020,21 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
             }
             VISIT_SEQ(st, type_param, s->v.AsyncFunctionDef.type_params);
         }
+        PySTEntryObject *new_ste = ste_new(st, s->v.FunctionDef.name, FunctionBlock, (void *)s,
+                                           LOCATION(s));
+        if (!new_ste) {
+            VISIT_QUIT(st, 0);
+        }
+
         if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args,
-                                        s->v.AsyncFunctionDef.returns))
+                                        s->v.AsyncFunctionDef.returns, new_ste))
             VISIT_QUIT(st, 0);
-        if (!symtable_enter_block(st, s->v.AsyncFunctionDef.name,
+        if (!symtable_enter_existing_block(st, new_ste)) {
-                                  FunctionBlock, (void *)s,
+            Py_DECREF(new_ste);
-                                  s->lineno, s->col_offset,
-                                  s->end_lineno, s->end_col_offset))
             VISIT_QUIT(st, 0);
+        }
+        Py_DECREF(new_ste);
+
         st->st_cur->ste_coroutine = 1;
         VISIT(st, arguments, s->v.AsyncFunctionDef.args);
         VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body);
@@ -2001,6 +2045,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
                 VISIT_QUIT(st, 0);
         }
         break;
+    }
     case AsyncWith_kind:
         VISIT_SEQ(st, withitem, s->v.AsyncWith.items);
         VISIT_SEQ(st, stmt, s->v.AsyncWith.body);
@@ -2444,18 +2489,44 @@ symtable_visit_params(struct symtable *st, asdl_arg_seq *args)
 }
 
 static int
-symtable_visit_annotation(struct symtable *st, expr_ty annotation)
+symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key)
 {
-    int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS;
+    struct _symtable_entry *parent_ste = st->st_cur;
-    if (future_annotations &&
+    if (parent_ste->ste_annotation_block == NULL) {
-        !symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock,
+        _Py_block_ty current_type = parent_ste->ste_type;
-                              (void *)annotation, annotation->lineno,
+        if (!symtable_enter_block(st, &_Py_ID(__annotate__), AnnotationBlock,
-                              annotation->col_offset, annotation->end_lineno,
+                                    key, LOCATION(annotation))) {
-                              annotation->end_col_offset)) {
+            VISIT_QUIT(st, 0);
-        VISIT_QUIT(st, 0);
+        }
+        parent_ste->ste_annotation_block =
+            (struct _symtable_entry *)Py_NewRef(st->st_cur);
+        int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS;
+        if (current_type == ClassBlock && !future_annotations) {
+            st->st_cur->ste_can_see_class_scope = 1;
+            if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(annotation))) {
+                return 0;
+            }
+        }
+
+        _Py_DECLARE_STR(format, ".format");
+        // The generated __annotate__ function takes a single parameter with the
+        // internal name ".format".
+        if (!symtable_add_def(st, &_Py_STR(format), DEF_PARAM,
+                                LOCATION(annotation))) {
+            return 0;
+        }
+        if (!symtable_add_def(st, &_Py_STR(format), USE,
+                                LOCATION(annotation))) {
+            return 0;
+        }
+    }
+    else {
+        if (!symtable_enter_existing_block(st, parent_ste->ste_annotation_block)) {
+            VISIT_QUIT(st, 0);
+        }
     }
     VISIT(st, expr, annotation);
-    if (future_annotations && !symtable_exit_block(st)) {
+    if (!symtable_exit_block(st)) {
         VISIT_QUIT(st, 0);
     }
     return 1;
@@ -2471,37 +2542,58 @@ symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args)
 
     for (i = 0; i < asdl_seq_LEN(args); i++) {
         arg_ty arg = (arg_ty)asdl_seq_GET(args, i);
-        if (arg->annotation)
+        if (arg->annotation) {
+            st->st_cur->ste_annotations_used = 1;
             VISIT(st, expr, arg->annotation);
+        }
     }
 
     return 1;
 }
 
 static int
-symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns)
+symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns,
+                           struct _symtable_entry *function_ste)
 {
-    int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS;
+    int is_in_class = st->st_cur->ste_can_see_class_scope;
-    if (future_annotations &&
+    _Py_block_ty current_type = st->st_cur->ste_type;
-        !symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock,
+    if (!symtable_enter_block(st, &_Py_ID(__annotate__), AnnotationBlock,
-                              (void *)o, o->lineno, o->col_offset, o->end_lineno,
+                              (void *)a, LOCATION(o))) {
-                              o->end_col_offset)) {
         VISIT_QUIT(st, 0);
     }
+    if (is_in_class || current_type == ClassBlock) {
+        st->st_cur->ste_can_see_class_scope = 1;
+        if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(o))) {
+            return 0;
+        }
+    }
+    _Py_DECLARE_STR(format, ".format");
+    // We need to insert code that reads this "parameter" to the function.
+    if (!symtable_add_def(st, &_Py_STR(format), DEF_PARAM, LOCATION(o))) {
+        return 0;
+    }
+    if (!symtable_add_def(st, &_Py_STR(format), USE, LOCATION(o))) {
+        return 0;
+    }
     if (a->posonlyargs && !symtable_visit_argannotations(st, a->posonlyargs))
         return 0;
     if (a->args && !symtable_visit_argannotations(st, a->args))
         return 0;
-    if (a->vararg && a->vararg->annotation)
+    if (a->vararg && a->vararg->annotation) {
+        st->st_cur->ste_annotations_used = 1;
         VISIT(st, expr, a->vararg->annotation);
-    if (a->kwarg && a->kwarg->annotation)
+    }
+    if (a->kwarg && a->kwarg->annotation) {
+        st->st_cur->ste_annotations_used = 1;
         VISIT(st, expr, a->kwarg->annotation);
+    }
     if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs))
         return 0;
-    if (future_annotations && !symtable_exit_block(st)) {
+    if (returns) {
-        VISIT_QUIT(st, 0);
+        st->st_cur->ste_annotations_used = 1;
+        VISIT(st, expr, returns);
     }
-    if (returns && !symtable_visit_annotation(st, returns)) {
+    if (!symtable_exit_block(st)) {
         VISIT_QUIT(st, 0);
     }
     return 1;
@@ -2733,7 +2825,7 @@ symtable_visit_dictcomp(struct symtable *st, expr_ty e)
 static int
 symtable_raise_if_annotation_block(struct symtable *st, const char *name, expr_ty e)
 {
-    enum _block_type type = st->st_cur->ste_type;
+    _Py_block_ty type = st->st_cur->ste_type;
     if (type == AnnotationBlock)
         PyErr_Format(PyExc_SyntaxError, ANNOTATION_NOT_ALLOWED, name);
     else if (type == TypeVarBoundBlock)