diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 1786ac6b503..96b74140218 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -341,6 +341,58 @@ Importing Modules .. versionadded:: 3.14 +.. c:function:: PyImport_LazyImportsMode PyImport_GetLazyImportsMode() + + Gets the current lazy imports mode. + + .. versionadded:: next + +.. c:function:: PyObject* PyImport_GetLazyImportsFilter() + + Return a :term:`strong reference` to the current lazy imports filter, + or ``NULL`` if none exists. This function always succeeds. + + .. versionadded:: next + +.. c:function:: int PyImport_SetLazyImportsMode(PyImport_LazyImportsMode mode) + + Similar to :c:func:`PyImport_ImportModuleAttr`, but names are UTF-8 encoded + strings instead of Python :class:`str` objects. + + This function always returns ``0``. + + .. versionadded:: next + +.. c:function:: int PyImport_SetLazyImportsFilter(PyObject *filter) + + Sets the current lazy imports filter. The *filter* should be a callable that + will receive ``(importing_module_name, imported_module_name, [fromlist])`` + when an import can potentially be lazy and that must return ``True`` if + the import should be lazy and ``False`` otherwise. + + Return ``0`` on success and ``-1`` with an exception set otherwise. + + .. versionadded:: next + +.. c:type:: PyImport_LazyImportsMode + + Enumeration of possible lazy import modes. + + .. c:enumerator:: PyImport_LAZY_NORMAL + + Respect the ``lazy`` keyword in source code. This is the default mode. + + .. c:enumerator:: PyImport_LAZY_ALL + + Make all imports lazy by default. + + .. c:enumerator:: PyImport_LAZY_NONE + + Disable lazy imports entirely. Even explicit ``lazy`` statements become + eager imports. + + .. versionadded:: next + .. c:function:: PyObject* PyImport_CreateModuleFromInitfunc(PyObject *spec, PyObject* (*initfunc)(void)) This function is a building block that enables embedders to implement diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 2e7d0dbc26e..f075b8d0fe4 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -1113,7 +1113,8 @@ Imports names=[ alias(name='x'), alias(name='y'), - alias(name='z')])]) + alias(name='z')], + is_lazy=0)]) .. class:: ImportFrom(module, names, level) @@ -1134,7 +1135,8 @@ Imports alias(name='x'), alias(name='y'), alias(name='z')], - level=0)]) + level=0, + is_lazy=0)]) .. class:: alias(name, asname) @@ -1152,7 +1154,8 @@ Imports names=[ alias(name='a', asname='b'), alias(name='c')], - level=2)]) + level=2, + is_lazy=0)]) Control flow ^^^^^^^^^^^^ diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index a0621d4b0db..f8e99bc7d24 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -911,6 +911,43 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. versionadded:: 3.11 + +.. function:: get_lazy_imports() + + Returns the current lazy imports mode as a string. + + * ``"normal"``: Only imports explicitly marked with the ``lazy`` keyword are lazy + * ``"all"``: All top-level imports are potentially lazy + * ``"none"``: All lazy imports are suppressed (even explicitly marked ones) + + See also :func:`set_lazy_imports` and :pep:`810`. + + .. versionadded:: 3.15 + + +.. function:: get_lazy_imports_filter() + + Returns the current lazy imports filter function, or ``None`` if no filter + is set. + + The filter function is called for every potentially lazy import to determine + whether it should actually be lazy. See :func:`set_lazy_imports_filter` for + details on the filter function signature. + + .. versionadded:: 3.15 + + +.. function:: get_lazy_modules() + + Returns a set of fully-qualified module names that have been lazily imported. + This is primarily useful for diagnostics and introspection. + + Note that modules are removed from this set when they are reified (actually + loaded on first use). + + .. versionadded:: 3.15 + + .. function:: getrefcount(object) Return the reference count of the *object*. The count returned is generally one @@ -1719,6 +1756,57 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. versionadded:: 3.11 + +.. function:: set_lazy_imports(mode) + + Sets the global lazy imports mode. The *mode* parameter must be one of the + following strings: + + * ``"normal"``: Only imports explicitly marked with the ``lazy`` keyword are lazy + * ``"all"``: All top-level imports become potentially lazy + * ``"none"``: All lazy imports are suppressed (even explicitly marked ones) + + This function is intended for advanced users who need to control lazy imports + across their entire application. Library developers should generally not use + this function as it affects the runtime execution of applications. + + In addition to the mode, lazy imports can be controlled via the filter + provided by :func:`set_lazy_imports_filter`. + + See also :func:`get_lazy_imports` and :pep:`810`. + + .. versionadded:: 3.15 + + +.. function:: set_lazy_imports_filter(filter) + + Sets the lazy imports filter callback. The *filter* parameter must be a + callable or ``None`` to clear the filter. + + The filter function is called for every potentially lazy import to determine + whether it should actually be lazy. It must have the following signature:: + + def filter(importing_module: str, imported_module: str, + fromlist: tuple[str, ...] | None) -> bool + + Where: + + * *importing_module* is the name of the module doing the import + * *imported_module* is the name of the module being imported + * *fromlist* is the tuple of names being imported (for ``from ... import`` + statements), or ``None`` for regular imports + + The filter should return ``True`` to allow the import to be lazy, or + ``False`` to force an eager import. + + This is an advanced feature intended for specialized users who need + fine-grained control over lazy import behavior. + + See also :func:`get_lazy_imports_filter` and :pep:`810`. + + .. versionadded:: 3.15 + + .. function:: setprofile(profilefunc) .. index:: diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 40b5f3db13d..01f4df3c890 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -343,6 +343,18 @@ Standard names are defined for the following types: .. seealso:: :pep:`667` +.. data:: LazyImportType + + The type of lazy import proxy objects. These objects are created when a + module is lazily imported and serve as placeholders until the module is + actually accessed. This type can be used to detect lazy imports + programmatically. + + .. versionadded:: next + + .. seealso:: :pep:`810` + + .. data:: GetSetDescriptorType The type of objects defined in extension modules with ``PyGetSetDef``, such diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 9322d8571f7..3665088a933 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -457,6 +457,7 @@ Some names are only reserved under specific contexts. These are known as - ``match``, ``case``, and ``_``, when used in the :keyword:`match` statement. - ``type``, when used in the :keyword:`type` statement. +- ``lazy``, when used before an :keyword:`import` statement. These syntactically act as keywords in their specific contexts, but this distinction is done at the parser level, not when tokenizing. @@ -468,6 +469,9 @@ identifier names. .. versionchanged:: 3.12 ``type`` is now a soft keyword. +.. versionchanged:: next + ``lazy`` is now a soft keyword. + .. index:: single: _, identifiers single: __, identifiers diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 9c022570e7e..671dea181e8 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -748,14 +748,15 @@ The :keyword:`!import` statement pair: name; binding pair: keyword; from pair: keyword; as + pair: keyword; lazy pair: exception; ImportError single: , (comma); import statement .. productionlist:: python-grammar - import_stmt: "import" `module` ["as" `identifier`] ("," `module` ["as" `identifier`])* - : | "from" `relative_module` "import" `identifier` ["as" `identifier`] + import_stmt: ["lazy"] "import" `module` ["as" `identifier`] ("," `module` ["as" `identifier`])* + : | ["lazy"] "from" `relative_module` "import" `identifier` ["as" `identifier`] : ("," `identifier` ["as" `identifier`])* - : | "from" `relative_module` "import" "(" `identifier` ["as" `identifier`] + : | ["lazy"] "from" `relative_module` "import" "(" `identifier` ["as" `identifier`] : ("," `identifier` ["as" `identifier`])* [","] ")" : | "from" `relative_module` "import" "*" module: (`identifier` ".")* `identifier` @@ -870,6 +871,56 @@ determine dynamically the modules to be loaded. .. audit-event:: import module,filename,sys.path,sys.meta_path,sys.path_hooks import + +.. _lazy-imports: +.. _lazy: + +Lazy imports +------------ + +.. index:: + pair: lazy; import + single: lazy import + +The :keyword:`lazy` keyword marks an import as lazy. It is a :ref:`soft keyword +` that only has special meaning when it appears immediately +before an :keyword:`import` or :keyword:`from` statement. + +When an import statement is preceded by the :keyword:`lazy` keyword, +the import becomes *lazy*: the module is not loaded immediately at the import +statement. Instead, a lazy proxy object is created and bound to the name. The +actual module is loaded on first use of that name. + +Lazy imports are only permitted at module scope. Using ``lazy`` inside a +function, class body, or :keyword:`try`/:keyword:`except`/:keyword:`finally` +block raises a :exc:`SyntaxError`. Star imports cannot be lazy (``lazy from +module import *`` is a syntax error), and :ref:`future statements ` +cannot be lazy. + +When using ``lazy from ... import``, each imported name is bound to a lazy +proxy object. The first access to any of these names triggers loading of the +entire module and resolves only that specific name to its actual value. Other +names remain as lazy proxies until they are accessed. + +Example:: + + lazy import json + + print('json' in sys.modules) # False - module not loaded yet + + # First use triggers loading + result = json.dumps({"hello": "world"}) + + print('json' in sys.modules) # True - now loaded + +If an error occurs during module loading (such as :exc:`ImportError` or +:exc:`SyntaxError`), it is raised at the point where the lazy import is first +used, not at the import statement itself. + +See :pep:`810` for the full specification of lazy imports. + +.. versionadded:: next + .. _future: Future statements diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index aff165191b7..837aa5fee61 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -694,6 +694,14 @@ Miscellaneous options .. versionadded:: 3.14 + * :samp:`-X lazy_imports={all,none,normal}` controls lazy import behavior. + ``all`` makes all imports lazy by default, ``none`` disables lazy imports + entirely (even explicit ``lazy`` statements become eager), and ``normal`` + (the default) respects the ``lazy`` keyword in source code. + See also :envvar:`PYTHON_LAZY_IMPORTS`. + + .. versionadded:: next + It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -1339,6 +1347,17 @@ conflict. .. versionadded:: 3.14 +.. envvar:: PYTHON_LAZY_IMPORTS + + Controls lazy import behavior. Accepts three values: ``all`` makes all + imports lazy by default, ``none`` disables lazy imports entirely (even + explicit ``lazy`` statements become eager), and ``normal`` (the default) + respects the ``lazy`` keyword in source code. + + See also the :option:`-X lazy_imports <-X>` command-line option. + + .. versionadded:: next + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 1bd82545e58..b231bd3eac9 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -65,6 +65,8 @@ Summary -- Release highlights .. PEP-sized items next. +* :pep:`810`: :ref:`Explicit lazy imports for faster startup times + ` * :pep:`799`: :ref:`A dedicated profiling package for organizing Python profiling tools ` * :pep:`686`: :ref:`Python now uses UTF-8 as the default encoding @@ -77,6 +79,98 @@ Summary -- Release highlights New features ============ +.. _whatsnew315-pep810: + +:pep:`810`: Explicit lazy imports +--------------------------------- + +Large Python applications often suffer from slow startup times. A significant +contributor to this problem is the import system: when a module is imported, +Python must locate the file, read it from disk, compile it to bytecode, and +execute all top-level code. For applications with deep dependency trees, this +process can take seconds, even when most of the imported code is never actually +used during a particular run. + +Developers have worked around this by moving imports inside functions, using +:mod:`importlib` to load modules on demand, or restructuring code to avoid +unnecessary dependencies. These approaches work but make code harder to read +and maintain, scatter import statements throughout the codebase, and require +discipline to apply consistently. + +Python now provides a cleaner solution through explicit lazy imports using the +new ``lazy`` soft keyword. When you mark an import as lazy, Python defers the +actual module loading until the imported name is first used. This gives you +the organizational benefits of declaring all imports at the top of the file +while only paying the loading cost for modules you actually use. + +The ``lazy`` keyword works with both ``import`` and ``from ... import`` statements. +When you write ``lazy import heavy_module``, Python does not immediately load the +module. Instead, it creates a lightweight proxy object. The actual module loading +happens transparently when you first access the name: + +.. code-block:: python + + lazy import json + lazy from datetime import datetime + + print("Starting up...") # json and datetime not loaded yet + + data = json.loads('{"key": "value"}') # json loads here + now = datetime() # datetime loads here + +This mechanism is particularly useful for applications that import many modules +at the top level but may only use a subset of them in any given run. The deferred +loading reduces startup latency without requiring code restructuring or conditional +imports scattered throughout the codebase. + +When a lazy import eventually fails (for example, if the module does not exist), +Python raises the exception at the point of first use rather than at import time. +The traceback includes both the location where the name was accessed and the +original import statement, making it straightforward to diagnose the problem. + +For cases where you want to enable lazy loading globally without modifying source +code, Python provides the :option:`-X lazy_imports <-X>` command-line option and +the :envvar:`PYTHON_LAZY_IMPORTS` environment variable. Both accept three values: +``all`` makes all imports lazy by default, ``none`` disables lazy imports entirely +(even explicit ``lazy`` statements become eager), and ``normal`` (the default) +respects the ``lazy`` keyword in source code. The :func:`sys.set_lazy_imports` and +:func:`sys.get_lazy_imports` functions allow changing and querying this mode at +runtime. + +For more selective control, :func:`sys.set_lazy_imports_filter` accepts a callable +that determines whether a specific module should be loaded lazily. The filter +receives three arguments: the importing module's name (or ``None``), the imported +module's name, and the fromlist (or ``None`` for regular imports). It should +return ``True`` to allow the import to be lazy, or ``False`` to force eager loading. +This allows patterns like making only your own application's modules lazy while +keeping third-party dependencies eager: + +.. code-block:: python + + import sys + + sys.set_lazy_imports_filter(lambda importing, imported, fromlist: imported.startswith("myapp.")) + sys.set_lazy_imports("all") + + import myapp.slow_module # lazy (matches filter) + import json # eager (does not match filter) + +For debugging and introspection, :func:`sys.get_lazy_modules` returns a set +containing the names of all modules that have been lazily imported but not yet +loaded. The proxy type itself is available as :data:`types.LazyImportType` for +code that needs to detect lazy imports programmatically. + +There are some restrictions on where ``lazy`` can appear. Lazy imports are only +permitted at module scope; using ``lazy`` inside a function, class body, or +``try``/``except``/``finally`` block raises a :exc:`SyntaxError`. Star imports +cannot be lazy (``lazy from module import *`` is a syntax error), and future +imports cannot be lazy either (``lazy from __future__ import ...`` raises +:exc:`SyntaxError`). + +.. seealso:: :pep:`810` for the full specification and rationale. + +(Contributed by Pablo Galindo Salgado and Dino Viehland in :gh:`142349`.) + .. _whatsnew315-profiling-package: :pep:`799`: A dedicated profiling package diff --git a/Grammar/python.gram b/Grammar/python.gram index 7ae00c6f005..5d1b68059f5 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -121,9 +121,9 @@ simple_stmts[asdl_stmt_seq*]: simple_stmt[stmt_ty] (memo): | assignment | &"type" type_alias + | &('import' | 'from' | "lazy") import_stmt | e=star_expressions { _PyAST_Expr(e, EXTRA) } | &'return' return_stmt - | &('import' | 'from') import_stmt | &'raise' raise_stmt | &'pass' pass_stmt | &'del' del_stmt @@ -216,7 +216,7 @@ assert_stmt[stmt_ty]: | invalid_assert_stmt | 'assert' a=expression b=[',' z=expression { z }] { _PyAST_Assert(a, b, EXTRA) } -import_stmt[stmt_ty]: +import_stmt[stmt_ty](memo): | invalid_import | import_name | import_from @@ -224,13 +224,15 @@ import_stmt[stmt_ty]: # Import statements # ----------------- -import_name[stmt_ty]: 'import' a=dotted_as_names { _PyAST_Import(a, EXTRA) } +import_name[stmt_ty]: + | lazy="lazy"? 'import' a=dotted_as_names { _PyAST_Import(a, lazy ? 1 : 0, EXTRA) } + # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS import_from[stmt_ty]: - | 'from' a=('.' | '...')* b=dotted_name 'import' c=import_from_targets { - _PyPegen_checked_future_import(p, b->v.Name.id, c, _PyPegen_seq_count_dots(a), EXTRA) } - | 'from' a=('.' | '...')+ 'import' b=import_from_targets { - _PyAST_ImportFrom(NULL, b, _PyPegen_seq_count_dots(a), EXTRA) } + | lazy="lazy"? 'from' a=('.' | '...')* b=dotted_name 'import' c=import_from_targets { + _PyPegen_checked_future_import(p, b->v.Name.id, c, _PyPegen_seq_count_dots(a), lazy, EXTRA) } + | lazy="lazy"? 'from' a=('.' | '...')+ 'import' b=import_from_targets { + _PyAST_ImportFrom(NULL, b, _PyPegen_seq_count_dots(a), lazy ? 1 : 0, EXTRA) } import_from_targets[asdl_alias_seq*]: | '(' a=import_from_as_names [','] ')' { a } | import_from_as_names !',' diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 1c979d91a40..045807ca1e8 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -190,6 +190,7 @@ typedef struct PyConfig { int enable_gil; int tlbc_enabled; #endif + int lazy_imports; /* --- Path configuration inputs ------------ */ int pathconfig_warnings; diff --git a/Include/import.h b/Include/import.h index d91ebe96ca8..cc7ad71f267 100644 --- a/Include/import.h +++ b/Include/import.h @@ -88,6 +88,20 @@ PyAPI_FUNC(int) PyImport_AppendInittab( PyObject* (*initfunc)(void) ); +typedef enum { + PyImport_LAZY_NORMAL, + PyImport_LAZY_ALL, + PyImport_LAZY_NONE, +} PyImport_LazyImportsMode; + +#ifndef Py_LIMITED_API +PyAPI_FUNC(int) PyImport_SetLazyImportsMode(PyImport_LazyImportsMode mode); +PyAPI_FUNC(int) PyImport_SetLazyImportsFilter(PyObject *filter); + +PyAPI_FUNC(PyImport_LazyImportsMode) PyImport_GetLazyImportsMode(void); +PyAPI_FUNC(PyObject *) PyImport_GetLazyImportsFilter(void); +#endif + #ifndef Py_LIMITED_API # define Py_CPYTHON_IMPORT_H # include "cpython/import.h" diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index 60367202bab..b47398669bb 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -329,12 +329,14 @@ struct _stmt { struct { asdl_alias_seq *names; + int is_lazy; } Import; struct { identifier module; asdl_alias_seq *names; int level; + int is_lazy; } ImportFrom; struct { @@ -764,11 +766,12 @@ stmt_ty _PyAST_TryStar(asdl_stmt_seq * body, asdl_excepthandler_seq * handlers, end_col_offset, PyArena *arena); stmt_ty _PyAST_Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); -stmt_ty _PyAST_Import(asdl_alias_seq * names, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); +stmt_ty _PyAST_Import(asdl_alias_seq * names, int is_lazy, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); stmt_ty _PyAST_ImportFrom(identifier module, asdl_alias_seq * names, int level, - int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); + int is_lazy, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); stmt_ty _PyAST_Global(asdl_identifier_seq * names, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); stmt_ty _PyAST_Nonlocal(asdl_identifier_seq * names, int lineno, int diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index d4ac419f51d..1caf200ee34 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -205,6 +205,7 @@ struct ast_state { PyObject *id; PyObject *ifs; PyObject *is_async; + PyObject *is_lazy; PyObject *items; PyObject *iter; PyObject *key; diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 6bf33bddd5b..b4bd93a8bcb 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -311,7 +311,14 @@ PyAPI_FUNC(void) _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, PyAPI_FUNC(void) _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg); PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs); PyAPI_FUNC(PyObject *) _PyEval_ImportFrom(PyThreadState *, PyObject *, PyObject *); -PyAPI_FUNC(PyObject *) _PyEval_ImportName(PyThreadState *, _PyInterpreterFrame *, PyObject *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyEval_LazyImportName(PyThreadState *tstate, PyObject *builtins, PyObject *globals, + PyObject *locals, PyObject *name, PyObject *fromlist, PyObject *level, int lazy); +PyAPI_FUNC(PyObject *) _PyEval_LazyImportFrom(PyThreadState *tstate, PyObject *v, PyObject *name); +PyAPI_FUNC(PyObject *) _PyEval_ImportName(PyThreadState *tstate, PyObject *builtins, PyObject *globals, PyObject *locals, + PyObject *name, PyObject *fromlist, PyObject *level); +PyObject * +_PyEval_ImportNameWithImport(PyThreadState *tstate, PyObject *import_func, PyObject *globals, PyObject *locals, + PyObject *name, PyObject *fromlist, PyObject *level); PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); PyAPI_FUNC(void) _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 527141b54d0..911cc1f10f1 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -131,6 +131,7 @@ int _PyCompile_PushFBlock(struct _PyCompiler *c, _Py_SourceLocation loc, void _PyCompile_PopFBlock(struct _PyCompiler *c, enum _PyCompile_FBlockType t, _PyJumpTargetLabel block_label); _PyCompile_FBlockInfo *_PyCompile_TopFBlock(struct _PyCompiler *c); +bool _PyCompile_InExceptionHandler(struct _PyCompiler *c); int _PyCompile_EnterScope(struct _PyCompiler *c, identifier name, int scope_type, void *key, int lineno, PyObject *private, diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 1193f496da1..f7bceb4605e 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -36,6 +36,15 @@ extern int _PyDict_DelItem_KnownHash_LockHeld(PyObject *mp, PyObject *key, extern int _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); +// "Id" variants +extern PyObject* _PyDict_GetItemIdWithError(PyObject *dp, + _Py_Identifier *key); +extern int _PyDict_ContainsId(PyObject *, _Py_Identifier *); +extern int _PyDict_SetItemId(PyObject *dp, _Py_Identifier *key, PyObject *item); +extern int _PyDict_DelItemId(PyObject *mp, _Py_Identifier *key); +extern void _PyDict_ClearKeysVersion(PyObject *mp); + + extern int _PyDict_Next( PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value, Py_hash_t *hash); diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index d23d6d4f91b..7e77a21ecb8 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1434,6 +1434,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__iter__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__itruediv__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__ixor__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__lazy_import__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__lazy_modules__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__le__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__len__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__length_hint__)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 5c3ea474ad0..748502ab740 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -157,6 +157,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(__iter__) STRUCT_FOR_ID(__itruediv__) STRUCT_FOR_ID(__ixor__) + STRUCT_FOR_ID(__lazy_import__) + STRUCT_FOR_ID(__lazy_modules__) STRUCT_FOR_ID(__le__) STRUCT_FOR_ID(__len__) STRUCT_FOR_ID(__length_hint__) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 4c8b8c0ed86..b1948df4fbf 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -32,6 +32,19 @@ extern int _PyImport_FixupBuiltin( PyObject *modules ); +extern PyObject * +_PyImport_ResolveName(PyThreadState *tstate, PyObject *name, PyObject *globals, int level); +extern PyObject * +_PyImport_GetAbsName(PyThreadState *tstate, PyObject *name, PyObject *globals, int level); +// Symbol is exported for the JIT on Windows builds. +PyAPI_FUNC(PyObject *) +_PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import); +extern PyObject * +_PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, PyObject *name, PyObject *builtins, PyObject *globals, + PyObject *locals, PyObject *fromlist, + int level); + + #ifdef HAVE_DLOPEN # include // RTLD_NOW, RTLD_LAZY # if HAVE_DECL_RTLD_NOW @@ -74,6 +87,10 @@ extern int _PyImport_IsDefaultImportFunc( PyInterpreterState *interp, PyObject *func); +extern int _PyImport_IsDefaultLazyImportFunc( + PyInterpreterState *interp, + PyObject *func); + extern PyObject * _PyImport_GetImportlibLoader( PyInterpreterState *interp, const char *loader_name); diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 6b3d5711b92..9507d6e5974 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -322,6 +322,12 @@ struct _import_state { int dlopenflags; #endif PyObject *import_func; + PyObject *lazy_import_func; + int lazy_imports_mode; + PyObject *lazy_imports_filter; + PyObject *lazy_importing_modules; + PyObject *lazy_modules; + PyObject *lazy_modules_set; /* Set of fully-qualified module names lazily imported (PEP 810) */ /* The global import lock. */ _PyRecursiveMutex lock; /* diagnostic info in PyImport_ImportModuleLevelObject() */ diff --git a/Include/internal/pycore_lazyimportobject.h b/Include/internal/pycore_lazyimportobject.h new file mode 100644 index 00000000000..513daeffd91 --- /dev/null +++ b/Include/internal/pycore_lazyimportobject.h @@ -0,0 +1,36 @@ +/* File added for Lazy Imports */ + +/* Lazy object interface */ + +#ifndef Py_INTERNAL_LAZYIMPORTOBJECT_H +#define Py_INTERNAL_LAZYIMPORTOBJECT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +PyAPI_DATA(PyTypeObject) PyLazyImport_Type; +#define PyLazyImport_CheckExact(op) Py_IS_TYPE((op), &PyLazyImport_Type) + +typedef struct { + PyObject_HEAD + PyObject *lz_builtins; + PyObject *lz_from; + PyObject *lz_attr; + /* Frame information for the original import location */ + PyCodeObject *lz_code; /* code object where the lazy import was created */ + int lz_instr_offset; /* instruction offset where the lazy import was created */ +} PyLazyImportObject; + + +PyAPI_FUNC(PyObject *) _PyLazyImport_GetName(PyObject *lazy_import); +PyAPI_FUNC(PyObject *) _PyLazyImport_New(PyObject *import_func, PyObject *from, PyObject *attr); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_LAZYIMPORTOBJECT_H */ diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 2fb46a6df50..db5fb08de75 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -287,6 +287,7 @@ Known values: Python 3.15a1 3654 (Fix missing exception handlers in logical expression) Python 3.15a1 3655 (Fix miscompilation of some module-level annotations) Python 3.15a1 3656 (Add TRACE_RECORD instruction, for platforms with switch based interpreter) + Python 3.15a3 3657 (Lazy imports IMPORT_NAME opcode changes) Python 3.16 will start with 3700 @@ -300,7 +301,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3656 +#define PYC_MAGIC_NUMBER 3657 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index 9a62daf6621..15a68523f47 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -28,6 +28,8 @@ typedef struct { PyObject *md_weaklist; // for logging purposes after md_dict is cleared PyObject *md_name; + // module version we last checked for lazy values + uint32_t m_dict_version; bool md_token_is_def; /* if true, `md_token` is the PyModuleDef */ #ifdef Py_GIL_DISABLED bool md_requires_gil; @@ -76,6 +78,8 @@ extern Py_ssize_t _PyModule_GetFilenameUTF8( PyObject* _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress); PyObject* _Py_module_getattro(PyObject *m, PyObject *name); +PyAPI_FUNC(int) _PyModule_ReplaceLazyValue(PyObject *dict, PyObject *name, PyObject *value); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 31d88339a13..b10d6de2963 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1432,6 +1432,8 @@ extern "C" { INIT_ID(__iter__), \ INIT_ID(__itruediv__), \ INIT_ID(__ixor__), \ + INIT_ID(__lazy_import__), \ + INIT_ID(__lazy_modules__), \ INIT_ID(__le__), \ INIT_ID(__len__), \ INIT_ID(__length_hint__), \ diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 9dbfa913219..ac40e5d6eee 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -126,6 +126,7 @@ typedef struct _symtable_entry { unsigned ste_method : 1; /* true if block is a function block defined in class scope */ unsigned ste_has_conditional_annotations : 1; /* true if block has conditionally executed annotations */ unsigned ste_in_conditional_block : 1; /* set while we are inside a conditionally executed block */ + unsigned ste_in_try_block : 1; /* set while we are inside a try/except block */ unsigned ste_in_unevaluated_annotation : 1; /* set while we are processing an annotation that will not be evaluated */ int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */ _Py_SourceLocation ste_loc; /* source location of block */ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index c5b01ff9876..f61ad458e8c 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -408,6 +408,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(__lazy_import__); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(__lazy_modules__); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__le__); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Include/pyerrors.h b/Include/pyerrors.h index 5d0028c116e..cfabbc5fe8d 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -91,6 +91,9 @@ PyAPI_DATA(PyObject *) PyExc_EOFError; PyAPI_DATA(PyObject *) PyExc_FloatingPointError; PyAPI_DATA(PyObject *) PyExc_OSError; PyAPI_DATA(PyObject *) PyExc_ImportError; +#if !defined(Py_LIMITED_API) +PyAPI_DATA(PyObject *) PyExc_ImportCycleError; +#endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03060000 PyAPI_DATA(PyObject *) PyExc_ModuleNotFoundError; #endif diff --git a/Lib/_compat_pickle.py b/Lib/_compat_pickle.py index a9813264324..928db663b44 100644 --- a/Lib/_compat_pickle.py +++ b/Lib/_compat_pickle.py @@ -240,6 +240,7 @@ REVERSE_NAME_MAPPING[('builtins', excname)] = ('exceptions', 'OSError') PYTHON3_IMPORTERROR_EXCEPTIONS = ( + 'ImportCycleError', 'ModuleNotFoundError', ) diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index 06cddef851b..98b178f1dd2 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -276,6 +276,8 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool: TI(T.NAME, string=s) ): return not keyword.iskeyword(s) + case (None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT), TI(string="lazy"), TI(string="import") | TI(string="from")): + return True case _: return False diff --git a/Lib/dis.py b/Lib/dis.py index d6d2c1386dd..e8f024c6de1 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -35,6 +35,7 @@ FUNCTION_ATTR_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure', 'annotate') ENTER_EXECUTOR = opmap['ENTER_EXECUTOR'] +IMPORT_NAME = opmap['IMPORT_NAME'] LOAD_GLOBAL = opmap['LOAD_GLOBAL'] LOAD_SMALL_INT = opmap['LOAD_SMALL_INT'] BINARY_OP = opmap['BINARY_OP'] @@ -601,6 +602,12 @@ def get_argval_argrepr(self, op, arg, offset): argval, argrepr = _get_name_info(arg//4, get_name) if (arg & 1) and argrepr: argrepr = f"{argrepr} + NULL|self" + elif deop == IMPORT_NAME: + argval, argrepr = _get_name_info(arg//4, get_name) + if (arg & 1) and argrepr: + argrepr = f"{argrepr} + lazy" + elif (arg & 2) and argrepr: + argrepr = f"{argrepr} + eager" else: argval, argrepr = _get_name_info(arg, get_name) elif deop in hasjump or deop in hasexc: @@ -1013,7 +1020,8 @@ def _find_imports(co): (level_op[0] in hasconst or level_op[0] == LOAD_SMALL_INT)): level = _get_const_value(level_op[0], level_op[1], consts) fromlist = _get_const_value(from_op[0], from_op[1], consts) - yield (names[oparg], level, fromlist) + # IMPORT_NAME encodes lazy/eager flags in bits 0-1, name index in bits 2+ + yield (names[oparg >> 2], level, fromlist) def _find_store_names(co): """Find names of variables which are written in the code diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index bffa2ddd3cd..6db38de3aa6 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -42,6 +42,11 @@ def make_pat(): ]) + r"))" ) + lazy_softkw = ( + r"^[ \t]*" + # at beginning of line + possible indentation + r"(?Plazy)" + + r"(?=[ \t]+(?:import|from)\b)" # followed by 'import' or 'from' + ) builtinlist = [str(name) for name in dir(builtins) if not name.startswith('_') and name not in keyword.kwlist] @@ -56,7 +61,7 @@ def make_pat(): prog = re.compile("|".join([ builtin, comment, string, kw, match_softkw, case_default, - case_softkw_and_pattern, + case_softkw_and_pattern, lazy_softkw, any("SYNC", [r"\n"]), ]), re.DOTALL | re.MULTILINE) @@ -70,6 +75,7 @@ def make_pat(): "CASE_SOFTKW": "KEYWORD", "CASE_DEFAULT_UNDERSCORE": "KEYWORD", "CASE_SOFTKW2": "KEYWORD", + "LAZY_SOFTKW": "KEYWORD", } diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index 40800df97b0..99cfc479759 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -542,6 +542,22 @@ def test_case_soft_keyword(self): self._assert_highlighting('case _:', {'KEYWORD': [('1.0', '1.4'), ('1.5', '1.6')]}) + def test_lazy_soft_keyword(self): + # lazy followed by import + self._assert_highlighting('lazy import foo', + {'KEYWORD': [('1.0', '1.4'), ('1.5', '1.11')]}) + self._assert_highlighting(' lazy import foo', + {'KEYWORD': [('1.4', '1.8'), ('1.9', '1.15')]}) + + # lazy followed by from + self._assert_highlighting('lazy from foo import bar', + {'KEYWORD': [('1.0', '1.4'), ('1.5', '1.9'), + ('1.14', '1.20')]}) + + # lazy not followed by import/from (not highlighted) + self._assert_highlighting('lazy = 1', {}) + self._assert_highlighting('lazy foo', {}) + def test_long_multiline_string(self): source = textwrap.dedent('''\ """a diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 43c66765dd9..bd0638016a1 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -1359,6 +1359,12 @@ def _find_and_load_unlocked(name, import_): except AttributeError: msg = f"Cannot set an attribute on {parent!r} for child module {child!r}" _warnings.warn(msg, ImportWarning) + # Set attributes to lazy submodules on the module. + try: + _imp._set_lazy_attributes(module, name) + except Exception as e: + msg = f"Cannot set lazy attributes on {name!r}: {e!r}" + _warnings.warn(msg, ImportWarning) return module diff --git a/Lib/keyword.py b/Lib/keyword.py index e22c837835e..98ffe2de28b 100644 --- a/Lib/keyword.py +++ b/Lib/keyword.py @@ -56,6 +56,7 @@ kwlist = [ softkwlist = [ '_', 'case', + 'lazy', 'match', 'type' ] diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index 23eb0020f42..230ce65092c 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -36,6 +36,7 @@ import re import __main__ import warnings +import types __all__ = ["Completer"] @@ -188,7 +189,16 @@ def attr_matches(self, text): # property method, which is not desirable. matches.append(match) continue - if (value := getattr(thisobject, word, None)) is not None: + + if (isinstance(thisobject, types.ModuleType) + and + isinstance(thisobject.__dict__.get(word), types.LazyImportType) + ): + value = thisobject.__dict__.get(word) + else: + value = getattr(thisobject, word, None) + + if value is not None: matches.append(self._callable_postfix(value, match)) else: matches.append(match) diff --git a/Lib/test/.ruff.toml b/Lib/test/.ruff.toml index a1b749798fa..a537d36ee70 100644 --- a/Lib/test/.ruff.toml +++ b/Lib/test/.ruff.toml @@ -13,6 +13,9 @@ extend-exclude = [ # New grammar constructions may not yet be recognized by Ruff, # and tests re-use the same names as only the grammar is being checked. "test_grammar.py", + # Lazy import syntax (PEP 810) not yet supported by Ruff + "test_import/data/lazy_imports/*.py", + "test_import/data/lazy_imports/**/*.py", ] [per-file-target-version] diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt index f2649aa2d41..98a5e950602 100644 --- a/Lib/test/exception_hierarchy.txt +++ b/Lib/test/exception_hierarchy.txt @@ -14,6 +14,7 @@ BaseException ├── EOFError ├── ExceptionGroup [BaseExceptionGroup] ├── ImportError + │ └── ImportCycleError │ └── ModuleNotFoundError ├── LookupError │ ├── IndexError diff --git a/Lib/test/test_ast/data/ast_repr.txt b/Lib/test/test_ast/data/ast_repr.txt index 1c1985519cd..cc6accd766b 100644 --- a/Lib/test/test_ast/data/ast_repr.txt +++ b/Lib/test/test_ast/data/ast_repr.txt @@ -69,10 +69,14 @@ Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='ex Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[]) Module(body=[Assert(test=Name(id='v', ctx=Load(...)), msg=None)], type_ignores=[]) Module(body=[Assert(test=Name(id='v', ctx=Load(...)), msg=Constant(value='message', kind=None))], type_ignores=[]) -Module(body=[Import(names=[alias(name='sys', asname=None)])], type_ignores=[]) -Module(body=[Import(names=[alias(name='foo', asname='bar')])], type_ignores=[]) -Module(body=[ImportFrom(module='sys', names=[alias(name='x', asname='y')], level=0)], type_ignores=[]) -Module(body=[ImportFrom(module='sys', names=[alias(name='v', asname=None)], level=0)], type_ignores=[]) +Module(body=[Import(names=[alias(name='sys', asname=None)], is_lazy=0)], type_ignores=[]) +Module(body=[Import(names=[alias(name='foo', asname='bar')], is_lazy=0)], type_ignores=[]) +Module(body=[ImportFrom(module='sys', names=[alias(name='x', asname='y')], level=0, is_lazy=0)], type_ignores=[]) +Module(body=[ImportFrom(module='sys', names=[alias(name='v', asname=None)], level=0, is_lazy=0)], type_ignores=[]) +Module(body=[Import(names=[alias(name='sys', asname=None)], is_lazy=1)], type_ignores=[]) +Module(body=[Import(names=[alias(name='foo', asname='bar')], is_lazy=1)], type_ignores=[]) +Module(body=[ImportFrom(module='sys', names=[alias(name='x', asname='y')], level=0, is_lazy=1)], type_ignores=[]) +Module(body=[ImportFrom(module='sys', names=[alias(name='v', asname=None)], level=0, is_lazy=1)], type_ignores=[]) Module(body=[Global(names=['v'])], type_ignores=[]) Module(body=[Expr(value=Constant(value=1, kind=None))], type_ignores=[]) Module(body=[Pass()], type_ignores=[]) diff --git a/Lib/test/test_ast/snippets.py b/Lib/test/test_ast/snippets.py index b76f98901d2..a565ed10f8b 100644 --- a/Lib/test/test_ast/snippets.py +++ b/Lib/test/test_ast/snippets.py @@ -118,6 +118,12 @@ # ImportFrom "from sys import x as y", "from sys import v", + # Lazy Import + "lazy import sys", + "lazy import foo as bar", + # Lazy ImportFrom + "lazy from sys import x as y", + "lazy from sys import v", # Global "global v", # Expr @@ -460,10 +466,14 @@ def main(): ('Module', [('TryStar', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), ('Module', [('Assert', (1, 0, 1, 8), ('Name', (1, 7, 1, 8), 'v', ('Load',)), None)], []), ('Module', [('Assert', (1, 0, 1, 19), ('Name', (1, 7, 1, 8), 'v', ('Load',)), ('Constant', (1, 10, 1, 19), 'message', None))], []), -('Module', [('Import', (1, 0, 1, 10), [('alias', (1, 7, 1, 10), 'sys', None)])], []), -('Module', [('Import', (1, 0, 1, 17), [('alias', (1, 7, 1, 17), 'foo', 'bar')])], []), -('Module', [('ImportFrom', (1, 0, 1, 22), 'sys', [('alias', (1, 16, 1, 22), 'x', 'y')], 0)], []), -('Module', [('ImportFrom', (1, 0, 1, 17), 'sys', [('alias', (1, 16, 1, 17), 'v', None)], 0)], []), +('Module', [('Import', (1, 0, 1, 10), [('alias', (1, 7, 1, 10), 'sys', None)], 0)], []), +('Module', [('Import', (1, 0, 1, 17), [('alias', (1, 7, 1, 17), 'foo', 'bar')], 0)], []), +('Module', [('ImportFrom', (1, 0, 1, 22), 'sys', [('alias', (1, 16, 1, 22), 'x', 'y')], 0, 0)], []), +('Module', [('ImportFrom', (1, 0, 1, 17), 'sys', [('alias', (1, 16, 1, 17), 'v', None)], 0, 0)], []), +('Module', [('Import', (1, 0, 1, 15), [('alias', (1, 12, 1, 15), 'sys', None)], 1)], []), +('Module', [('Import', (1, 0, 1, 22), [('alias', (1, 12, 1, 22), 'foo', 'bar')], 1)], []), +('Module', [('ImportFrom', (1, 0, 1, 27), 'sys', [('alias', (1, 21, 1, 27), 'x', 'y')], 0, 1)], []), +('Module', [('ImportFrom', (1, 0, 1, 22), 'sys', [('alias', (1, 21, 1, 22), 'v', None)], 0, 1)], []), ('Module', [('Global', (1, 0, 1, 8), ['v'])], []), ('Module', [('Expr', (1, 0, 1, 1), ('Constant', (1, 0, 1, 1), 1, None))], []), ('Module', [('Pass', (1, 0, 1, 4))], []), diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index d2b76b46dbe..1eaf7f15bff 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -1692,8 +1692,8 @@ def check_text(code, empty, full, **kwargs): check_text( "import _ast as ast; from module import sub", - empty="Module(body=[Import(names=[alias(name='_ast', asname='ast')]), ImportFrom(module='module', names=[alias(name='sub')], level=0)])", - full="Module(body=[Import(names=[alias(name='_ast', asname='ast')]), ImportFrom(module='module', names=[alias(name='sub')], level=0)], type_ignores=[])", + empty="Module(body=[Import(names=[alias(name='_ast', asname='ast')], is_lazy=0), ImportFrom(module='module', names=[alias(name='sub')], level=0, is_lazy=0)])", + full="Module(body=[Import(names=[alias(name='_ast', asname='ast')], is_lazy=0), ImportFrom(module='module', names=[alias(name='sub')], level=0, is_lazy=0)], type_ignores=[])", ) def test_copy_location(self): diff --git a/Lib/test/test_capi/test_config.py b/Lib/test/test_capi/test_config.py index 04a27de8d84..3535b95e899 100644 --- a/Lib/test/test_capi/test_config.py +++ b/Lib/test/test_capi/test_config.py @@ -62,6 +62,7 @@ def test_config_get(self): ("int_max_str_digits", int, None), ("interactive", bool, None), ("isolated", bool, None), + ("lazy_imports", int, None), ("malloc_stats", bool, None), ("module_search_paths", list[str], "path"), ("optimization_level", int, None), diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 3e747748720..3b1564a4584 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -292,7 +292,7 @@ def wrap_func_w_kwargs(): 1 LOAD_SMALL_INT 0 LOAD_CONST 1 (('*',)) - IMPORT_NAME 0 (math) + IMPORT_NAME 2 (math + eager) CALL_INTRINSIC_1 2 (INTRINSIC_IMPORT_STAR) POP_TOP LOAD_CONST 2 (None) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index b5367941227..8df6a184e5b 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -635,6 +635,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'tracemalloc': 0, 'perf_profiling': 0, 'import_time': 0, + 'lazy_imports': -1, 'thread_inherit_context': DEFAULT_THREAD_INHERIT_CONTEXT, 'context_aware_warnings': DEFAULT_CONTEXT_AWARE_WARNINGS, 'code_debug_ranges': True, diff --git a/Lib/test/test_import/data/lazy_imports/basic2.py b/Lib/test/test_import/data/lazy_imports/basic2.py new file mode 100644 index 00000000000..f93ec89d5ab --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/basic2.py @@ -0,0 +1,4 @@ +def f(): + pass + +x = 42 diff --git a/Lib/test/test_import/data/lazy_imports/basic_compatibility_mode.py b/Lib/test/test_import/data/lazy_imports/basic_compatibility_mode.py new file mode 100644 index 00000000000..5076fa4894e --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/basic_compatibility_mode.py @@ -0,0 +1,2 @@ +__lazy_modules__ = ['test.test_import.data.lazy_imports.basic2'] +import test.test_import.data.lazy_imports.basic2 diff --git a/Lib/test/test_import/data/lazy_imports/basic_compatibility_mode_relative.py b/Lib/test/test_import/data/lazy_imports/basic_compatibility_mode_relative.py new file mode 100644 index 00000000000..e37759348f3 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/basic_compatibility_mode_relative.py @@ -0,0 +1,2 @@ +__lazy_modules__ = ['test.test_import.data.lazy_imports.basic2'] +lazy from .basic2 import f diff --git a/Lib/test/test_import/data/lazy_imports/basic_compatibility_mode_used.py b/Lib/test/test_import/data/lazy_imports/basic_compatibility_mode_used.py new file mode 100644 index 00000000000..64f36645f68 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/basic_compatibility_mode_used.py @@ -0,0 +1,3 @@ +__lazy_modules__ = ['test.test_import.data.lazy_imports.basic2'] +import test.test_import.data.lazy_imports.basic2 +test.test_import.data.lazy_imports.basic2.f() diff --git a/Lib/test/test_import/data/lazy_imports/basic_dir.py b/Lib/test/test_import/data/lazy_imports/basic_dir.py new file mode 100644 index 00000000000..ca9e29d3d99 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/basic_dir.py @@ -0,0 +1,2 @@ +lazy import test.test_import.data.lazy_imports.basic2 +x = dir() diff --git a/Lib/test/test_import/data/lazy_imports/basic_from_unused.py b/Lib/test/test_import/data/lazy_imports/basic_from_unused.py new file mode 100644 index 00000000000..686caa86a6c --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/basic_from_unused.py @@ -0,0 +1 @@ +lazy from test.test_import.data.lazy_imports import basic2 diff --git a/Lib/test/test_import/data/lazy_imports/basic_unused.py b/Lib/test/test_import/data/lazy_imports/basic_unused.py new file mode 100644 index 00000000000..bf8ae4613e4 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/basic_unused.py @@ -0,0 +1 @@ +lazy import test.test_import.data.lazy_imports.basic2 diff --git a/Lib/test/test_import/data/lazy_imports/basic_used.py b/Lib/test/test_import/data/lazy_imports/basic_used.py new file mode 100644 index 00000000000..84e354750f8 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/basic_used.py @@ -0,0 +1,3 @@ +lazy import test.test_import.data.lazy_imports.basic2 as basic2 + +basic2.f() diff --git a/Lib/test/test_import/data/lazy_imports/broken_attr_module.py b/Lib/test/test_import/data/lazy_imports/broken_attr_module.py new file mode 100644 index 00000000000..a60bca2bad0 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/broken_attr_module.py @@ -0,0 +1,3 @@ +# Module that exists but doesn't have expected attributes +x = 42 +# No 'nonexistent_attr' here diff --git a/Lib/test/test_import/data/lazy_imports/broken_module.py b/Lib/test/test_import/data/lazy_imports/broken_module.py new file mode 100644 index 00000000000..b49d4a4a15f --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/broken_module.py @@ -0,0 +1,2 @@ +# Module that raises an error during import +raise ValueError("This module always fails to import") diff --git a/Lib/test/test_import/data/lazy_imports/compatibility_mode_func.py b/Lib/test/test_import/data/lazy_imports/compatibility_mode_func.py new file mode 100644 index 00000000000..307338a0886 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/compatibility_mode_func.py @@ -0,0 +1,5 @@ +__lazy_modules__ = ['test.test_import.data.lazy_imports.basic2'] +def f(): + import test.test_import.data.lazy_imports.basic2 + +f() diff --git a/Lib/test/test_import/data/lazy_imports/compatibility_mode_try_except.py b/Lib/test/test_import/data/lazy_imports/compatibility_mode_try_except.py new file mode 100644 index 00000000000..6d54e69a9a4 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/compatibility_mode_try_except.py @@ -0,0 +1,5 @@ +__lazy_modules__ = ['test.test_import.data.lazy_imports.basic2'] +try: + import test.test_import.data.lazy_imports.basic2 +except: + pass diff --git a/Lib/test/test_import/data/lazy_imports/dunder_lazy_import.py b/Lib/test/test_import/data/lazy_imports/dunder_lazy_import.py new file mode 100644 index 00000000000..1a8a19c3c90 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/dunder_lazy_import.py @@ -0,0 +1 @@ +basic = __lazy_import__('test.test_import.data.lazy_imports.basic2') diff --git a/Lib/test/test_import/data/lazy_imports/dunder_lazy_import_builtins.py b/Lib/test/test_import/data/lazy_imports/dunder_lazy_import_builtins.py new file mode 100644 index 00000000000..594bf2b3042 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/dunder_lazy_import_builtins.py @@ -0,0 +1,13 @@ +import sys + +def myimport(*args): + return sys.modules[__name__] + + +new_globals = dict(globals()) +new_globals["__builtins__"] = { + "__import__": myimport, +} +basic2 = 42 +basic = __lazy_import__("test.test_import.data.lazy_imports", fromlist="basic2", globals=new_globals) +basic diff --git a/Lib/test/test_import/data/lazy_imports/dunder_lazy_import_used.py b/Lib/test/test_import/data/lazy_imports/dunder_lazy_import_used.py new file mode 100644 index 00000000000..635828b50ef --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/dunder_lazy_import_used.py @@ -0,0 +1,2 @@ +basic = __lazy_import__('test.test_import.data.lazy_imports', fromlist="basic2") +basic diff --git a/Lib/test/test_import/data/lazy_imports/eager_import_func.py b/Lib/test/test_import/data/lazy_imports/eager_import_func.py new file mode 100644 index 00000000000..89e643ac183 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/eager_import_func.py @@ -0,0 +1,3 @@ +def f(): + import test.test_import.data.lazy_imports.basic2 as basic2 + return basic2 diff --git a/Lib/test/test_import/data/lazy_imports/global_filter.py b/Lib/test/test_import/data/lazy_imports/global_filter.py new file mode 100644 index 00000000000..72cb5f2ef5a --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/global_filter.py @@ -0,0 +1,10 @@ +import sys + +def filter(module_name, imported_name, from_list): + assert module_name == __name__ + assert imported_name == "test.test_import.data.lazy_imports.basic2" + return False + +sys.set_lazy_imports_filter(filter) + +lazy import test.test_import.data.lazy_imports.basic2 as basic2 diff --git a/Lib/test/test_import/data/lazy_imports/global_filter_from.py b/Lib/test/test_import/data/lazy_imports/global_filter_from.py new file mode 100644 index 00000000000..733839d9c1e --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/global_filter_from.py @@ -0,0 +1,11 @@ +import importlib + +def filter(module_name, imported_name, from_list): + assert module_name == __name__ + assert imported_name == "test.test_import.data.lazy_imports.basic2" + assert from_list == ['f'] + return False + +importlib.set_lazy_imports(None, filter) + +lazy from import test.test_import.data.lazy_imports.basic2 import f diff --git a/Lib/test/test_import/data/lazy_imports/global_filter_from_true.py b/Lib/test/test_import/data/lazy_imports/global_filter_from_true.py new file mode 100644 index 00000000000..c019f1ae811 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/global_filter_from_true.py @@ -0,0 +1,11 @@ +import importlib + +def filter(module_name, imported_name, from_list): + assert module_name == __name__ + assert imported_name == "test.test_import.data.lazy_imports.basic2" + assert from_list == ['f'] + return True + +importlib.set_lazy_imports(None, filter) + +lazy from import test.test_import.data.lazy_imports.basic2 import f diff --git a/Lib/test/test_import/data/lazy_imports/global_filter_true.py b/Lib/test/test_import/data/lazy_imports/global_filter_true.py new file mode 100644 index 00000000000..4881b30fb02 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/global_filter_true.py @@ -0,0 +1,11 @@ +import sys + +def filter(module_name, imported_name, from_list): + assert module_name == __name__ + assert imported_name == "test.test_import.data.lazy_imports.basic2" + return True + +sys.set_lazy_imports("normal") +sys.set_lazy_imports_filter(filter) + +lazy import test.test_import.data.lazy_imports.basic2 as basic2 diff --git a/Lib/test/test_import/data/lazy_imports/global_off.py b/Lib/test/test_import/data/lazy_imports/global_off.py new file mode 100644 index 00000000000..4f202744a9e --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/global_off.py @@ -0,0 +1,5 @@ +import sys + +sys.set_lazy_imports("none") + +lazy import test.test_import.data.lazy_imports.basic2 as basic2 diff --git a/Lib/test/test_import/data/lazy_imports/global_on.py b/Lib/test/test_import/data/lazy_imports/global_on.py new file mode 100644 index 00000000000..3f8e1d2aa01 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/global_on.py @@ -0,0 +1,5 @@ +import sys + +sys.set_lazy_imports("all") + +import test.test_import.data.lazy_imports.basic2 as basic2 diff --git a/Lib/test/test_import/data/lazy_imports/globals_access.py b/Lib/test/test_import/data/lazy_imports/globals_access.py new file mode 100644 index 00000000000..c12c6a029c2 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/globals_access.py @@ -0,0 +1,9 @@ +# Test that globals() returns lazy proxy objects without reifying +lazy import test.test_import.data.lazy_imports.basic2 as basic2 + +def get_from_globals(): + g = globals() + return g['basic2'] + +def get_direct(): + return basic2 diff --git a/Lib/test/test_import/data/lazy_imports/lazy_class_body.py b/Lib/test/test_import/data/lazy_imports/lazy_class_body.py new file mode 100644 index 00000000000..e154b78f6cd --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/lazy_class_body.py @@ -0,0 +1,3 @@ +# SyntaxError: lazy import inside class body is not allowed +class Foo: + lazy import json diff --git a/Lib/test/test_import/data/lazy_imports/lazy_compat_from.py b/Lib/test/test_import/data/lazy_imports/lazy_compat_from.py new file mode 100644 index 00000000000..f887f47b92c --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/lazy_compat_from.py @@ -0,0 +1,6 @@ +# Test __lazy_modules__ with from imports +__lazy_modules__ = ['test.test_import.data.lazy_imports.basic2'] +from test.test_import.data.lazy_imports.basic2 import x, f + +def get_x(): + return x diff --git a/Lib/test/test_import/data/lazy_imports/lazy_future_import.py b/Lib/test/test_import/data/lazy_imports/lazy_future_import.py new file mode 100644 index 00000000000..8bd258b76b4 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/lazy_future_import.py @@ -0,0 +1 @@ +lazy from __future__ import annotations diff --git a/Lib/test/test_import/data/lazy_imports/lazy_get_value.py b/Lib/test/test_import/data/lazy_imports/lazy_get_value.py new file mode 100644 index 00000000000..0ff572fa1e3 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/lazy_get_value.py @@ -0,0 +1,7 @@ +lazy import test.test_import.data.lazy_imports.basic2 as basic2 + +def f(): + x = globals() + return x['basic2'].resolve() + +f() diff --git a/Lib/test/test_import/data/lazy_imports/lazy_import_func.py b/Lib/test/test_import/data/lazy_imports/lazy_import_func.py new file mode 100644 index 00000000000..af758b51127 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/lazy_import_func.py @@ -0,0 +1,2 @@ +def f(): + lazy import foo diff --git a/Lib/test/test_import/data/lazy_imports/lazy_import_pkg.py b/Lib/test/test_import/data/lazy_imports/lazy_import_pkg.py new file mode 100644 index 00000000000..79aa9a56739 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/lazy_import_pkg.py @@ -0,0 +1,2 @@ +lazy import test.test_import.data.lazy_imports.pkg.bar +x = test.test_import.data.lazy_imports.pkg.bar.f diff --git a/Lib/test/test_import/data/lazy_imports/lazy_try_except.py b/Lib/test/test_import/data/lazy_imports/lazy_try_except.py new file mode 100644 index 00000000000..e58d1f929ca --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/lazy_try_except.py @@ -0,0 +1,4 @@ +try: + lazy import foo +except: + pass diff --git a/Lib/test/test_import/data/lazy_imports/lazy_try_except_from.py b/Lib/test/test_import/data/lazy_imports/lazy_try_except_from.py new file mode 100644 index 00000000000..8c97553f486 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/lazy_try_except_from.py @@ -0,0 +1,4 @@ +try: + lazy from foo import bar +except: + pass diff --git a/Lib/test/test_import/data/lazy_imports/lazy_try_except_from_star.py b/Lib/test/test_import/data/lazy_imports/lazy_try_except_from_star.py new file mode 100644 index 00000000000..b2370eaae77 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/lazy_try_except_from_star.py @@ -0,0 +1 @@ +lazy from foo import * diff --git a/Lib/test/test_import/data/lazy_imports/lazy_with.py b/Lib/test/test_import/data/lazy_imports/lazy_with.py new file mode 100644 index 00000000000..b383879936a --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/lazy_with.py @@ -0,0 +1,3 @@ +import contextlib +with contextlib.nullcontext(): + lazy import test.test_import.data.lazy_imports.basic2 diff --git a/Lib/test/test_import/data/lazy_imports/lazy_with_from.py b/Lib/test/test_import/data/lazy_imports/lazy_with_from.py new file mode 100644 index 00000000000..7936326a9e3 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/lazy_with_from.py @@ -0,0 +1,3 @@ +import contextlib +with contextlib.nullcontext(): + lazy import test.test_import.data.lazy_imports.basic2 as basic2 diff --git a/Lib/test/test_import/data/lazy_imports/modules_dict.py b/Lib/test/test_import/data/lazy_imports/modules_dict.py new file mode 100644 index 00000000000..327f866398c --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/modules_dict.py @@ -0,0 +1,5 @@ +lazy import test.test_import.data.lazy_imports.basic2 as basic2 + +import sys +mod = sys.modules[__name__] +x = mod.__dict__ diff --git a/Lib/test/test_import/data/lazy_imports/modules_getattr.py b/Lib/test/test_import/data/lazy_imports/modules_getattr.py new file mode 100644 index 00000000000..ae1d4bb3f97 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/modules_getattr.py @@ -0,0 +1,5 @@ +lazy import test.test_import.data.lazy_imports.basic2 as basic2 + +import sys +mod = sys.modules[__name__] +x = mod.basic2 diff --git a/Lib/test/test_import/data/lazy_imports/modules_getattr_other.py b/Lib/test/test_import/data/lazy_imports/modules_getattr_other.py new file mode 100644 index 00000000000..e4d83e6336d --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/modules_getattr_other.py @@ -0,0 +1,5 @@ +lazy import test.test_import.data.lazy_imports.basic2 as basic2 + +import sys +mod = sys.modules[__name__] +x = mod.__name__ diff --git a/Lib/test/test_import/data/lazy_imports/multi_from_import.py b/Lib/test/test_import/data/lazy_imports/multi_from_import.py new file mode 100644 index 00000000000..96dc9757500 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/multi_from_import.py @@ -0,0 +1,5 @@ +# Test that lazy from import with multiple names only reifies accessed names +lazy from test.test_import.data.lazy_imports.basic2 import f, x + +def get_globals(): + return globals() diff --git a/Lib/test/test_import/data/lazy_imports/pkg/__init__.py b/Lib/test/test_import/data/lazy_imports/pkg/__init__.py new file mode 100644 index 00000000000..2d76abaa89f --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/pkg/__init__.py @@ -0,0 +1 @@ +x = 42 diff --git a/Lib/test/test_import/data/lazy_imports/pkg/b.py b/Lib/test/test_import/data/lazy_imports/pkg/b.py new file mode 100644 index 00000000000..a266b7c7c0d --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/pkg/b.py @@ -0,0 +1,2 @@ +def foo(): + return 'foo' diff --git a/Lib/test/test_import/data/lazy_imports/pkg/bar.py b/Lib/test/test_import/data/lazy_imports/pkg/bar.py new file mode 100644 index 00000000000..b7a4d7111a6 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/pkg/bar.py @@ -0,0 +1 @@ +def f(): pass diff --git a/Lib/test/test_import/data/lazy_imports/pkg/c.py b/Lib/test/test_import/data/lazy_imports/pkg/c.py new file mode 100644 index 00000000000..0bb03119864 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/pkg/c.py @@ -0,0 +1,4 @@ +lazy from . import b, x + +def get_globals(): + return globals() diff --git a/Lib/test/test_import/data/lazy_imports/relative_lazy.py b/Lib/test/test_import/data/lazy_imports/relative_lazy.py new file mode 100644 index 00000000000..6273d3883ab --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/relative_lazy.py @@ -0,0 +1,5 @@ +# Test relative imports with lazy keyword +lazy from . import basic2 + +def get_basic2(): + return basic2 diff --git a/Lib/test/test_import/data/lazy_imports/relative_lazy_from.py b/Lib/test/test_import/data/lazy_imports/relative_lazy_from.py new file mode 100644 index 00000000000..1bae2d6853d --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/relative_lazy_from.py @@ -0,0 +1,8 @@ +# Test relative from imports with lazy keyword +lazy from .basic2 import x, f + +def get_x(): + return x + +def get_f(): + return f diff --git a/Lib/test/test_import/data/lazy_imports/try_except_eager.py b/Lib/test/test_import/data/lazy_imports/try_except_eager.py new file mode 100644 index 00000000000..4cdaa9a9b48 --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/try_except_eager.py @@ -0,0 +1,4 @@ +try: + import test.test_import.data.lazy_imports.basic2 +except: + pass diff --git a/Lib/test/test_import/data/lazy_imports/try_except_eager_from.py b/Lib/test/test_import/data/lazy_imports/try_except_eager_from.py new file mode 100644 index 00000000000..6eadaaa71ca --- /dev/null +++ b/Lib/test/test_import/data/lazy_imports/try_except_eager_from.py @@ -0,0 +1,4 @@ +try: + from test.test_import.data.lazy_imports.basic2 import f +except: + pass diff --git a/Lib/test/test_import/test_lazy_imports.py b/Lib/test/test_import/test_lazy_imports.py new file mode 100644 index 00000000000..60039ecb085 --- /dev/null +++ b/Lib/test/test_import/test_lazy_imports.py @@ -0,0 +1,1596 @@ +"""Tests for PEP 810 lazy imports.""" + +import sys +import types +import unittest +import threading +import textwrap +import subprocess +from test.support import import_helper + + +class LazyImportTests(unittest.TestCase): + """Tests for basic lazy import functionality.""" + + def tearDown(self): + """Clean up any test modules from sys.modules.""" + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_basic_unused(self): + """Lazy imported module should not be loaded if never accessed.""" + import test.test_import.data.lazy_imports.basic_unused + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_basic_unused_use_externally(self): + """Lazy import should load module when accessed from outside.""" + from test.test_import.data.lazy_imports import basic_unused + + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + x = basic_unused.test.test_import.data.lazy_imports.basic2 + self.assertTrue("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_basic_from_unused_use_externally(self): + """Lazy 'from' import should load when accessed from outside.""" + from test.test_import.data.lazy_imports import basic_from_unused + + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + x = basic_from_unused.basic2 + self.assertTrue("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_basic_unused_dir(self): + """dir() on module should not trigger lazy import reification.""" + import test.test_import.data.lazy_imports.basic_unused + + x = dir(test.test_import.data.lazy_imports.basic_unused) + self.assertIn("test", x) + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_basic_dir(self): + """dir() at module scope should not trigger lazy import reification.""" + from test.test_import.data.lazy_imports import basic_dir + + self.assertIn("test", basic_dir.x) + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_basic_used(self): + """Lazy import should load when accessed within the module.""" + import test.test_import.data.lazy_imports.basic_used + self.assertTrue("test.test_import.data.lazy_imports.basic2" in sys.modules) + + +class GlobalLazyImportModeTests(unittest.TestCase): + """Tests for sys.set_lazy_imports() global mode control.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_global_off(self): + """Mode 'none' should disable lazy imports entirely.""" + import test.test_import.data.lazy_imports.global_off + self.assertTrue("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_global_on(self): + """Mode 'all' should make regular imports lazy.""" + import test.test_import.data.lazy_imports.global_on + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_global_filter(self): + """Filter returning False should prevent lazy loading.""" + import test.test_import.data.lazy_imports.global_filter + self.assertTrue("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_global_filter_true(self): + """Filter returning True should allow lazy loading.""" + import test.test_import.data.lazy_imports.global_filter_true + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_global_filter_from(self): + """Filter should work with 'from' imports.""" + import test.test_import.data.lazy_imports.global_filter + self.assertTrue("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_global_filter_from_true(self): + """Filter returning True should allow lazy 'from' imports.""" + import test.test_import.data.lazy_imports.global_filter_true + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + + +class CompatibilityModeTests(unittest.TestCase): + """Tests for __lazy_modules__ compatibility mode.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_compatibility_mode(self): + """__lazy_modules__ should enable lazy imports for listed modules.""" + import test.test_import.data.lazy_imports.basic_compatibility_mode + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_compatibility_mode_used(self): + """Using a lazy import from __lazy_modules__ should load the module.""" + import test.test_import.data.lazy_imports.basic_compatibility_mode_used + self.assertTrue("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_compatibility_mode_func(self): + """Imports inside functions should be eager even in compatibility mode.""" + import test.test_import.data.lazy_imports.compatibility_mode_func + self.assertTrue("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_compatibility_mode_try_except(self): + """Imports in try/except should be eager even in compatibility mode.""" + import test.test_import.data.lazy_imports.compatibility_mode_try_except + self.assertTrue("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_compatibility_mode_relative(self): + """__lazy_modules__ should work with relative imports.""" + import test.test_import.data.lazy_imports.basic_compatibility_mode_relative + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + + +class ModuleIntrospectionTests(unittest.TestCase): + """Tests for module dict and getattr behavior with lazy imports.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_modules_dict(self): + """Accessing module.__dict__ should not trigger reification.""" + import test.test_import.data.lazy_imports.modules_dict + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_modules_getattr(self): + """Module __getattr__ for lazy import name should trigger reification.""" + import test.test_import.data.lazy_imports.modules_getattr + self.assertTrue("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_modules_getattr_other(self): + """Module __getattr__ for other names should not trigger reification.""" + import test.test_import.data.lazy_imports.modules_getattr_other + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + + +class LazyImportTypeTests(unittest.TestCase): + """Tests for the LazyImportType and its resolve() method.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_lazy_value_resolve(self): + """resolve() method should force the lazy import to load.""" + import test.test_import.data.lazy_imports.lazy_get_value + self.assertTrue("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_lazy_import_type_exposed(self): + """LazyImportType should be exposed in types module.""" + self.assertTrue(hasattr(types, 'LazyImportType')) + self.assertEqual(types.LazyImportType.__name__, 'lazy_import') + + def test_lazy_import_type_invalid_fromlist_type(self): + """LazyImportType should reject invalid fromlist types.""" + # fromlist must be None, a string, or a tuple - not an int + with self.assertRaises(TypeError) as cm: + types.LazyImportType({}, "module", 0) + self.assertIn("fromlist must be None, a string, or a tuple", str(cm.exception)) + + # Also test with other invalid types + with self.assertRaises(TypeError): + types.LazyImportType({}, "module", []) # list not allowed + + with self.assertRaises(TypeError): + types.LazyImportType({}, "module", {"x": 1}) # dict not allowed + + def test_lazy_import_type_valid_fromlist(self): + """LazyImportType should accept valid fromlist types.""" + # None is valid (implicit) + lazy1 = types.LazyImportType({}, "module") + self.assertIsNotNone(lazy1) + + # Explicit None is valid + lazy2 = types.LazyImportType({}, "module", None) + self.assertIsNotNone(lazy2) + + # String is valid + lazy3 = types.LazyImportType({}, "module", "attr") + self.assertIsNotNone(lazy3) + + # Tuple is valid + lazy4 = types.LazyImportType({}, "module", ("attr1", "attr2")) + self.assertIsNotNone(lazy4) + + +class SyntaxRestrictionTests(unittest.TestCase): + """Tests for syntax restrictions on lazy imports.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_lazy_try_except(self): + """lazy import inside try/except should raise SyntaxError.""" + with self.assertRaises(SyntaxError): + import test.test_import.data.lazy_imports.lazy_try_except + + def test_lazy_try_except_from(self): + """lazy from import inside try/except should raise SyntaxError.""" + with self.assertRaises(SyntaxError): + import test.test_import.data.lazy_imports.lazy_try_except_from + + def test_lazy_try_except_from_star(self): + """lazy from import * should raise SyntaxError.""" + with self.assertRaises(SyntaxError): + import test.test_import.data.lazy_imports.lazy_try_except_from_star + + def test_lazy_future_import(self): + """lazy from __future__ import should raise SyntaxError.""" + with self.assertRaises(SyntaxError) as cm: + import test.test_import.data.lazy_imports.lazy_future_import + # Check we highlight 'lazy' (column offset 0, end offset 4) + self.assertEqual(cm.exception.offset, 1) + self.assertEqual(cm.exception.end_offset, 5) + + def test_lazy_import_func(self): + """lazy import inside function should raise SyntaxError.""" + with self.assertRaises(SyntaxError): + import test.test_import.data.lazy_imports.lazy_import_func + + def test_lazy_import_exec_in_function(self): + """lazy import via exec() inside a function should raise SyntaxError.""" + # exec() inside a function creates a non-module-level context + # where lazy imports are not allowed + def f(): + exec("lazy import json") + + with self.assertRaises(SyntaxError) as cm: + f() + self.assertIn("only allowed at module level", str(cm.exception)) + + def test_lazy_import_exec_at_module_level(self): + """lazy import via exec() at module level should work.""" + # exec() at module level (globals == locals) should allow lazy imports + code = textwrap.dedent(""" + import sys + exec("lazy import json") + # Should be lazy - not loaded yet + assert 'json' not in sys.modules + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + +class EagerImportInLazyModeTests(unittest.TestCase): + """Tests for imports that should remain eager even in lazy mode.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_try_except_eager(self): + """Imports in try/except should be eager even with mode='all'.""" + sys.set_lazy_imports("all") + import test.test_import.data.lazy_imports.try_except_eager + self.assertTrue("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_try_except_eager_from(self): + """From imports in try/except should be eager even with mode='all'.""" + sys.set_lazy_imports("all") + import test.test_import.data.lazy_imports.try_except_eager_from + self.assertTrue("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_eager_import_func(self): + """Imports inside functions should return modules, not proxies.""" + sys.set_lazy_imports("all") + import test.test_import.data.lazy_imports.eager_import_func + + f = test.test_import.data.lazy_imports.eager_import_func.f + self.assertEqual(type(f()), type(sys)) + + +class WithStatementTests(unittest.TestCase): + """Tests for lazy imports in with statement context.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_lazy_with(self): + """lazy import with 'with' statement should work.""" + import test.test_import.data.lazy_imports.lazy_with + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_lazy_with_from(self): + """lazy from import with 'with' statement should work.""" + import test.test_import.data.lazy_imports.lazy_with_from + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + + +class PackageTests(unittest.TestCase): + """Tests for lazy imports with packages.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_lazy_import_pkg(self): + """lazy import of package submodule should load the package.""" + import test.test_import.data.lazy_imports.lazy_import_pkg + + self.assertTrue("test.test_import.data.lazy_imports.pkg" in sys.modules) + self.assertTrue("test.test_import.data.lazy_imports.pkg.bar" in sys.modules) + + def test_lazy_import_pkg_cross_import(self): + """Cross-imports within package should preserve lazy imports.""" + import test.test_import.data.lazy_imports.pkg.c + + self.assertTrue("test.test_import.data.lazy_imports.pkg" in sys.modules) + self.assertTrue("test.test_import.data.lazy_imports.pkg.c" in sys.modules) + self.assertFalse("test.test_import.data.lazy_imports.pkg.b" in sys.modules) + + g = test.test_import.data.lazy_imports.pkg.c.get_globals() + self.assertEqual(type(g["x"]), int) + self.assertEqual(type(g["b"]), types.LazyImportType) + + +class DunderLazyImportTests(unittest.TestCase): + """Tests for __lazy_import__ builtin function.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_dunder_lazy_import(self): + """__lazy_import__ should create lazy import proxy.""" + import test.test_import.data.lazy_imports.dunder_lazy_import + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_dunder_lazy_import_used(self): + """Using __lazy_import__ result should trigger module load.""" + import test.test_import.data.lazy_imports.dunder_lazy_import_used + self.assertTrue("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_dunder_lazy_import_builtins(self): + """__lazy_import__ should use module's __builtins__ for __import__.""" + from test.test_import.data.lazy_imports import dunder_lazy_import_builtins + + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + self.assertEqual(dunder_lazy_import_builtins.basic, 42) + + +class SysLazyImportsAPITests(unittest.TestCase): + """Tests for sys lazy imports API functions.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_set_lazy_imports_requires_string(self): + """set_lazy_imports should reject non-string arguments.""" + with self.assertRaises(TypeError): + sys.set_lazy_imports(True) + with self.assertRaises(TypeError): + sys.set_lazy_imports(None) + with self.assertRaises(TypeError): + sys.set_lazy_imports(1) + + def test_set_lazy_imports_rejects_invalid_mode(self): + """set_lazy_imports should reject invalid mode strings.""" + with self.assertRaises(ValueError): + sys.set_lazy_imports("invalid") + with self.assertRaises(ValueError): + sys.set_lazy_imports("on") + with self.assertRaises(ValueError): + sys.set_lazy_imports("off") + + def test_get_lazy_imports_returns_string(self): + """get_lazy_imports should return string modes.""" + sys.set_lazy_imports("normal") + self.assertEqual(sys.get_lazy_imports(), "normal") + + sys.set_lazy_imports("all") + self.assertEqual(sys.get_lazy_imports(), "all") + + sys.set_lazy_imports("none") + self.assertEqual(sys.get_lazy_imports(), "none") + + def test_get_lazy_imports_filter_default(self): + """get_lazy_imports_filter should return None by default.""" + sys.set_lazy_imports_filter(None) + self.assertIsNone(sys.get_lazy_imports_filter()) + + def test_set_and_get_lazy_imports_filter(self): + """set/get_lazy_imports_filter should round-trip filter function.""" + def my_filter(name): + return name.startswith("test.") + + sys.set_lazy_imports_filter(my_filter) + self.assertIs(sys.get_lazy_imports_filter(), my_filter) + + def test_get_lazy_modules_returns_set(self): + """get_lazy_modules should return a set per PEP 810.""" + result = sys.get_lazy_modules() + self.assertIsInstance(result, set) + + def test_lazy_modules_attribute_is_set(self): + """sys.lazy_modules should be a set per PEP 810.""" + self.assertIsInstance(sys.lazy_modules, set) + self.assertIs(sys.lazy_modules, sys.get_lazy_modules()) + + def test_lazy_modules_tracks_lazy_imports(self): + """sys.lazy_modules should track lazily imported module names.""" + code = textwrap.dedent(""" + import sys + initial_count = len(sys.lazy_modules) + import test.test_import.data.lazy_imports.basic_unused + assert "test.test_import.data.lazy_imports.basic2" in sys.lazy_modules + assert len(sys.lazy_modules) > initial_count + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + +class ErrorHandlingTests(unittest.TestCase): + """Tests for error handling during lazy import reification. + + PEP 810: Errors during reification should show exception chaining with + both the lazy import definition location and the access location. + """ + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_import_error_shows_chained_traceback(self): + """ImportError during reification should chain to show both definition and access.""" + # Errors at reification must show where the lazy import was defined + # AND where the access happened, per PEP 810 "Reification" section + code = textwrap.dedent(""" + import sys + lazy import test.test_import.data.lazy_imports.nonexistent_module + + try: + x = test.test_import.data.lazy_imports.nonexistent_module + except ImportError as e: + # Should have __cause__ showing the original error + # The exception chain shows both where import was defined and where access happened + assert e.__cause__ is not None, "Expected chained exception" + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_attribute_error_on_from_import_shows_chained_traceback(self): + """Accessing missing attribute from lazy from-import should chain errors.""" + # Tests 'lazy from module import nonexistent' behavior + code = textwrap.dedent(""" + import sys + lazy from test.test_import.data.lazy_imports.basic2 import nonexistent_name + + try: + x = nonexistent_name + except ImportError as e: + # PEP 810: Enhanced error reporting through exception chaining + assert e.__cause__ is not None, "Expected chained exception" + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_reification_retries_on_failure(self): + """Failed reification should allow retry on subsequent access. + + PEP 810: "If reification fails, the lazy object is not reified or replaced. + Subsequent uses of the lazy object will re-try the reification." + """ + code = textwrap.dedent(""" + import sys + import types + + lazy import test.test_import.data.lazy_imports.broken_module + + # First access - should fail + try: + x = test.test_import.data.lazy_imports.broken_module + except ValueError: + pass + + # The lazy object should still be a lazy proxy (not reified) + g = globals() + lazy_obj = g['test'] + # The root 'test' binding should still allow retry + # Second access - should also fail (retry the import) + try: + x = test.test_import.data.lazy_imports.broken_module + except ValueError: + print("OK - retry worked") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_error_during_module_execution_propagates(self): + """Errors in module code during reification should propagate correctly.""" + # Module that raises during import should propagate with chaining + code = textwrap.dedent(""" + import sys + lazy import test.test_import.data.lazy_imports.broken_module + + try: + _ = test.test_import.data.lazy_imports.broken_module + print("FAIL - should have raised") + except ValueError as e: + # The ValueError from the module should be the cause + if "always fails" in str(e) or (e.__cause__ and "always fails" in str(e.__cause__)): + print("OK") + else: + print(f"FAIL - wrong error: {e}") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + +class GlobalsAndDictTests(unittest.TestCase): + """Tests for globals() and __dict__ behavior with lazy imports. + + PEP 810: "Calling globals() or accessing a module's __dict__ does not trigger + reification – they return the module's dictionary, and accessing lazy objects + through that dictionary still returns lazy proxy objects." + """ + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_globals_returns_lazy_proxy_when_accessed_from_function(self): + """globals() accessed from a function should return lazy proxy without reification. + + Note: At module level, accessing globals()['name'] triggers LOAD_NAME which + automatically resolves lazy imports. Inside a function, accessing globals()['name'] + uses BINARY_SUBSCR which returns the lazy proxy without resolution. + """ + code = textwrap.dedent(""" + import sys + import types + + lazy from test.test_import.data.lazy_imports.basic2 import x + + # Check that module is not yet loaded + assert 'test.test_import.data.lazy_imports.basic2' not in sys.modules + + def check_lazy(): + # Access through globals() from inside a function + g = globals() + lazy_obj = g['x'] + return type(lazy_obj) is types.LazyImportType + + # Inside function, should get lazy proxy + is_lazy = check_lazy() + assert is_lazy, "Expected LazyImportType from function scope" + + # Module should STILL not be loaded + assert 'test.test_import.data.lazy_imports.basic2' not in sys.modules + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_globals_dict_access_returns_lazy_proxy_inline(self): + """Accessing globals()['name'] inline should return lazy proxy. + + Note: Assigning g['name'] to a local variable at module level triggers + reification due to STORE_NAME bytecode. Inline access preserves laziness. + """ + code = textwrap.dedent(""" + import sys + import types + lazy import json + # Inline access without assignment to local variable preserves lazy proxy + assert type(globals()['json']) is types.LazyImportType + assert 'json' not in sys.modules + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_module_dict_returns_lazy_proxy_without_reifying(self): + """module.__dict__ access should not trigger reification.""" + import test.test_import.data.lazy_imports.globals_access + + # Module not loaded yet via direct dict access + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + + # Access via get_from_globals should return lazy proxy + lazy_obj = test.test_import.data.lazy_imports.globals_access.get_from_globals() + self.assertEqual(type(lazy_obj), types.LazyImportType) + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_direct_access_triggers_reification(self): + """Direct name access (not through globals()) should trigger reification.""" + import test.test_import.data.lazy_imports.globals_access + + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + + # Direct access should reify + result = test.test_import.data.lazy_imports.globals_access.get_direct() + self.assertTrue("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_resolve_method_forces_reification(self): + """Calling resolve() on lazy proxy should force reification. + + Note: Must access lazy proxy from within a function to avoid automatic + reification by LOAD_NAME at module level. + """ + code = textwrap.dedent(""" + import sys + import types + + lazy from test.test_import.data.lazy_imports.basic2 import x + + assert 'test.test_import.data.lazy_imports.basic2' not in sys.modules + + def test_resolve(): + g = globals() + lazy_obj = g['x'] + assert type(lazy_obj) is types.LazyImportType, f"Expected lazy proxy, got {type(lazy_obj)}" + + resolved = lazy_obj.resolve() + + # Now module should be loaded + assert 'test.test_import.data.lazy_imports.basic2' in sys.modules + assert resolved == 42 # x is 42 in basic2.py + return True + + assert test_resolve() + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + +class MultipleNameFromImportTests(unittest.TestCase): + """Tests for lazy from ... import with multiple names. + + PEP 810: "When using lazy from ... import, each imported name is bound to a + lazy proxy object. The first access to any of these names triggers loading + of the entire module and reifies only that specific name to its actual value. + Other names remain as lazy proxies until they are accessed." + """ + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_accessing_one_name_leaves_others_as_proxies(self): + """Accessing one name from multi-name import should leave others lazy.""" + code = textwrap.dedent(""" + import sys + import types + + lazy from test.test_import.data.lazy_imports.basic2 import f, x + + # Neither should be loaded yet + assert 'test.test_import.data.lazy_imports.basic2' not in sys.modules + + g = globals() + assert type(g['f']) is types.LazyImportType + assert type(g['x']) is types.LazyImportType + + # Access 'x' - this loads the module and reifies only 'x' + value = x + assert value == 42 + + # Module is now loaded + assert 'test.test_import.data.lazy_imports.basic2' in sys.modules + + # 'x' should be reified (int), 'f' should still be lazy proxy + assert type(g['x']) is int, f"Expected int, got {type(g['x'])}" + assert type(g['f']) is types.LazyImportType, f"Expected LazyImportType, got {type(g['f'])}" + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_all_names_reified_after_all_accessed(self): + """All names should be reified after each is accessed.""" + code = textwrap.dedent(""" + import sys + import types + + lazy from test.test_import.data.lazy_imports.basic2 import f, x + + g = globals() + + # Access both + _ = x + _ = f + + # Both should be reified now + assert type(g['x']) is int + assert callable(g['f']) + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + +class SysLazyModulesTrackingTests(unittest.TestCase): + """Tests for sys.lazy_modules tracking behavior. + + PEP 810: "When the module is reified, it's removed from sys.lazy_modules" + """ + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_module_added_to_lazy_modules_on_lazy_import(self): + """Module should be added to sys.lazy_modules when lazily imported.""" + # PEP 810 states lazy_modules tracks modules that have been lazily imported + # Note: The current implementation keeps modules in lazy_modules even after + # reification (primarily for diagnostics and introspection) + code = textwrap.dedent(""" + import sys + + initial_count = len(sys.lazy_modules) + + lazy import test.test_import.data.lazy_imports.basic2 + + # Should be in lazy_modules after lazy import + assert "test.test_import.data.lazy_imports.basic2" in sys.lazy_modules + assert len(sys.lazy_modules) > initial_count + + # Trigger reification + _ = test.test_import.data.lazy_imports.basic2.x + + # Module should still be tracked (for diagnostics per PEP 810) + assert "test.test_import.data.lazy_imports.basic2" in sys.lazy_modules + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_lazy_modules_is_per_interpreter(self): + """Each interpreter should have independent sys.lazy_modules.""" + # Basic test that sys.lazy_modules exists and is a set + self.assertIsInstance(sys.lazy_modules, set) + self.assertIs(sys.lazy_modules, sys.get_lazy_modules()) + + +class CommandLineAndEnvVarTests(unittest.TestCase): + """Tests for command-line and environment variable control. + + PEP 810: The global lazy imports flag can be controlled through: + - The -X lazy_imports= command-line option + - The PYTHON_LAZY_IMPORTS= environment variable + """ + + def test_cli_lazy_imports_all_makes_regular_imports_lazy(self): + """-X lazy_imports=all should make all imports potentially lazy.""" + code = textwrap.dedent(""" + import sys + # In 'all' mode, regular imports become lazy + import json + # json should not be in sys.modules yet (lazy) + # Actually accessing it triggers reification + if 'json' not in sys.modules: + print("LAZY") + else: + print("EAGER") + """) + result = subprocess.run( + [sys.executable, "-X", "lazy_imports=all", "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") + self.assertIn("LAZY", result.stdout) + + def test_cli_lazy_imports_none_forces_all_imports_eager(self): + """-X lazy_imports=none should force all imports to be eager.""" + code = textwrap.dedent(""" + import sys + # Even explicit lazy imports should be eager in 'none' mode + lazy import json + if 'json' in sys.modules: + print("EAGER") + else: + print("LAZY") + """) + result = subprocess.run( + [sys.executable, "-X", "lazy_imports=none", "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") + self.assertIn("EAGER", result.stdout) + + def test_cli_lazy_imports_normal_respects_lazy_keyword_only(self): + """-X lazy_imports=normal should respect lazy keyword only.""" + # Note: Use test modules instead of stdlib modules to avoid + # modules already loaded by the interpreter startup + code = textwrap.dedent(""" + import sys + import test.test_import.data.lazy_imports.basic2 # Should be eager + lazy import test.test_import.data.lazy_imports.pkg.b # Should be lazy + + eager_loaded = 'test.test_import.data.lazy_imports.basic2' in sys.modules + lazy_loaded = 'test.test_import.data.lazy_imports.pkg.b' in sys.modules + + if eager_loaded and not lazy_loaded: + print("OK") + else: + print(f"FAIL: eager={eager_loaded}, lazy={lazy_loaded}") + """) + result = subprocess.run( + [sys.executable, "-X", "lazy_imports=normal", "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_env_var_lazy_imports_all_enables_global_lazy(self): + """PYTHON_LAZY_IMPORTS=all should enable global lazy imports.""" + code = textwrap.dedent(""" + import sys + import json + if 'json' not in sys.modules: + print("LAZY") + else: + print("EAGER") + """) + import os + env = os.environ.copy() + env["PYTHON_LAZY_IMPORTS"] = "all" + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + env=env + ) + self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") + self.assertIn("LAZY", result.stdout) + + def test_env_var_lazy_imports_none_disables_all_lazy(self): + """PYTHON_LAZY_IMPORTS=none should disable all lazy imports.""" + code = textwrap.dedent(""" + import sys + lazy import json + if 'json' in sys.modules: + print("EAGER") + else: + print("LAZY") + """) + import os + env = os.environ.copy() + env["PYTHON_LAZY_IMPORTS"] = "none" + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + env=env + ) + self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") + self.assertIn("EAGER", result.stdout) + + def test_cli_overrides_env_var(self): + """Command-line option should take precedence over environment variable.""" + # PEP 810: -X lazy_imports takes precedence over PYTHON_LAZY_IMPORTS + code = textwrap.dedent(""" + import sys + lazy import json + if 'json' in sys.modules: + print("EAGER") + else: + print("LAZY") + """) + import os + env = os.environ.copy() + env["PYTHON_LAZY_IMPORTS"] = "all" # env says all + result = subprocess.run( + [sys.executable, "-X", "lazy_imports=none", "-c", code], # CLI says none + capture_output=True, + text=True, + env=env + ) + self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") + # CLI should win - imports should be eager + self.assertIn("EAGER", result.stdout) + + def test_sys_set_lazy_imports_overrides_cli(self): + """sys.set_lazy_imports() should take precedence over CLI option.""" + code = textwrap.dedent(""" + import sys + sys.set_lazy_imports("none") # Override CLI + lazy import json + if 'json' in sys.modules: + print("EAGER") + else: + print("LAZY") + """) + result = subprocess.run( + [sys.executable, "-X", "lazy_imports=all", "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") + self.assertIn("EAGER", result.stdout) + + +class FilterFunctionSignatureTests(unittest.TestCase): + """Tests for the filter function signature per PEP 810. + + PEP 810: func(importer: str, name: str, fromlist: tuple[str, ...] | None) -> bool + """ + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_filter_receives_correct_arguments_for_import(self): + """Filter should receive (importer, name, fromlist=None) for 'import x'.""" + code = textwrap.dedent(""" + import sys + + received_args = [] + + def my_filter(importer, name, fromlist): + received_args.append((importer, name, fromlist)) + return True + + sys.set_lazy_imports_filter(my_filter) + + lazy import json + + assert len(received_args) == 1, f"Expected 1 call, got {len(received_args)}" + importer, name, fromlist = received_args[0] + assert name == "json", f"Expected name='json', got {name!r}" + assert fromlist is None, f"Expected fromlist=None, got {fromlist!r}" + assert isinstance(importer, str), f"Expected str importer, got {type(importer)}" + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_filter_receives_fromlist_for_from_import(self): + """Filter should receive fromlist tuple for 'from x import y, z'.""" + code = textwrap.dedent(""" + import sys + + received_args = [] + + def my_filter(importer, name, fromlist): + received_args.append((importer, name, fromlist)) + return True + + sys.set_lazy_imports_filter(my_filter) + + lazy from json import dumps, loads + + assert len(received_args) == 1, f"Expected 1 call, got {len(received_args)}" + importer, name, fromlist = received_args[0] + assert name == "json", f"Expected name='json', got {name!r}" + assert fromlist == ("dumps", "loads"), f"Expected ('dumps', 'loads'), got {fromlist!r}" + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_filter_returning_false_forces_eager_import(self): + """Filter returning False should make import eager.""" + code = textwrap.dedent(""" + import sys + + def deny_filter(importer, name, fromlist): + return False + + sys.set_lazy_imports_filter(deny_filter) + + lazy import json + + # Should be eager due to filter + if 'json' in sys.modules: + print("EAGER") + else: + print("LAZY") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") + self.assertIn("EAGER", result.stdout) + + +class AdditionalSyntaxRestrictionTests(unittest.TestCase): + """Additional syntax restriction tests per PEP 810.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_lazy_import_inside_class_raises_syntax_error(self): + """lazy import inside class body should raise SyntaxError.""" + # PEP 810: "The soft keyword is only allowed at the global (module) level, + # not inside functions, class bodies, try blocks, or import *" + with self.assertRaises(SyntaxError): + import test.test_import.data.lazy_imports.lazy_class_body + + +class MixedLazyEagerImportTests(unittest.TestCase): + """Tests for mixing lazy and eager imports of the same module. + + PEP 810: "If module foo is imported both lazily and eagerly in the same + program, the eager import takes precedence and both bindings resolve to + the same module object." + """ + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_eager_import_before_lazy_resolves_to_same_module(self): + """Eager import before lazy should make lazy resolve to same module.""" + code = textwrap.dedent(""" + import sys + import json # Eager import first + + lazy import json as lazy_json # Lazy import same module + + # lazy_json should resolve to the same object + assert json is lazy_json, "Lazy and eager imports should resolve to same module" + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_lazy_import_before_eager_resolves_to_same_module(self): + """Lazy import followed by eager should both point to same module.""" + code = textwrap.dedent(""" + import sys + + lazy import json as lazy_json + + # Lazy not loaded yet + assert 'json' not in sys.modules + + import json # Eager import triggers load + + # Both should be the same object + assert json is lazy_json + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + +class RelativeImportTests(unittest.TestCase): + """Tests for relative imports with lazy keyword.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_relative_lazy_import(self): + """lazy from . import submodule should work.""" + from test.test_import.data.lazy_imports import relative_lazy + + # basic2 should not be loaded yet + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + + # Access triggers reification + result = relative_lazy.get_basic2() + self.assertTrue("test.test_import.data.lazy_imports.basic2" in sys.modules) + + def test_relative_lazy_from_import(self): + """lazy from .module import name should work.""" + from test.test_import.data.lazy_imports import relative_lazy_from + + # basic2 should not be loaded yet + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + + # Access triggers reification + result = relative_lazy_from.get_x() + self.assertEqual(result, 42) + self.assertTrue("test.test_import.data.lazy_imports.basic2" in sys.modules) + + +class LazyModulesCompatibilityFromImportTests(unittest.TestCase): + """Tests for __lazy_modules__ with from imports. + + PEP 810: "When a module is made lazy this way, from-imports using that + module are also lazy" + """ + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_lazy_modules_makes_from_imports_lazy(self): + """__lazy_modules__ should make from imports of listed modules lazy.""" + from test.test_import.data.lazy_imports import lazy_compat_from + + # basic2 should not be loaded yet because it's in __lazy_modules__ + self.assertFalse("test.test_import.data.lazy_imports.basic2" in sys.modules) + + # Access triggers reification + result = lazy_compat_from.get_x() + self.assertEqual(result, 42) + self.assertTrue("test.test_import.data.lazy_imports.basic2" in sys.modules) + + +class ImportStateAtReificationTests(unittest.TestCase): + """Tests for import system state at reification time. + + PEP 810: "Reification still calls __import__ to resolve the import, which uses + the state of the import system (e.g. sys.path, sys.meta_path, sys.path_hooks + and __import__) at reification time, not the state when the lazy import + statement was evaluated." + """ + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_sys_path_at_reification_time_is_used(self): + """sys.path changes after lazy import should affect reification.""" + code = textwrap.dedent(""" + import sys + import tempfile + import os + + # Create a temporary module + with tempfile.TemporaryDirectory() as tmpdir: + mod_path = os.path.join(tmpdir, "dynamic_test_module.py") + with open(mod_path, "w") as f: + f.write("VALUE = 'from_temp_dir'\\n") + + # Lazy import before adding to path + lazy import dynamic_test_module + + # Module cannot be found yet + try: + _ = dynamic_test_module + print("FAIL - should not find module") + except ModuleNotFoundError: + pass + + # Now add temp dir to path + sys.path.insert(0, tmpdir) + + # Now reification should succeed using current sys.path + assert dynamic_test_module.VALUE == 'from_temp_dir' + print("OK") + + sys.path.remove(tmpdir) + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + +class ThreadSafetyTests(unittest.TestCase): + """Tests for thread-safety of lazy imports.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_import.data.lazy_imports'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_concurrent_lazy_import_reification(self): + """Multiple threads racing to reify the same lazy import should succeed.""" + from test.test_import.data.lazy_imports import basic_unused + + num_threads = 10 + results = [None] * num_threads + errors = [] + barrier = threading.Barrier(num_threads) + + def access_lazy_import(idx): + try: + barrier.wait() + module = basic_unused.test.test_import.data.lazy_imports.basic2 + results[idx] = module + except Exception as e: + errors.append((idx, e)) + + threads = [ + threading.Thread(target=access_lazy_import, args=(i,)) + for i in range(num_threads) + ] + + for t in threads: + t.start() + for t in threads: + t.join() + + self.assertEqual(errors, [], f"Errors occurred: {errors}") + self.assertTrue(all(r is not None for r in results)) + first_module = results[0] + for r in results[1:]: + self.assertIs(r, first_module) + + def test_concurrent_reification_multiple_modules(self): + """Multiple threads reifying different lazy imports concurrently.""" + code = textwrap.dedent(""" + import sys + import threading + + sys.set_lazy_imports("all") + + lazy import json + lazy import os + lazy import io + lazy import re + + num_threads = 8 + results = {} + errors = [] + barrier = threading.Barrier(num_threads) + + def access_modules(idx): + try: + barrier.wait() + mods = [json, os, io, re] + results[idx] = [type(m).__name__ for m in mods] + except Exception as e: + errors.append((idx, e)) + + threads = [ + threading.Thread(target=access_modules, args=(i,)) + for i in range(num_threads) + ] + + for t in threads: + t.start() + for t in threads: + t.join() + + assert not errors, f"Errors: {errors}" + for idx, mods in results.items(): + assert all(m == 'module' for m in mods), f"Thread {idx} got: {mods}" + + print("OK") + """) + + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_concurrent_lazy_modules_set_updates(self): + """Multiple threads creating lazy imports should safely update sys.lazy_modules.""" + code = textwrap.dedent(""" + import sys + import threading + + num_threads = 16 + iterations = 50 + errors = [] + barrier = threading.Barrier(num_threads) + + def create_lazy_imports(idx): + try: + barrier.wait() + for i in range(iterations): + exec(f"lazy import json as json_{idx}_{i}", globals()) + exec(f"lazy import os as os_{idx}_{i}", globals()) + except Exception as e: + errors.append((idx, e)) + + threads = [ + threading.Thread(target=create_lazy_imports, args=(i,)) + for i in range(num_threads) + ] + + for t in threads: + t.start() + for t in threads: + t.join() + + assert not errors, f"Errors: {errors}" + assert isinstance(sys.lazy_modules, set), "sys.lazy_modules is not a set" + print("OK") + """) + + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_concurrent_reification_same_module_high_contention(self): + """High contention: many threads reifying the exact same lazy import.""" + code = textwrap.dedent(""" + import sys + import threading + import types + + sys.set_lazy_imports("all") + + lazy import json + + num_threads = 20 + results = [None] * num_threads + errors = [] + barrier = threading.Barrier(num_threads) + + def access_json(idx): + try: + barrier.wait() + for _ in range(100): + _ = json.dumps + _ = json.loads + results[idx] = json + except Exception as e: + errors.append((idx, e)) + + threads = [ + threading.Thread(target=access_json, args=(i,)) + for i in range(num_threads) + ] + + for t in threads: + t.start() + for t in threads: + t.join() + + assert not errors, f"Errors: {errors}" + assert all(r is not None for r in results), "Some threads got None" + first = results[0] + assert all(r is first for r in results), "Inconsistent module objects" + assert not isinstance(first, types.LazyImportType), "Got lazy import instead of module" + print("OK") + """) + + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_concurrent_reification_with_module_attribute_access(self): + """Threads racing to reify and immediately access module attributes.""" + code = textwrap.dedent(""" + import sys + import threading + + sys.set_lazy_imports("all") + + lazy import collections + lazy import functools + lazy import itertools + + num_threads = 12 + results = {} + errors = [] + barrier = threading.Barrier(num_threads) + + def stress_lazy_imports(idx): + try: + barrier.wait() + for _ in range(50): + _ = collections.OrderedDict + _ = functools.partial + _ = itertools.chain + _ = collections.defaultdict + _ = functools.lru_cache + _ = itertools.islice + results[idx] = ( + type(collections).__name__, + type(functools).__name__, + type(itertools).__name__, + ) + except Exception as e: + errors.append((idx, e)) + + threads = [ + threading.Thread(target=stress_lazy_imports, args=(i,)) + for i in range(num_threads) + ] + + for t in threads: + t.start() + for t in threads: + t.join() + + assert not errors, f"Errors: {errors}" + for idx, types_tuple in results.items(): + assert all(t == 'module' for t in types_tuple), f"Thread {idx}: {types_tuple}" + print("OK") + """) + + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_pyrepl/test_utils.py b/Lib/test/test_pyrepl/test_utils.py index 656a1e441e0..7b22353708c 100644 --- a/Lib/test/test_pyrepl/test_utils.py +++ b/Lib/test/test_pyrepl/test_utils.py @@ -89,10 +89,14 @@ def test_gen_colors_keyword_highlighting(self): ("obj.list", [(".", "op")]), ("obj.match", [(".", "op")]), ("b. \\\n format", [(".", "op")]), + ("lazy", []), + ("lazy()", [('(', 'op'), (')', 'op')]), # highlights ("set", [("set", "builtin")]), ("list", [("list", "builtin")]), (" \n dict", [("dict", "builtin")]), + (" lazy import", [("lazy", "soft_keyword"), ("import", "keyword")]), + ("lazy from cool_people import pablo", [('lazy', 'soft_keyword'), ('from', 'keyword'), ('import', 'keyword')]) ] for code, expected_highlights in cases: with self.subTest(code=code): diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index ade8f273a1e..a3714c93f7f 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -3461,6 +3461,119 @@ def test_ifexp_body_stmt_else_stmt(self): ]: self._check_error(f"x = {lhs_stmt} if 1 else {rhs_stmt}", msg) + +class LazyImportRestrictionTestCase(SyntaxErrorTestCase): + """Test syntax restrictions for lazy imports.""" + + def test_lazy_import_in_try_block(self): + """Test that lazy imports are not allowed inside try blocks.""" + self._check_error("""\ +try: + lazy import os +except: + pass +""", "lazy import not allowed inside try/except blocks") + + self._check_error("""\ +try: + lazy from sys import path +except ImportError: + pass +""", "lazy from ... import not allowed inside try/except blocks") + + def test_lazy_import_in_trystar_block(self): + """Test that lazy imports are not allowed inside try* blocks.""" + self._check_error("""\ +try: + lazy import json +except* Exception: + pass +""", "lazy import not allowed inside try/except blocks") + + self._check_error("""\ +try: + lazy from collections import defaultdict +except* ImportError: + pass +""", "lazy from ... import not allowed inside try/except blocks") + + def test_lazy_import_in_function(self): + """Test that lazy imports are not allowed inside functions.""" + self._check_error("""\ +def func(): + lazy import math +""", "lazy import not allowed inside functions") + + self._check_error("""\ +def func(): + lazy from datetime import datetime +""", "lazy from ... import not allowed inside functions") + + def test_lazy_import_in_async_function(self): + """Test that lazy imports are not allowed inside async functions.""" + self._check_error("""\ +async def async_func(): + lazy import asyncio +""", "lazy import not allowed inside functions") + + self._check_error("""\ +async def async_func(): + lazy from json import loads +""", "lazy from ... import not allowed inside functions") + + def test_lazy_import_in_class(self): + """Test that lazy imports are not allowed inside classes.""" + self._check_error("""\ +class MyClass: + lazy import typing +""", "lazy import not allowed inside classes") + + self._check_error("""\ +class MyClass: + lazy from abc import ABC +""", "lazy from ... import not allowed inside classes") + + def test_lazy_import_star_forbidden(self): + """Test that 'lazy from ... import *' is forbidden everywhere.""" + # At module level should also be forbidden + self._check_error("lazy from os import *", + "lazy from ... import \\* is not allowed") + + # Inside function should give lazy function error first + self._check_error("""\ +def func(): + lazy from sys import * +""", "lazy from ... import not allowed inside functions") + + def test_lazy_import_nested_scopes(self): + """Test lazy imports in nested scopes.""" + self._check_error("""\ +class Outer: + def method(self): + lazy import sys +""", "lazy import not allowed inside functions") + + self._check_error("""\ +def outer(): + class Inner: + lazy import json +""", "lazy import not allowed inside classes") + + self._check_error("""\ +def outer(): + def inner(): + lazy from collections import deque +""", "lazy from ... import not allowed inside functions") + + def test_lazy_import_valid_cases(self): + """Test that lazy imports work at module level.""" + # These should compile without errors + compile("lazy import os", "", "exec") + compile("lazy from sys import path", "", "exec") + compile("lazy import json as j", "", "exec") + compile("lazy from datetime import datetime as dt", "", "exec") + + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite()) return tests diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 04018e9603f..ff99002abfc 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -866,7 +866,8 @@ def test_sys_flags(self): "dont_write_bytecode", "no_user_site", "no_site", "ignore_environment", "verbose", "bytes_warning", "quiet", "hash_randomization", "isolated", "dev_mode", "utf8_mode", - "warn_default_encoding", "safe_path", "int_max_str_digits") + "warn_default_encoding", "safe_path", "int_max_str_digits", + "lazy_imports") for attr in attrs: self.assertHasAttr(sys.flags, attr) attr_type = bool if attr in ("dev_mode", "safe_path") else int @@ -1726,7 +1727,7 @@ def get_gen(): yield 1 md_gil = '?' else: md_gil = '' - check(unittest, size('PPPP?' + md_gil + 'NPPPPP')) + check(unittest, size('PPPPI?' + md_gil + 'NPPPPP')) # None check(None, size('')) # NotImplementedType diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 4595e7e5d3e..da386409282 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -55,7 +55,7 @@ def test_names(self): 'CoroutineType', 'EllipsisType', 'FrameType', 'FunctionType', 'FrameLocalsProxyType', 'GeneratorType', 'GenericAlias', 'GetSetDescriptorType', - 'LambdaType', 'MappingProxyType', 'MemberDescriptorType', + 'LambdaType', 'LazyImportType', 'MappingProxyType', 'MemberDescriptorType', 'MethodDescriptorType', 'MethodType', 'MethodWrapperType', 'ModuleType', 'NoneType', 'NotImplementedType', 'SimpleNamespace', 'TracebackType', 'UnionType', 'WrapperDescriptorType', diff --git a/Lib/types.py b/Lib/types.py index f96c75b46da..259dd57eb15 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -76,6 +76,10 @@ def _m(self): pass # CapsuleType cannot be accessed from pure Python, # so there is no fallback definition. + exec("lazy import sys as _lazy_sys", _lz := {}) + LazyImportType = type(_lz['_lazy_sys']) + del _lz + del sys, _f, _g, _C, _c, _ag, _cell_factory # Not for export diff --git a/Makefile.pre.in b/Makefile.pre.in index f3086ec1462..691a4cb2931 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -539,6 +539,7 @@ OBJECT_OBJS= \ Objects/funcobject.o \ Objects/interpolationobject.o \ Objects/iterobject.o \ + Objects/lazyimportobject.o \ Objects/listobject.o \ Objects/longobject.o \ Objects/dictobject.o \ @@ -1372,6 +1373,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_interpolation.h \ $(srcdir)/Include/internal/pycore_intrinsics.h \ $(srcdir)/Include/internal/pycore_jit.h \ + $(srcdir)/Include/internal/pycore_lazyimportobject.h \ $(srcdir)/Include/internal/pycore_list.h \ $(srcdir)/Include/internal/pycore_llist.h \ $(srcdir)/Include/internal/pycore_lock.h \ @@ -2652,6 +2654,8 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_import/data/package3 \ test/test_import/data/package4 \ test/test_import/data/unwritable \ + test/test_import/data/lazy_imports \ + test/test_import/data/lazy_imports/pkg \ test/test_importlib \ test/test_importlib/builtin \ test/test_importlib/extension \ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-06-15-46-32.gh-issue-142349.IdTuYL.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-06-15-46-32.gh-issue-142349.IdTuYL.rst new file mode 100644 index 00000000000..73cc167fd04 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-06-15-46-32.gh-issue-142349.IdTuYL.rst @@ -0,0 +1 @@ +Implement :pep:`810`. Patch by Pablo Galindo and Dino Viehland. diff --git a/Modules/_typesmodule.c b/Modules/_typesmodule.c index df6b4c93cb8..6c9e7a0a3ba 100644 --- a/Modules/_typesmodule.c +++ b/Modules/_typesmodule.c @@ -2,6 +2,7 @@ #include "Python.h" #include "pycore_descrobject.h" // _PyMethodWrapper_Type +#include "pycore_lazyimportobject.h" // PyLazyImport_Type #include "pycore_namespace.h" // _PyNamespace_Type #include "pycore_object.h" // _PyNone_Type, _PyNotImplemented_Type #include "pycore_unionobject.h" // _PyUnion_Type @@ -35,6 +36,7 @@ _types_exec(PyObject *m) EXPORT_STATIC_TYPE("GetSetDescriptorType", PyGetSetDescr_Type); // LambdaType is the same as FunctionType EXPORT_STATIC_TYPE("LambdaType", PyFunction_Type); + EXPORT_STATIC_TYPE("LazyImportType", PyLazyImport_Type); EXPORT_STATIC_TYPE("MappingProxyType", PyDictProxy_Type); EXPORT_STATIC_TYPE("MemberDescriptorType", PyMemberDescr_Type); EXPORT_STATIC_TYPE("MethodDescriptorType", PyMethodDescr_Type); diff --git a/Objects/dictobject.c b/Objects/dictobject.c index e0eef7b46df..e08f3acef1c 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -4694,6 +4694,14 @@ _PyDict_SizeOf_LockHeld(PyDictObject *mp) return (Py_ssize_t)res; } +void +_PyDict_ClearKeysVersion(PyObject *mp) +{ + ASSERT_DICT_LOCKED(mp); + + FT_ATOMIC_STORE_UINT32_RELAXED(((PyDictObject *)mp)->ma_keys->dk_version, 0); +} + Py_ssize_t _PyDict_SizeOf(PyDictObject *mp) { diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 9a43057b383..7b4b63e9a4a 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -2022,6 +2022,12 @@ static PyTypeObject _PyExc_ImportError = { }; PyObject *PyExc_ImportError = (PyObject *)&_PyExc_ImportError; +/* + * ImportCycleError extends ImportError + */ + +MiddlingExtendsException(PyExc_ImportError, ImportCycleError, ImportError, + "Import produces a cycle."); /* * ModuleNotFoundError extends ImportError */ @@ -4455,6 +4461,7 @@ static struct static_exception static_exceptions[] = { {&_PyExc_IncompleteInputError, "_IncompleteInputError"}, // base: SyntaxError(Exception) ITEM(IndexError), // base: LookupError(Exception) ITEM(KeyError), // base: LookupError(Exception) + ITEM(ImportCycleError), // base: ImportError(Exception) ITEM(ModuleNotFoundError), // base: ImportError(Exception) ITEM(NotImplementedError), // base: RuntimeError(Exception) ITEM(PythonFinalizationError), // base: RuntimeError(Exception) @@ -4650,4 +4657,3 @@ _PyException_AddNote(PyObject *exc, PyObject *note) Py_XDECREF(r); return res; } - diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c new file mode 100644 index 00000000000..d33970dbf19 --- /dev/null +++ b/Objects/lazyimportobject.c @@ -0,0 +1,180 @@ +/* Lazy object implementation */ + +#include "Python.h" +#include "pycore_ceval.h" +#include "pycore_frame.h" +#include "pycore_import.h" +#include "pycore_interpframe.h" +#include "pycore_lazyimportobject.h" +#include "pycore_modsupport.h" + +#define PyLazyImportObject_CAST(op) ((PyLazyImportObject *)(op)) + +PyObject * +_PyLazyImport_New(PyObject *builtins, PyObject *from, PyObject *attr) +{ + PyLazyImportObject *m; + if (!from || !PyUnicode_Check(from)) { + PyErr_BadArgument(); + return NULL; + } + if (attr == Py_None || attr == NULL) { + attr = NULL; + } + else if (!PyUnicode_Check(attr) && !PyTuple_Check(attr)) { + PyErr_SetString(PyExc_TypeError, + "lazy_import: fromlist must be None, a string, or a tuple"); + return NULL; + } + m = PyObject_GC_New(PyLazyImportObject, &PyLazyImport_Type); + if (m == NULL) { + return NULL; + } + m->lz_builtins = Py_XNewRef(builtins); + m->lz_from = Py_NewRef(from); + m->lz_attr = Py_XNewRef(attr); + + /* Capture frame information for the original import location */ + m->lz_code = NULL; + m->lz_instr_offset = -1; + + _PyInterpreterFrame *frame = _PyEval_GetFrame(); + if (frame != NULL) { + PyCodeObject *code = _PyFrame_GetCode(frame); + if (code != NULL) { + m->lz_code = (PyCodeObject *)Py_NewRef(code); + /* Calculate the instruction offset from the current frame */ + m->lz_instr_offset = _PyInterpreterFrame_LASTI(frame); + } + } + + _PyObject_GC_TRACK(m); + return (PyObject *)m; +} + +static int +lazy_import_traverse(PyObject *op, visitproc visit, void *arg) +{ + PyLazyImportObject *m = PyLazyImportObject_CAST(op); + Py_VISIT(m->lz_builtins); + Py_VISIT(m->lz_from); + Py_VISIT(m->lz_attr); + Py_VISIT(m->lz_code); + return 0; +} + +static int +lazy_import_clear(PyObject *op) +{ + PyLazyImportObject *m = PyLazyImportObject_CAST(op); + Py_CLEAR(m->lz_builtins); + Py_CLEAR(m->lz_from); + Py_CLEAR(m->lz_attr); + Py_CLEAR(m->lz_code); + return 0; +} + +static void +lazy_import_dealloc(PyObject *op) +{ + _PyObject_GC_UNTRACK(op); + (void)lazy_import_clear(op); + Py_TYPE(op)->tp_free(op); +} + +static PyObject * +lazy_import_name(PyLazyImportObject *m) +{ + if (m->lz_attr != NULL) { + if (PyUnicode_Check(m->lz_attr)) { + return PyUnicode_FromFormat("%U.%U", m->lz_from, m->lz_attr); + } + else { + return PyUnicode_FromFormat("%U...", m->lz_from); + } + } + return Py_NewRef(m->lz_from); +} + +static PyObject * +lazy_import_repr(PyObject *op) +{ + PyLazyImportObject *m = PyLazyImportObject_CAST(op); + PyObject *name = lazy_import_name(m); + if (name == NULL) { + return NULL; + } + PyObject *res = PyUnicode_FromFormat("<%T '%U'>", op, name); + Py_DECREF(name); + return res; +} + +static PyObject * +lazy_import_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyTypeObject *base_tp = &PyLazyImport_Type; + if ( + (type == base_tp || type->tp_init == base_tp->tp_init) + && !_PyArg_NoKeywords("lazy_import", kwds) + ) { + return NULL; + } + + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + if (!_PyArg_CheckPositional("lazy_import", nargs, 2, 3)) { + return NULL; + } + + PyObject *builtins = PyTuple_GET_ITEM(args, 0); + PyObject *from = PyTuple_GET_ITEM(args, 1); + PyObject *attr = nargs == 3 ? PyTuple_GET_ITEM(args, 2) : NULL; + return _PyLazyImport_New(builtins, from, attr); +} + +PyObject * +_PyLazyImport_GetName(PyObject *op) +{ + PyLazyImportObject *lazy_import = PyLazyImportObject_CAST(op); + assert(PyLazyImport_CheckExact(lazy_import)); + return lazy_import_name(lazy_import); +} + +static PyObject * +lazy_import_resolve(PyObject *self, PyObject *args) +{ + return _PyImport_LoadLazyImportTstate(PyThreadState_GET(), self); +} + +static PyMethodDef lazy_import_methods[] = { + { + "resolve", lazy_import_resolve, METH_NOARGS, + PyDoc_STR("resolves the lazy import and returns the actual object") + }, + {NULL, NULL} +}; + + +PyDoc_STRVAR(lazy_import_doc, +"lazy_import(builtins, name, fromlist=None, /)\n" +"--\n" +"\n" +"Represents a deferred import that will be resolved on first use.\n" +"\n" +"This type is used internally by the 'lazy import' statement.\n" +"Users should not typically create instances directly."); + +PyTypeObject PyLazyImport_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "lazy_import", + .tp_basicsize = sizeof(PyLazyImportObject), + .tp_dealloc = lazy_import_dealloc, + .tp_repr = lazy_import_repr, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, + .tp_doc = lazy_import_doc, + .tp_traverse = lazy_import_traverse, + .tp_clear = lazy_import_clear, + .tp_methods = lazy_import_methods, + .tp_alloc = PyType_GenericAlloc, + .tp_new = lazy_import_new, + .tp_free = PyObject_GC_Del, +}; diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 5a0b16ba572..8bd994fe91d 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -7,6 +7,7 @@ #include "pycore_fileutils.h" // _Py_wgetcwd #include "pycore_import.h" // _PyImport_GetNextModuleIndex() #include "pycore_interp.h" // PyInterpreterState.importlib +#include "pycore_lazyimportobject.h" // _PyLazyImportObject_Check() #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyModule_CreateInitialized() #include "pycore_moduleobject.h" // _PyModule_GetDefOrNull() @@ -177,6 +178,7 @@ new_module_notrack(PyTypeObject *mt) m->md_state = NULL; m->md_weaklist = NULL; m->md_name = NULL; + m->m_dict_version = 0; m->md_token_is_def = false; #ifdef Py_GIL_DISABLED m->md_requires_gil = true; @@ -1265,6 +1267,23 @@ _PyModule_IsPossiblyShadowing(PyObject *origin) return result; } +int +_PyModule_ReplaceLazyValue(PyObject *dict, PyObject *name, PyObject *value) +{ + // The adaptive interpreter uses the dictionary version to return the slot at + // a given index from the module. When replacing a value the version number doesn't + // change, so we need to atomically clear the version before replacing so that it + // doesn't return a lazy value. + int err; + Py_BEGIN_CRITICAL_SECTION(dict); + + _PyDict_ClearKeysVersion(dict); + err = _PyDict_SetItem_LockHeld((PyDictObject *)dict, name, value); + + Py_END_CRITICAL_SECTION(); + return err; +} + PyObject* _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) { @@ -1272,6 +1291,25 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) PyObject *attr, *mod_name, *getattr; attr = _PyObject_GenericGetAttrWithDict((PyObject *)m, name, NULL, suppress); if (attr) { + if (PyLazyImport_CheckExact(attr)) { + PyObject *new_value = _PyImport_LoadLazyImportTstate(PyThreadState_GET(), attr); + if (new_value == NULL) { + if (suppress && PyErr_ExceptionMatches(PyExc_ImportCycleError)) { + // ImportCycleError is raised when a lazy object tries to import itself. + // In this case, the error should not propagate to the caller and + // instead treated as if the attribute doesn't exist. + PyErr_Clear(); + } + Py_DECREF(attr); + return NULL; + } + + if (_PyModule_ReplaceLazyValue(m->md_dict, name, new_value) < 0) { + Py_CLEAR(new_value); + } + Py_DECREF(attr); + return new_value; + } return attr; } if (suppress == 1) { @@ -1470,7 +1508,12 @@ static PyObject * module_dir(PyObject *self, PyObject *args) { PyObject *result = NULL; - PyObject *dict = PyObject_GetAttr(self, &_Py_ID(__dict__)); + PyObject *dict; + if (PyModule_CheckExact(self)) { + dict = Py_NewRef(((PyModuleObject *)self)->md_dict); + } else { + dict = PyObject_GetAttr(self, &_Py_ID(__dict__)); + } if (dict != NULL) { if (PyDict_Check(dict)) { @@ -1498,7 +1541,7 @@ static PyMethodDef module_methods[] = { }; static PyObject * -module_get_dict(PyModuleObject *m) +module_load_dict(PyModuleObject *m) { PyObject *dict = PyObject_GetAttr((PyObject *)m, &_Py_ID(__dict__)); if (dict == NULL) { @@ -1517,7 +1560,7 @@ module_get_annotate(PyObject *self, void *Py_UNUSED(ignored)) { PyModuleObject *m = _PyModule_CAST(self); - PyObject *dict = module_get_dict(m); + PyObject *dict = module_load_dict(m); if (dict == NULL) { return NULL; } @@ -1542,7 +1585,7 @@ module_set_annotate(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) return -1; } - PyObject *dict = module_get_dict(m); + PyObject *dict = module_load_dict(m); if (dict == NULL) { return -1; } @@ -1572,7 +1615,7 @@ module_get_annotations(PyObject *self, void *Py_UNUSED(ignored)) { PyModuleObject *m = _PyModule_CAST(self); - PyObject *dict = module_get_dict(m); + PyObject *dict = module_load_dict(m); if (dict == NULL) { return NULL; } @@ -1644,7 +1687,7 @@ module_set_annotations(PyObject *self, PyObject *value, void *Py_UNUSED(ignored) { PyModuleObject *m = _PyModule_CAST(self); - PyObject *dict = module_get_dict(m); + PyObject *dict = module_load_dict(m); if (dict == NULL) { return -1; } @@ -1673,7 +1716,6 @@ module_set_annotations(PyObject *self, PyObject *value, void *Py_UNUSED(ignored) return ret; } - static PyGetSetDef module_getsets[] = { {"__annotations__", module_get_annotations, module_set_annotations}, {"__annotate__", module_get_annotate, module_set_annotate}, diff --git a/Objects/object.c b/Objects/object.c index fcea3503de8..21353fba44b 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2682,8 +2682,8 @@ _Py_SetImmortalUntracked(PyObject *op) } #ifdef Py_GIL_DISABLED op->ob_tid = _Py_UNOWNED_TID; - op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; - op->ob_ref_shared = 0; + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, _Py_IMMORTAL_REFCNT_LOCAL); + _Py_atomic_store_ssize_relaxed(&op->ob_ref_shared, 0); _Py_atomic_or_uint8(&op->ob_gc_bits, _PyGC_BITS_DEFERRED); #elif SIZEOF_VOID_P > 4 op->ob_flags = _Py_IMMORTAL_FLAGS; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index f737a885f19..183f1101358 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -1329,7 +1329,7 @@ PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar) else data = unicode + 1; _PyUnicode_LENGTH(unicode) = size; - _PyUnicode_HASH(unicode) = -1; + PyUnicode_SET_HASH((PyObject *)unicode, -1); _PyUnicode_STATE(unicode).interned = 0; _PyUnicode_STATE(unicode).kind = kind; _PyUnicode_STATE(unicode).compact = 1; @@ -13903,9 +13903,9 @@ unicode_subtype_new(PyTypeObject *type, PyObject *unicode) _PyUnicode_LENGTH(self) = length; #ifdef Py_DEBUG - _PyUnicode_HASH(self) = -1; + PyUnicode_SET_HASH((PyObject *)self, -1); #else - _PyUnicode_HASH(self) = _PyUnicode_HASH(unicode); + PyUnicode_SET_HASH((PyObject *)self, PyUnicode_HASH(unicode)); #endif _PyUnicode_STATE(self).interned = 0; _PyUnicode_STATE(self).kind = kind; @@ -13950,7 +13950,7 @@ unicode_subtype_new(PyTypeObject *type, PyObject *unicode) memcpy(data, PyUnicode_DATA(unicode), kind * (length + 1)); assert(_PyUnicode_CheckConsistency(self, 1)); #ifdef Py_DEBUG - _PyUnicode_HASH(self) = _PyUnicode_HASH(unicode); + PyUnicode_SET_HASH((PyObject *)self, PyUnicode_HASH(unicode)); #endif return self; diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 605861ad3fd..cb806459596 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -146,6 +146,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index c67fe53363e..6dcf0e87129 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -265,6 +265,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 85363949c23..4f32648e2a9 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -277,6 +277,7 @@ + @@ -547,6 +548,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 17999690990..1ed709f96ea 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -735,6 +735,9 @@ Include\internal + + Include\internal + Include\internal @@ -1247,6 +1250,9 @@ Objects + + Objects + Objects diff --git a/Parser/Python.asdl b/Parser/Python.asdl index 96f3914b029..6579642540e 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -45,8 +45,8 @@ module Python | TryStar(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) | Assert(expr test, expr? msg) - | Import(alias* names) - | ImportFrom(identifier? module, alias* names, int? level) + | Import(alias* names, int? is_lazy) + | ImportFrom(identifier? module, alias* names, int? level, int? is_lazy) | Global(identifier* names) | Nonlocal(identifier* names) diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index b7a5b9d5e30..165900456e3 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1943,9 +1943,13 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, stmt_ty _PyPegen_checked_future_import(Parser *p, identifier module, asdl_alias_seq * names, int level, - int lineno, int col_offset, int end_lineno, int end_col_offset, - PyArena *arena) { + expr_ty lazy_token, int lineno, int col_offset, int end_lineno, int end_col_offset, + PyArena *arena) { if (level == 0 && PyUnicode_CompareWithASCIIString(module, "__future__") == 0) { + if (lazy_token) { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(lazy_token, "lazy from __future__ import is not allowed"); + return NULL; + } for (Py_ssize_t i = 0; i < asdl_seq_LEN(names); i++) { alias_ty alias = asdl_seq_GET(names, i); if (PyUnicode_CompareWithASCIIString(alias->name, "barry_as_FLUFL") == 0) { @@ -1953,7 +1957,7 @@ _PyPegen_checked_future_import(Parser *p, identifier module, asdl_alias_seq * na } } } - return _PyAST_ImportFrom(module, names, level, lineno, col_offset, end_lineno, end_col_offset, arena); + return _PyAST_ImportFrom(module, names, level, lazy_token ? 1 : 0, lineno, col_offset, end_lineno, end_col_offset, arena); } asdl_stmt_seq* diff --git a/Parser/parser.c b/Parser/parser.c index 648b3702d8f..b031029f512 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -60,8 +60,8 @@ static KeywordToken *reserved_keywords[] = { {NULL, -1}, }, (KeywordToken[]) { - {"return", 522}, {"import", 643}, + {"return", 522}, {"assert", 634}, {"global", 530}, {"except", 686}, @@ -81,6 +81,7 @@ static KeywordToken *reserved_keywords[] = { static char *soft_keywords[] = { "_", "case", + "lazy", "match", "type", NULL, @@ -1549,9 +1550,9 @@ simple_stmts_rule(Parser *p) // simple_stmt: // | assignment // | &"type" type_alias +// | &('import' | 'from' | "lazy") import_stmt // | star_expressions // | &'return' return_stmt -// | &('import' | 'from') import_stmt // | &'raise' raise_stmt // | &'pass' pass_stmt // | &'del' del_stmt @@ -1626,6 +1627,27 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c%s simple_stmt[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "&\"type\" type_alias")); } + { // &('import' | 'from' | "lazy") import_stmt + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('import' | 'from' | \"lazy\") import_stmt")); + stmt_ty import_stmt_var; + if ( + _PyPegen_lookahead(1, _tmp_5_rule, p) + && + (import_stmt_var = import_stmt_rule(p)) // import_stmt + ) + { + D(fprintf(stderr, "%*c+ simple_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "&('import' | 'from' | \"lazy\") import_stmt")); + _res = import_stmt_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s simple_stmt[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "&('import' | 'from' | \"lazy\") import_stmt")); + } { // star_expressions if (p->error_indicator) { p->level--; @@ -1680,27 +1702,6 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c%s simple_stmt[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "&'return' return_stmt")); } - { // &('import' | 'from') import_stmt - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('import' | 'from') import_stmt")); - stmt_ty import_stmt_var; - if ( - _PyPegen_lookahead(1, _tmp_5_rule, p) - && - (import_stmt_var = import_stmt_rule(p)) // import_stmt - ) - { - D(fprintf(stderr, "%*c+ simple_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "&('import' | 'from') import_stmt")); - _res = import_stmt_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s simple_stmt[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "&('import' | 'from') import_stmt")); - } { // &'raise' raise_stmt if (p->error_indicator) { p->level--; @@ -3495,6 +3496,10 @@ import_stmt_rule(Parser *p) return NULL; } stmt_ty _res = NULL; + if (_PyPegen_is_memoized(p, import_stmt_type, &_res)) { + p->level--; + return _res; + } int _mark = p->mark; if (p->call_invalid_rules) { // invalid_import if (p->error_indicator) { @@ -3555,11 +3560,12 @@ import_stmt_rule(Parser *p) } _res = NULL; done: + _PyPegen_insert_memo(p, _mark, import_stmt_type, _res); p->level--; return _res; } -// import_name: 'import' dotted_as_names +// import_name: "lazy"? 'import' dotted_as_names static stmt_ty import_name_rule(Parser *p) { @@ -3581,21 +3587,24 @@ import_name_rule(Parser *p) UNUSED(_start_lineno); // Only used by EXTRA macro int _start_col_offset = p->tokens[_mark]->col_offset; UNUSED(_start_col_offset); // Only used by EXTRA macro - { // 'import' dotted_as_names + { // "lazy"? 'import' dotted_as_names if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> import_name[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'import' dotted_as_names")); + D(fprintf(stderr, "%*c> import_name[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "\"lazy\"? 'import' dotted_as_names")); Token * _keyword; asdl_alias_seq* a; + void *lazy; if ( + (lazy = _PyPegen_expect_soft_keyword(p, "lazy"), !p->error_indicator) // "lazy"? + && (_keyword = _PyPegen_expect_token(p, 643)) // token='import' && (a = dotted_as_names_rule(p)) // dotted_as_names ) { - D(fprintf(stderr, "%*c+ import_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'import' dotted_as_names")); + D(fprintf(stderr, "%*c+ import_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "\"lazy\"? 'import' dotted_as_names")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -3605,7 +3614,7 @@ import_name_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_Import ( a , EXTRA ); + _res = _PyAST_Import ( a , lazy ? 1 : 0 , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -3615,7 +3624,7 @@ import_name_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s import_name[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'import' dotted_as_names")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "\"lazy\"? 'import' dotted_as_names")); } _res = NULL; done: @@ -3624,8 +3633,8 @@ import_name_rule(Parser *p) } // import_from: -// | 'from' (('.' | '...'))* dotted_name 'import' import_from_targets -// | 'from' (('.' | '...'))+ 'import' import_from_targets +// | "lazy"? 'from' (('.' | '...'))* dotted_name 'import' import_from_targets +// | "lazy"? 'from' (('.' | '...'))+ 'import' import_from_targets static stmt_ty import_from_rule(Parser *p) { @@ -3647,18 +3656,21 @@ import_from_rule(Parser *p) UNUSED(_start_lineno); // Only used by EXTRA macro int _start_col_offset = p->tokens[_mark]->col_offset; UNUSED(_start_col_offset); // Only used by EXTRA macro - { // 'from' (('.' | '...'))* dotted_name 'import' import_from_targets + { // "lazy"? 'from' (('.' | '...'))* dotted_name 'import' import_from_targets if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> import_from[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'from' (('.' | '...'))* dotted_name 'import' import_from_targets")); + D(fprintf(stderr, "%*c> import_from[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "\"lazy\"? 'from' (('.' | '...'))* dotted_name 'import' import_from_targets")); Token * _keyword; Token * _keyword_1; asdl_seq * a; expr_ty b; asdl_alias_seq* c; + void *lazy; if ( + (lazy = _PyPegen_expect_soft_keyword(p, "lazy"), !p->error_indicator) // "lazy"? + && (_keyword = _PyPegen_expect_token(p, 642)) // token='from' && (a = _loop0_17_rule(p)) // (('.' | '...'))* @@ -3670,7 +3682,7 @@ import_from_rule(Parser *p) (c = import_from_targets_rule(p)) // import_from_targets ) { - D(fprintf(stderr, "%*c+ import_from[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'from' (('.' | '...'))* dotted_name 'import' import_from_targets")); + D(fprintf(stderr, "%*c+ import_from[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "\"lazy\"? 'from' (('.' | '...'))* dotted_name 'import' import_from_targets")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -3680,7 +3692,7 @@ import_from_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyPegen_checked_future_import ( p , b -> v . Name . id , c , _PyPegen_seq_count_dots ( a ) , EXTRA ); + _res = _PyPegen_checked_future_import ( p , b -> v . Name . id , c , _PyPegen_seq_count_dots ( a ) , lazy , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -3690,19 +3702,22 @@ import_from_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s import_from[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'from' (('.' | '...'))* dotted_name 'import' import_from_targets")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "\"lazy\"? 'from' (('.' | '...'))* dotted_name 'import' import_from_targets")); } - { // 'from' (('.' | '...'))+ 'import' import_from_targets + { // "lazy"? 'from' (('.' | '...'))+ 'import' import_from_targets if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> import_from[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'from' (('.' | '...'))+ 'import' import_from_targets")); + D(fprintf(stderr, "%*c> import_from[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "\"lazy\"? 'from' (('.' | '...'))+ 'import' import_from_targets")); Token * _keyword; Token * _keyword_1; asdl_seq * a; asdl_alias_seq* b; + void *lazy; if ( + (lazy = _PyPegen_expect_soft_keyword(p, "lazy"), !p->error_indicator) // "lazy"? + && (_keyword = _PyPegen_expect_token(p, 642)) // token='from' && (a = _loop1_18_rule(p)) // (('.' | '...'))+ @@ -3712,7 +3727,7 @@ import_from_rule(Parser *p) (b = import_from_targets_rule(p)) // import_from_targets ) { - D(fprintf(stderr, "%*c+ import_from[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'from' (('.' | '...'))+ 'import' import_from_targets")); + D(fprintf(stderr, "%*c+ import_from[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "\"lazy\"? 'from' (('.' | '...'))+ 'import' import_from_targets")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -3722,7 +3737,7 @@ import_from_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_ImportFrom ( NULL , b , _PyPegen_seq_count_dots ( a ) , EXTRA ); + _res = _PyAST_ImportFrom ( NULL , b , _PyPegen_seq_count_dots ( a ) , lazy ? 1 : 0 , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -3732,7 +3747,7 @@ import_from_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s import_from[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'from' (('.' | '...'))+ 'import' import_from_targets")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "\"lazy\"? 'from' (('.' | '...'))+ 'import' import_from_targets")); } _res = NULL; done: @@ -27974,7 +27989,7 @@ _gather_4_rule(Parser *p) return _res; } -// _tmp_5: 'import' | 'from' +// _tmp_5: 'import' | 'from' | "lazy" static void * _tmp_5_rule(Parser *p) { @@ -28025,6 +28040,25 @@ _tmp_5_rule(Parser *p) D(fprintf(stderr, "%*c%s _tmp_5[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'from'")); } + { // "lazy" + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_5[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "\"lazy\"")); + expr_ty _keyword; + if ( + (_keyword = _PyPegen_expect_soft_keyword(p, "lazy")) // soft_keyword='"lazy"' + ) + { + D(fprintf(stderr, "%*c+ _tmp_5[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "\"lazy\"")); + _res = _keyword; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_5[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "\"lazy\"")); + } _res = NULL; done: p->level--; diff --git a/Parser/pegen.h b/Parser/pegen.h index b8f887608b1..a0f10e7ae78 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -366,7 +366,7 @@ void *_PyPegen_arguments_parsing_error(Parser *, expr_ty); expr_ty _PyPegen_get_last_comprehension_item(comprehension_ty comprehension); void *_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq *comprehensions); stmt_ty _PyPegen_checked_future_import(Parser *p, identifier module, asdl_alias_seq *, - int , int, int , int , int , PyArena *); + int, expr_ty, int, int, int, int, PyArena *); asdl_stmt_seq* _PyPegen_register_stmts(Parser *p, asdl_stmt_seq* stmts); stmt_ty _PyPegen_register_stmt(Parser *p, stmt_ty s); diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index dbeedb7ffe0..f808544045e 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -2,7 +2,7 @@ unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0, 0,0,0,0,0,243,184,0,0,0,128,0,94,0,82,1, - 73,0,116,0,94,0,82,1,73,1,116,1,93,2,33,0, + 73,0,116,0,94,0,82,1,73,4,116,1,93,2,33,0, 82,2,52,1,0,0,0,0,0,0,31,0,93,2,33,0, 82,3,93,0,80,6,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,52,2,0,0,0,0,0,0, diff --git a/Python/Python-ast.c b/Python/Python-ast.c index aac24ed7d3c..6009399dd39 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -222,6 +222,7 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(state->id); Py_CLEAR(state->ifs); Py_CLEAR(state->is_async); + Py_CLEAR(state->is_lazy); Py_CLEAR(state->items); Py_CLEAR(state->iter); Py_CLEAR(state->key); @@ -327,6 +328,7 @@ static int init_identifiers(struct ast_state *state) if ((state->id = PyUnicode_InternFromString("id")) == NULL) return -1; if ((state->ifs = PyUnicode_InternFromString("ifs")) == NULL) return -1; if ((state->is_async = PyUnicode_InternFromString("is_async")) == NULL) return -1; + if ((state->is_lazy = PyUnicode_InternFromString("is_lazy")) == NULL) return -1; if ((state->items = PyUnicode_InternFromString("items")) == NULL) return -1; if ((state->iter = PyUnicode_InternFromString("iter")) == NULL) return -1; if ((state->key = PyUnicode_InternFromString("key")) == NULL) return -1; @@ -527,11 +529,13 @@ static const char * const Assert_fields[]={ }; static const char * const Import_fields[]={ "names", + "is_lazy", }; static const char * const ImportFrom_fields[]={ "module", "names", "level", + "is_lazy", }; static const char * const Global_fields[]={ "names", @@ -2254,6 +2258,21 @@ add_ast_annotations(struct ast_state *state) return 0; } } + { + PyObject *type = (PyObject *)&PyLong_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Import_annotations); + return 0; + } + cond = PyDict_SetItemString(Import_annotations, "is_lazy", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Import_annotations); + return 0; + } + } cond = PyObject_SetAttrString(state->Import_type, "_field_types", Import_annotations) == 0; if (!cond) { @@ -2315,6 +2334,22 @@ add_ast_annotations(struct ast_state *state) return 0; } } + { + PyObject *type = (PyObject *)&PyLong_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + cond = PyDict_SetItemString(ImportFrom_annotations, "is_lazy", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + } cond = PyObject_SetAttrString(state->ImportFrom_type, "_field_types", ImportFrom_annotations) == 0; if (!cond) { @@ -6218,8 +6253,8 @@ init_types(void *arg) " | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)\n" " | TryStar(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)\n" " | Assert(expr test, expr? msg)\n" - " | Import(alias* names)\n" - " | ImportFrom(identifier? module, alias* names, int? level)\n" + " | Import(alias* names, int? is_lazy)\n" + " | ImportFrom(identifier? module, alias* names, int? level, int? is_lazy)\n" " | Global(identifier* names)\n" " | Nonlocal(identifier* names)\n" " | Expr(expr value)\n" @@ -6348,17 +6383,21 @@ init_types(void *arg) if (PyObject_SetAttr(state->Assert_type, state->msg, Py_None) == -1) return -1; state->Import_type = make_type(state, "Import", state->stmt_type, - Import_fields, 1, - "Import(alias* names)"); + Import_fields, 2, + "Import(alias* names, int? is_lazy)"); if (!state->Import_type) return -1; + if (PyObject_SetAttr(state->Import_type, state->is_lazy, Py_None) == -1) + return -1; state->ImportFrom_type = make_type(state, "ImportFrom", state->stmt_type, - ImportFrom_fields, 3, - "ImportFrom(identifier? module, alias* names, int? level)"); + ImportFrom_fields, 4, + "ImportFrom(identifier? module, alias* names, int? level, int? is_lazy)"); if (!state->ImportFrom_type) return -1; if (PyObject_SetAttr(state->ImportFrom_type, state->module, Py_None) == -1) return -1; if (PyObject_SetAttr(state->ImportFrom_type, state->level, Py_None) == -1) return -1; + if (PyObject_SetAttr(state->ImportFrom_type, state->is_lazy, Py_None) == -1) + return -1; state->Global_type = make_type(state, "Global", state->stmt_type, Global_fields, 1, "Global(identifier* names)"); @@ -7598,8 +7637,8 @@ _PyAST_Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, int } stmt_ty -_PyAST_Import(asdl_alias_seq * names, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena) +_PyAST_Import(asdl_alias_seq * names, int is_lazy, int lineno, int col_offset, + int end_lineno, int end_col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)_PyArena_Malloc(arena, sizeof(*p)); @@ -7607,6 +7646,7 @@ _PyAST_Import(asdl_alias_seq * names, int lineno, int col_offset, int return NULL; p->kind = Import_kind; p->v.Import.names = names; + p->v.Import.is_lazy = is_lazy; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -7616,8 +7656,8 @@ _PyAST_Import(asdl_alias_seq * names, int lineno, int col_offset, int stmt_ty _PyAST_ImportFrom(identifier module, asdl_alias_seq * names, int level, int - lineno, int col_offset, int end_lineno, int end_col_offset, - PyArena *arena) + is_lazy, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)_PyArena_Malloc(arena, sizeof(*p)); @@ -7627,6 +7667,7 @@ _PyAST_ImportFrom(identifier module, asdl_alias_seq * names, int level, int p->v.ImportFrom.module = module; p->v.ImportFrom.names = names; p->v.ImportFrom.level = level; + p->v.ImportFrom.is_lazy = is_lazy; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -9465,6 +9506,11 @@ ast2obj_stmt(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->names, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_int(state, o->v.Import.is_lazy); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->is_lazy, value) == -1) + goto failed; + Py_DECREF(value); break; case ImportFrom_kind: tp = (PyTypeObject *)state->ImportFrom_type; @@ -9486,6 +9532,11 @@ ast2obj_stmt(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->level, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_int(state, o->v.ImportFrom.is_lazy); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->is_lazy, value) == -1) + goto failed; + Py_DECREF(value); break; case Global_kind: tp = (PyTypeObject *)state->Global_type; @@ -13481,6 +13532,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* } if (isinstance) { asdl_alias_seq* names; + int is_lazy; if (PyObject_GetOptionalAttr(obj, state->names, &tmp) < 0) { return -1; @@ -13520,7 +13572,24 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* } Py_CLEAR(tmp); } - *out = _PyAST_Import(names, lineno, col_offset, end_lineno, + if (PyObject_GetOptionalAttr(obj, state->is_lazy, &tmp) < 0) { + return -1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + is_lazy = 0; + } + else { + int res; + if (_Py_EnterRecursiveCall(" while traversing 'Import' node")) { + goto failed; + } + res = obj2ast_int(state, tmp, &is_lazy, arena); + _Py_LeaveRecursiveCall(); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = _PyAST_Import(names, is_lazy, lineno, col_offset, end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; @@ -13534,6 +13603,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* identifier module; asdl_alias_seq* names; int level; + int is_lazy; if (PyObject_GetOptionalAttr(obj, state->module, &tmp) < 0) { return -1; @@ -13607,8 +13677,25 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = _PyAST_ImportFrom(module, names, level, lineno, col_offset, - end_lineno, end_col_offset, arena); + if (PyObject_GetOptionalAttr(obj, state->is_lazy, &tmp) < 0) { + return -1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + is_lazy = 0; + } + else { + int res; + if (_Py_EnterRecursiveCall(" while traversing 'ImportFrom' node")) { + goto failed; + } + res = obj2ast_int(state, tmp, &is_lazy, arena); + _Py_LeaveRecursiveCall(); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = _PyAST_ImportFrom(module, names, level, is_lazy, lineno, + col_offset, end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index c2d780ac9b9..7959726f25a 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -9,6 +9,7 @@ #include "pycore_fileutils.h" // _PyFile_Flush #include "pycore_floatobject.h" // _PyFloat_ExactDealloc() #include "pycore_interp.h" // _PyInterpreterState_GetConfig() +#include "pycore_import.h" // _PyImport_LazyImportModuleLevelObject () #include "pycore_long.h" // _PyLong_CompactValue #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _Py_AddToAllObjects() @@ -287,6 +288,59 @@ builtin___import___impl(PyObject *module, PyObject *name, PyObject *globals, } +/*[clinic input] +__lazy_import__ as builtin___lazy_import__ + + name: object + globals: object(c_default="NULL") = None + locals: object(c_default="NULL") = None + fromlist: object(c_default="NULL") = () + level: int = 0 + +Lazily imports a module. + +Returns either the module to be imported or a imp.lazy_module object which +indicates the module to be lazily imported. +[clinic start generated code]*/ + +static PyObject * +builtin___lazy_import___impl(PyObject *module, PyObject *name, + PyObject *globals, PyObject *locals, + PyObject *fromlist, int level) +/*[clinic end generated code: output=300f1771094b9e8c input=9394874f340b2948]*/ +{ + PyObject *builtins; + PyThreadState *tstate = PyThreadState_GET(); + if (globals == NULL) { + globals = PyEval_GetGlobals(); + } + if (locals == NULL) { + locals = globals; + } + + if (PyDict_GetItemRef(globals, &_Py_ID(__builtins__), &builtins) < 0) { + return NULL; + } + if (builtins == NULL) { + PyErr_SetString(PyExc_ValueError, "unable to get builtins for lazy import"); + return NULL; + } + if (PyModule_Check(builtins)) { + PyObject *builtins_dict = Py_XNewRef(PyModule_GetDict(builtins)); + if (builtins_dict == NULL) { + Py_DECREF(builtins); + PyErr_SetString(PyExc_AttributeError, "builtins module has no dict"); + return NULL; + } + Py_SETREF(builtins, builtins_dict); + } + + PyObject *res = _PyImport_LazyImportModuleLevelObject(tstate, name, builtins, + globals, locals, fromlist, level); + Py_DECREF(builtins); + return res; +} + /*[clinic input] abs as builtin_abs @@ -3362,6 +3416,7 @@ static PyMethodDef builtin_methods[] = { {"__build_class__", _PyCFunction_CAST(builtin___build_class__), METH_FASTCALL | METH_KEYWORDS, build_class_doc}, BUILTIN___IMPORT___METHODDEF + BUILTIN___LAZY_IMPORT___METHODDEF BUILTIN_ABS_METHODDEF BUILTIN_ALL_METHODDEF BUILTIN_ANY_METHODDEF diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4ba255d28bd..10b636844d8 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -11,12 +11,15 @@ #include "pycore_audit.h" // _PySys_Audit() #include "pycore_backoff.h" #include "pycore_cell.h" // PyCell_GetRef() +#include "pycore_ceval.h" // _PyEval_LazyImportName(), _PyEval_LazyImportFrom() #include "pycore_code.h" #include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS #include "pycore_function.h" +#include "pycore_import.h" // _PyImport_LoadLazyImportTstate() #include "pycore_instruments.h" #include "pycore_interpolation.h" // _PyInterpolation_Build() #include "pycore_intrinsics.h" +#include "pycore_lazyimportobject.h" // PyLazyImport_CheckExact() #include "pycore_long.h" // _PyLong_ExactDealloc(), _PyLong_GetZero() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_object.h" // _PyObject_GC_TRACK() @@ -1757,6 +1760,12 @@ dummy_func( } ERROR_NO_POP(); } + + if (PyLazyImport_CheckExact(v_o)) { + PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + Py_SETREF(v_o, l_v); + ERROR_IF(v_o == NULL); + } } else { /* Slow-path if globals or builtins is not a dict */ @@ -1774,6 +1783,11 @@ dummy_func( ERROR_IF(true); } } + if (PyLazyImport_CheckExact(v_o)) { + PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + Py_SETREF(v_o, l_v); + ERROR_IF(v_o == NULL); + } } } v = PyStackRef_FromPyObjectSteal(v_o); @@ -1783,6 +1797,22 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *v_o = _PyEval_LoadName(tstate, frame, name); ERROR_IF(v_o == NULL); + if (PyLazyImport_CheckExact(v_o)) { + PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + // cannot early-decref v_o as it may cause a side-effect on l_v + if (l_v == NULL) { + Py_DECREF(v_o); + ERROR_IF(true); + } + int err = _PyModule_ReplaceLazyValue(GLOBALS(), name, l_v); + if (err < 0) { + Py_DECREF(v_o); + Py_DECREF(l_v); + ERROR_IF(true); + } + Py_SETREF(v_o, l_v); + } + v = PyStackRef_FromPyObjectSteal(v_o); } @@ -1808,7 +1838,21 @@ dummy_func( op(_LOAD_GLOBAL, ( -- res[1])) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); + ERROR_IF(PyStackRef_IsNull(*res)); + + PyObject *res_o = PyStackRef_AsPyObjectBorrow(*res); + if (PyLazyImport_CheckExact(res_o)) { + PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, res_o); + PyStackRef_CLOSE(res[0]); + ERROR_IF(l_v == NULL); + int err = _PyModule_ReplaceLazyValue(GLOBALS(), name, l_v); + if (err < 0) { + Py_DECREF(l_v); + ERROR_IF(true); + } + *res = PyStackRef_FromPyObjectSteal(l_v); + } } op(_PUSH_NULL_CONDITIONAL, ( -- null[oparg & 1])) { @@ -2898,11 +2942,21 @@ dummy_func( b = res ? PyStackRef_True : PyStackRef_False; } - inst(IMPORT_NAME, (level, fromlist -- res)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *res_o = _PyEval_ImportName(tstate, frame, name, - PyStackRef_AsPyObjectBorrow(fromlist), - PyStackRef_AsPyObjectBorrow(level)); + inst(IMPORT_NAME, (level, fromlist -- res)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyObject *res_o; + if (!(oparg & 0x02)) { + res_o = _PyEval_LazyImportName(tstate, BUILTINS(), GLOBALS(), LOCALS(), name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level), + oparg & 0x01); + + } + else { + res_o = _PyEval_ImportName(tstate, BUILTINS(), GLOBALS(), LOCALS(), name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + } DECREF_INPUTS(); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); @@ -2910,7 +2964,14 @@ dummy_func( inst(IMPORT_FROM, (from -- from, res)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + PyObject *res_o; + if (PyLazyImport_CheckExact(PyStackRef_AsPyObjectBorrow(from))) { + res_o = _PyEval_LazyImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + } + else { + res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + } + ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } diff --git a/Python/ceval.c b/Python/ceval.c index a1d54bd058b..7d9101861fa 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -22,6 +22,7 @@ #include "pycore_interpolation.h" // _PyInterpolation_Build() #include "pycore_intrinsics.h" #include "pycore_jit.h" +#include "pycore_lazyimportobject.h" #include "pycore_list.h" // _PyList_GetItemRef() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // PyModuleObject @@ -3444,11 +3445,11 @@ _PyEval_SliceIndexNotNone(PyObject *v, Py_ssize_t *pi) } PyObject * -_PyEval_ImportName(PyThreadState *tstate, _PyInterpreterFrame *frame, +_PyEval_ImportName(PyThreadState *tstate, PyObject *builtins, PyObject *globals, PyObject *locals, PyObject *name, PyObject *fromlist, PyObject *level) { PyObject *import_func; - if (PyMapping_GetOptionalItem(frame->f_builtins, &_Py_ID(__import__), &import_func) < 0) { + if (PyMapping_GetOptionalItem(builtins, &_Py_ID(__import__), &import_func) < 0) { return NULL; } if (import_func == NULL) { @@ -3456,29 +3457,136 @@ _PyEval_ImportName(PyThreadState *tstate, _PyInterpreterFrame *frame, return NULL; } - PyObject *locals = frame->f_locals; + PyObject *res = _PyEval_ImportNameWithImport(tstate, import_func, globals, locals, name, fromlist, level); + Py_DECREF(import_func); + return res; +} + +PyObject * +_PyEval_ImportNameWithImport(PyThreadState *tstate, PyObject *import_func, PyObject *globals, PyObject *locals, + PyObject *name, PyObject *fromlist, PyObject *level) +{ if (locals == NULL) { locals = Py_None; } /* Fast path for not overloaded __import__. */ if (_PyImport_IsDefaultImportFunc(tstate->interp, import_func)) { - Py_DECREF(import_func); int ilevel = PyLong_AsInt(level); if (ilevel == -1 && _PyErr_Occurred(tstate)) { return NULL; } return PyImport_ImportModuleLevelObject( name, - frame->f_globals, + globals, locals, fromlist, ilevel); } - PyObject* args[5] = {name, frame->f_globals, locals, fromlist, level}; + PyObject *args[5] = {name, globals, locals, fromlist, level}; PyObject *res = PyObject_Vectorcall(import_func, args, 5, NULL); - Py_DECREF(import_func); + return res; +} + +static int +check_lazy_import_compatibility(PyThreadState *tstate, PyObject *globals, + PyObject *name, PyObject *level) +{ + // Check if this module should be imported lazily due to + // the compatibility mode support via __lazy_modules__. + PyObject *lazy_modules = NULL; + PyObject *abs_name = NULL; + int res = -1; + + if (globals != NULL && + PyMapping_GetOptionalItem(globals, &_Py_ID(__lazy_modules__), &lazy_modules) < 0) + { + return -1; + } + if (lazy_modules == NULL) { + assert(!PyErr_Occurred()); + return 0; + } + + int ilevel = PyLong_AsInt(level); + if (ilevel == -1 && _PyErr_Occurred(tstate)) { + goto error; + } + + abs_name = _PyImport_GetAbsName(tstate, name, globals, ilevel); + if (abs_name == NULL) { + goto error; + } + + res = PySequence_Contains(lazy_modules, abs_name); +error: + Py_XDECREF(abs_name); + Py_XDECREF(lazy_modules); + return res; +} + +PyObject * +_PyEval_LazyImportName(PyThreadState *tstate, PyObject *builtins, PyObject *globals, + PyObject *locals, PyObject *name, PyObject *fromlist, PyObject *level, + int lazy) +{ + PyObject *res = NULL; + // Check if global policy overrides the local syntax + switch (PyImport_GetLazyImportsMode()) { + case PyImport_LAZY_NONE: + lazy = 0; + break; + case PyImport_LAZY_ALL: + lazy = 1; + break; + case PyImport_LAZY_NORMAL: + break; + } + + if (!lazy) { + // see if __lazy_imports__ forces this to be lazy + lazy = check_lazy_import_compatibility(tstate, globals, name, level); + if (lazy < 0) { + return NULL; + } + } + + if (!lazy) { + // Not a lazy import or lazy imports are disabled, fallback to the regular import + return _PyEval_ImportName(tstate, builtins, globals, locals, name, fromlist, level); + } + + PyObject *lazy_import_func; + if (PyMapping_GetOptionalItem(builtins, &_Py_ID(__lazy_import__), &lazy_import_func) < 0) { + goto error; + } + if (lazy_import_func == NULL) { + assert(!PyErr_Occurred()); + _PyErr_SetString(tstate, PyExc_ImportError, "__lazy_import__ not found"); + goto error; + } + + if (locals == NULL) { + locals = Py_None; + } + + if (_PyImport_IsDefaultLazyImportFunc(tstate->interp, lazy_import_func)) { + int ilevel = PyLong_AsInt(level); + if (ilevel == -1 && PyErr_Occurred()) { + goto error; + } + + res = _PyImport_LazyImportModuleLevelObject( + tstate, name, builtins, globals, locals, fromlist, ilevel + ); + goto error; + } + + PyObject *args[6] = {name, globals, locals, fromlist, level, builtins}; + res = PyObject_Vectorcall(lazy_import_func, args, 6, NULL); +error: + Py_XDECREF(lazy_import_func); return res; } @@ -3650,6 +3758,62 @@ _PyEval_ImportFrom(PyThreadState *tstate, PyObject *v, PyObject *name) return NULL; } +PyObject * +_PyEval_LazyImportFrom(PyThreadState *tstate, PyObject *v, PyObject *name) +{ + assert(PyLazyImport_CheckExact(v)); + assert(name); + assert(PyUnicode_Check(name)); + PyObject *ret; + PyLazyImportObject *d = (PyLazyImportObject *)v; + PyObject *mod = PyImport_GetModule(d->lz_from); + if (mod != NULL) { + // Check if the module already has the attribute, if so, resolve it eagerly. + if (PyModule_Check(mod)) { + PyObject *mod_dict = PyModule_GetDict(mod); + if (mod_dict != NULL) { + if (PyDict_GetItemRef(mod_dict, name, &ret) < 0) { + Py_DECREF(mod); + return NULL; + } + if (ret != NULL) { + Py_DECREF(mod); + return ret; + } + } + } + Py_DECREF(mod); + } + + if (d->lz_attr != NULL) { + if (PyUnicode_Check(d->lz_attr)) { + PyObject *from = PyUnicode_FromFormat("%U.%U", d->lz_from, d->lz_attr); + if (from == NULL) { + return NULL; + } + ret = _PyLazyImport_New(d->lz_builtins, from, name); + Py_DECREF(from); + return ret; + } + } + else { + Py_ssize_t dot = PyUnicode_FindChar( + d->lz_from, '.', 0, PyUnicode_GET_LENGTH(d->lz_from), 1 + ); + if (dot >= 0) { + PyObject *from = PyUnicode_Substring(d->lz_from, 0, dot); + if (from == NULL) { + return NULL; + } + ret = _PyLazyImport_New(d->lz_builtins, from, name); + Py_DECREF(from); + return ret; + } + } + ret = _PyLazyImport_New(d->lz_builtins, d->lz_from, name); + return ret; +} + #define CANNOT_CATCH_MSG "catching classes that do not inherit from "\ "BaseException is not allowed" diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index f08e5847abe..c8c141f863d 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -113,6 +113,101 @@ exit: return return_value; } +PyDoc_STRVAR(builtin___lazy_import____doc__, +"__lazy_import__($module, /, name, globals=None, locals=None,\n" +" fromlist=(), level=0)\n" +"--\n" +"\n" +"Lazily imports a module.\n" +"\n" +"Returns either the module to be imported or a imp.lazy_module object which\n" +"indicates the module to be lazily imported."); + +#define BUILTIN___LAZY_IMPORT___METHODDEF \ + {"__lazy_import__", _PyCFunction_CAST(builtin___lazy_import__), METH_FASTCALL|METH_KEYWORDS, builtin___lazy_import____doc__}, + +static PyObject * +builtin___lazy_import___impl(PyObject *module, PyObject *name, + PyObject *globals, PyObject *locals, + PyObject *fromlist, int level); + +static PyObject * +builtin___lazy_import__(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 5 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(name), &_Py_ID(globals), &_Py_ID(locals), &_Py_ID(fromlist), &_Py_ID(level), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"name", "globals", "locals", "fromlist", "level", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "__lazy_import__", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[5]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *name; + PyObject *globals = NULL; + PyObject *locals = NULL; + PyObject *fromlist = NULL; + int level = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 5, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + name = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1]) { + globals = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[2]) { + locals = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[3]) { + fromlist = args[3]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + level = PyLong_AsInt(args[4]); + if (level == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_pos: + return_value = builtin___lazy_import___impl(module, name, globals, locals, fromlist, level); + +exit: + return return_value; +} + PyDoc_STRVAR(builtin_abs__doc__, "abs($module, number, /)\n" "--\n" @@ -1285,4 +1380,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=06500bcc9a341e68 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1c3327da8885bb8e input=a9049054013a1b77]*/ diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h index 9bbb13f7566..7c7a9d68c04 100644 --- a/Python/clinic/import.c.h +++ b/Python/clinic/import.c.h @@ -622,6 +622,41 @@ exit: return return_value; } +PyDoc_STRVAR(_imp__set_lazy_attributes__doc__, +"_set_lazy_attributes($module, child_module, name, /)\n" +"--\n" +"\n" +"Sets attributes to lazy submodules on the module, as side effects."); + +#define _IMP__SET_LAZY_ATTRIBUTES_METHODDEF \ + {"_set_lazy_attributes", _PyCFunction_CAST(_imp__set_lazy_attributes), METH_FASTCALL, _imp__set_lazy_attributes__doc__}, + +static PyObject * +_imp__set_lazy_attributes_impl(PyObject *module, PyObject *child_module, + PyObject *name); + +static PyObject * +_imp__set_lazy_attributes(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *child_module; + PyObject *name; + + if (!_PyArg_CheckPositional("_set_lazy_attributes", nargs, 2, 2)) { + goto exit; + } + child_module = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("_set_lazy_attributes", "argument 2", "str", args[1]); + goto exit; + } + name = args[1]; + return_value = _imp__set_lazy_attributes_impl(module, child_module, name); + +exit: + return return_value; +} + #ifndef _IMP_CREATE_DYNAMIC_METHODDEF #define _IMP_CREATE_DYNAMIC_METHODDEF #endif /* !defined(_IMP_CREATE_DYNAMIC_METHODDEF) */ @@ -629,4 +664,4 @@ exit: #ifndef _IMP_EXEC_DYNAMIC_METHODDEF #define _IMP_EXEC_DYNAMIC_METHODDEF #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */ -/*[clinic end generated code: output=24f597d6b0f3feed input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0fe31ade5e29e8d6 input=a9049054013a1b77]*/ diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 4c4a86de2f9..5d30faeaa17 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -1820,6 +1820,203 @@ exit: return return_value; } +PyDoc_STRVAR(sys_set_lazy_imports_filter__doc__, +"set_lazy_imports_filter($module, /, filter)\n" +"--\n" +"\n" +"Set the lazy imports filter callback.\n" +"\n" +"The filter is a callable which disables lazy imports when they\n" +"would otherwise be enabled. Returns True if the import is still enabled\n" +"or False to disable it. The callable is called with:\n" +"\n" +"(importing_module_name, imported_module_name, [fromlist])\n" +"\n" +"Pass None to clear the filter."); + +#define SYS_SET_LAZY_IMPORTS_FILTER_METHODDEF \ + {"set_lazy_imports_filter", _PyCFunction_CAST(sys_set_lazy_imports_filter), METH_FASTCALL|METH_KEYWORDS, sys_set_lazy_imports_filter__doc__}, + +static PyObject * +sys_set_lazy_imports_filter_impl(PyObject *module, PyObject *filter); + +static PyObject * +sys_set_lazy_imports_filter(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(filter), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"filter", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "set_lazy_imports_filter", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *filter; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + filter = args[0]; + return_value = sys_set_lazy_imports_filter_impl(module, filter); + +exit: + return return_value; +} + +PyDoc_STRVAR(sys_get_lazy_imports_filter__doc__, +"get_lazy_imports_filter($module, /)\n" +"--\n" +"\n" +"Get the current lazy imports filter callback.\n" +"\n" +"Returns the filter callable or None if no filter is set."); + +#define SYS_GET_LAZY_IMPORTS_FILTER_METHODDEF \ + {"get_lazy_imports_filter", (PyCFunction)sys_get_lazy_imports_filter, METH_NOARGS, sys_get_lazy_imports_filter__doc__}, + +static PyObject * +sys_get_lazy_imports_filter_impl(PyObject *module); + +static PyObject * +sys_get_lazy_imports_filter(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return sys_get_lazy_imports_filter_impl(module); +} + +PyDoc_STRVAR(sys_set_lazy_imports__doc__, +"set_lazy_imports($module, /, mode)\n" +"--\n" +"\n" +"Sets the global lazy imports mode.\n" +"\n" +"The mode parameter must be one of the following strings:\n" +"- \"all\": All top-level imports become potentially lazy\n" +"- \"none\": All lazy imports are suppressed (even explicitly marked ones)\n" +"- \"normal\": Only explicitly marked imports (with \'lazy\' keyword) are lazy\n" +"\n" +"In addition to the mode, lazy imports can be controlled via the filter\n" +"provided to sys.set_lazy_imports_filter"); + +#define SYS_SET_LAZY_IMPORTS_METHODDEF \ + {"set_lazy_imports", _PyCFunction_CAST(sys_set_lazy_imports), METH_FASTCALL|METH_KEYWORDS, sys_set_lazy_imports__doc__}, + +static PyObject * +sys_set_lazy_imports_impl(PyObject *module, PyObject *mode); + +static PyObject * +sys_set_lazy_imports(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(mode), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"mode", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "set_lazy_imports", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *mode; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + mode = args[0]; + return_value = sys_set_lazy_imports_impl(module, mode); + +exit: + return return_value; +} + +PyDoc_STRVAR(sys_get_lazy_imports__doc__, +"get_lazy_imports($module, /)\n" +"--\n" +"\n" +"Gets the global lazy imports mode.\n" +"\n" +"Returns \"all\" if all top level imports are potentially lazy.\n" +"Returns \"none\" if all explicitly marked lazy imports are suppressed.\n" +"Returns \"normal\" if only explicitly marked imports are lazy."); + +#define SYS_GET_LAZY_IMPORTS_METHODDEF \ + {"get_lazy_imports", (PyCFunction)sys_get_lazy_imports, METH_NOARGS, sys_get_lazy_imports__doc__}, + +static PyObject * +sys_get_lazy_imports_impl(PyObject *module); + +static PyObject * +sys_get_lazy_imports(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return sys_get_lazy_imports_impl(module); +} + +PyDoc_STRVAR(sys_get_lazy_modules__doc__, +"get_lazy_modules($module, /)\n" +"--\n" +"\n" +"Gets the set of module names that have been lazily imported.\n" +"\n" +"Returns a set of fully-qualified module names that have been lazily\n" +"imported at some point (primarily for diagnostics and introspection).\n" +"Note that modules are removed from this set when they are reified\n" +"(actually loaded)."); + +#define SYS_GET_LAZY_MODULES_METHODDEF \ + {"get_lazy_modules", (PyCFunction)sys_get_lazy_modules, METH_NOARGS, sys_get_lazy_modules__doc__}, + +static PyObject * +sys_get_lazy_modules_impl(PyObject *module); + +static PyObject * +sys_get_lazy_modules(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return sys_get_lazy_modules_impl(module); +} + PyDoc_STRVAR(_jit_is_available__doc__, "is_available($module, /)\n" "--\n" @@ -1947,4 +2144,4 @@ exit: #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=5f7d84c5bf00d557 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a64004ec17cceb34 input=a9049054013a1b77]*/ diff --git a/Python/codegen.c b/Python/codegen.c index c4109fcaa48..1c5781d189a 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -351,8 +351,8 @@ codegen_addop_o(compiler *c, location loc, #define LOAD_ZERO_SUPER_METHOD -4 static int -codegen_addop_name(compiler *c, location loc, - int opcode, PyObject *dict, PyObject *o) +codegen_addop_name_custom(compiler *c, location loc, + int opcode, PyObject *dict, PyObject *o, int shift, int low) { PyObject *mangled = _PyCompile_MaybeMangle(c, o); if (!mangled) { @@ -363,40 +363,51 @@ codegen_addop_name(compiler *c, location loc, if (arg < 0) { return ERROR; } + ADDOP_I(c, loc, opcode, (arg << shift) | low); + return SUCCESS; +} + +static int +codegen_addop_name(compiler *c, location loc, + int opcode, PyObject *dict, PyObject *o) +{ + int shift = 0, low = 0; if (opcode == LOAD_ATTR) { - arg <<= 1; + shift = 1; } if (opcode == LOAD_METHOD) { opcode = LOAD_ATTR; - arg <<= 1; - arg |= 1; + shift = 1; + low = 1; } if (opcode == LOAD_SUPER_ATTR) { - arg <<= 2; - arg |= 2; + shift = 2; + low = 2; } if (opcode == LOAD_SUPER_METHOD) { opcode = LOAD_SUPER_ATTR; - arg <<= 2; - arg |= 3; + shift = 2; + low = 3; } if (opcode == LOAD_ZERO_SUPER_ATTR) { opcode = LOAD_SUPER_ATTR; - arg <<= 2; + shift = 2; } if (opcode == LOAD_ZERO_SUPER_METHOD) { opcode = LOAD_SUPER_ATTR; - arg <<= 2; - arg |= 1; + shift = 2; + low = 1; } - ADDOP_I(c, loc, opcode, arg); - return SUCCESS; + return codegen_addop_name_custom(c, loc, opcode, dict, o, shift, low); } #define ADDOP_NAME(C, LOC, OP, O, TYPE) \ RETURN_IF_ERROR(codegen_addop_name((C), (LOC), (OP), METADATA(C)->u_ ## TYPE, (O))) -static int +#define ADDOP_NAME_CUSTOM(C, LOC, OP, O, TYPE, SHIFT, LOW) \ + RETURN_IF_ERROR(codegen_addop_name_custom((C), (LOC), (OP), METADATA(C)->u_ ## TYPE, (O), SHIFT, LOW)) + + static int codegen_addop_j(instr_sequence *seq, location loc, int opcode, jump_target_label target) { @@ -2841,6 +2852,16 @@ codegen_import_as(compiler *c, location loc, return codegen_nameop(c, loc, asname, Store); } +static int +codegen_validate_lazy_import(compiler *c, location loc) +{ + if (_PyCompile_ScopeType(c) != COMPILE_SCOPE_MODULE) { + return _PyCompile_Error(c, loc, "lazy imports only allowed in module scope"); + } + + return SUCCESS; +} + static int codegen_import(compiler *c, stmt_ty s) { @@ -2861,7 +2882,17 @@ codegen_import(compiler *c, stmt_ty s) ADDOP_LOAD_CONST(c, loc, zero); ADDOP_LOAD_CONST(c, loc, Py_None); - ADDOP_NAME(c, loc, IMPORT_NAME, alias->name, names); + if (s->v.Import.is_lazy) { + RETURN_IF_ERROR(codegen_validate_lazy_import(c, loc)); + ADDOP_NAME_CUSTOM(c, loc, IMPORT_NAME, alias->name, names, 2, 1); + } else { + if (_PyCompile_InExceptionHandler(c) || _PyCompile_ScopeType(c) != COMPILE_SCOPE_MODULE) { + // force eager import in try/except block + ADDOP_NAME_CUSTOM(c, loc, IMPORT_NAME, alias->name, names, 2, 2); + } else { + ADDOP_NAME_CUSTOM(c, loc, IMPORT_NAME, alias->name, names, 2, 0); + } + } if (alias->asname) { r = codegen_import_as(c, loc, alias->name, alias->asname); @@ -2907,13 +2938,29 @@ codegen_from_import(compiler *c, stmt_ty s) ADDOP_LOAD_CONST_NEW(c, LOC(s), names); + identifier from = &_Py_STR(empty); if (s->v.ImportFrom.module) { - ADDOP_NAME(c, LOC(s), IMPORT_NAME, s->v.ImportFrom.module, names); + from = s->v.ImportFrom.module; } - else { - _Py_DECLARE_STR(empty, ""); - ADDOP_NAME(c, LOC(s), IMPORT_NAME, &_Py_STR(empty), names); + if (s->v.ImportFrom.is_lazy) { + alias_ty alias = (alias_ty)asdl_seq_GET(s->v.ImportFrom.names, 0); + if (PyUnicode_READ_CHAR(alias->name, 0) == '*') { + return _PyCompile_Error(c, LOC(s), "cannot lazy import *"); + } + RETURN_IF_ERROR(codegen_validate_lazy_import(c, LOC(s))); + ADDOP_NAME_CUSTOM(c, LOC(s), IMPORT_NAME, from, names, 2, 1); + } else { + alias_ty alias = (alias_ty)asdl_seq_GET(s->v.ImportFrom.names, 0); + if (_PyCompile_InExceptionHandler(c) || + _PyCompile_ScopeType(c) != COMPILE_SCOPE_MODULE || + PyUnicode_READ_CHAR(alias->name, 0) == '*') { + // forced non-lazy import due to try/except or import * + ADDOP_NAME_CUSTOM(c, LOC(s), IMPORT_NAME, from, names, 2, 2); + } else { + ADDOP_NAME_CUSTOM(c, LOC(s), IMPORT_NAME, from, names, 2, 0); + } } + for (Py_ssize_t i = 0; i < n; i++) { alias_ty alias = (alias_ty)asdl_seq_GET(s->v.ImportFrom.names, i); identifier store_name; diff --git a/Python/compile.c b/Python/compile.c index 6951c98500d..d3cff59d9d1 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -800,6 +800,26 @@ _PyCompile_TopFBlock(compiler *c) return &c->u->u_fblock[c->u->u_nfblocks - 1]; } +bool +_PyCompile_InExceptionHandler(compiler *c) +{ + for (Py_ssize_t i = 0; i < c->u->u_nfblocks; i++) { + fblockinfo *block = &c->u->u_fblock[i]; + switch (block->fb_type) { + case COMPILE_FBLOCK_TRY_EXCEPT: + case COMPILE_FBLOCK_FINALLY_TRY: + case COMPILE_FBLOCK_FINALLY_END: + case COMPILE_FBLOCK_EXCEPTION_HANDLER: + case COMPILE_FBLOCK_EXCEPTION_GROUP_HANDLER: + case COMPILE_FBLOCK_HANDLER_CLEANUP: + return true; + default: + break; + } + } + return false; +} + void _PyCompile_DeferredAnnotations(compiler *c, PyObject **deferred_annotations, diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 7273a87681b..c8943adff05 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2458,6 +2458,30 @@ if (v_o == NULL) { JUMP_TO_ERROR(); } + if (PyLazyImport_CheckExact(v_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (l_v == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyModule_ReplaceLazyValue(GLOBALS(), name, l_v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(v_o); + Py_DECREF(l_v); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_SETREF(v_o, l_v); + stack_pointer = _PyFrame_GetStackPointer(frame); + } v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; @@ -2476,6 +2500,26 @@ if (PyStackRef_IsNull(*res)) { JUMP_TO_ERROR(); } + PyObject *res_o = PyStackRef_AsPyObjectBorrow(*res); + if (PyLazyImport_CheckExact(res_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, res_o); + PyStackRef_CLOSE(res[0]); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (l_v == NULL) { + JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyModule_ReplaceLazyValue(GLOBALS(), name, l_v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(l_v); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_ERROR(); + } + *res = PyStackRef_FromPyObjectSteal(l_v); + } stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -4073,11 +4117,24 @@ oparg = CURRENT_OPARG(); fromlist = stack_pointer[-1]; level = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyObject *res_o; + if (!(oparg & 0x02)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = _PyEval_LazyImportName(tstate, BUILTINS(), GLOBALS(), LOCALS(), name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level), + oparg & 0x01); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = _PyEval_ImportName(tstate, BUILTINS(), GLOBALS(), LOCALS(), name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportName(tstate, frame, name, - PyStackRef_AsPyObjectBorrow(fromlist), - PyStackRef_AsPyObjectBorrow(level)); _PyStackRef tmp = fromlist; fromlist = PyStackRef_NULL; stack_pointer[-1] = fromlist; @@ -4105,9 +4162,17 @@ oparg = CURRENT_OPARG(); from = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + PyObject *res_o; + if (PyLazyImport_CheckExact(PyStackRef_AsPyObjectBorrow(from))) { + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = _PyEval_LazyImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } if (res_o == NULL) { JUMP_TO_ERROR(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 68d73cccec4..e0a5206506e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5857,9 +5857,17 @@ _PyStackRef res; from = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + PyObject *res_o; + if (PyLazyImport_CheckExact(PyStackRef_AsPyObjectBorrow(from))) { + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = _PyEval_LazyImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -5883,11 +5891,24 @@ _PyStackRef res; fromlist = stack_pointer[-1]; level = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyObject *res_o; + if (!(oparg & 0x02)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = _PyEval_LazyImportName(tstate, BUILTINS(), GLOBALS(), LOCALS(), name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level), + oparg & 0x01); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = _PyEval_ImportName(tstate, BUILTINS(), GLOBALS(), LOCALS(), name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportName(tstate, frame, name, - PyStackRef_AsPyObjectBorrow(fromlist), - PyStackRef_AsPyObjectBorrow(level)); _PyStackRef tmp = fromlist; fromlist = PyStackRef_NULL; stack_pointer[-1] = fromlist; @@ -8733,6 +8754,15 @@ } JUMP_TO_LABEL(error); } + if (PyLazyImport_CheckExact(v_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + Py_SETREF(v_o, l_v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) { + JUMP_TO_LABEL(error); + } + } } else { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -8757,6 +8787,15 @@ JUMP_TO_LABEL(error); } } + if (PyLazyImport_CheckExact(v_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + Py_SETREF(v_o, l_v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) { + JUMP_TO_LABEL(error); + } + } } } v = PyStackRef_FromPyObjectSteal(v_o); @@ -8809,6 +8848,26 @@ if (PyStackRef_IsNull(*res)) { JUMP_TO_LABEL(error); } + PyObject *res_o = PyStackRef_AsPyObjectBorrow(*res); + if (PyLazyImport_CheckExact(res_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, res_o); + PyStackRef_CLOSE(res[0]); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (l_v == NULL) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyModule_ReplaceLazyValue(GLOBALS(), name, l_v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(l_v); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + *res = PyStackRef_FromPyObjectSteal(l_v); + } } // _PUSH_NULL_CONDITIONAL { @@ -9009,6 +9068,30 @@ if (v_o == NULL) { JUMP_TO_LABEL(error); } + if (PyLazyImport_CheckExact(v_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (l_v == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyModule_ReplaceLazyValue(GLOBALS(), name, l_v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(v_o); + Py_DECREF(l_v); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_SETREF(v_o, l_v); + stack_pointer = _PyFrame_GetStackPointer(frame); + } v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; diff --git a/Python/import.c b/Python/import.c index e91c95b40d9..90b2e544de0 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3,20 +3,27 @@ #include "Python.h" #include "pycore_audit.h" // _PySys_Audit() #include "pycore_ceval.h" +#include "pycore_dict.h" // _PyDict_Contains_KnownHash() #include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() #include "pycore_hashtable.h" // _Py_hashtable_new_full() #include "pycore_import.h" // _PyImport_BootstrapImp() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // struct _import_runtime_state +#include "pycore_long.h" // _PyLong_GetZero +#include "pycore_lazyimportobject.h" +#include "pycore_traceback.h" +#include "pycore_interpframe.h" #include "pycore_magic_number.h" // PYC_MAGIC_NUMBER_TOKEN #include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_namespace.h" // _PyNamespace_Type #include "pycore_object.h" // _Py_SetImmortal() +#include "pycore_pyatomic_ft_wrappers.h" #include "pycore_pyerrors.h" // _PyErr_SetString() #include "pycore_pyhash.h" // _Py_KeyedHash() #include "pycore_pylifecycle.h" #include "pycore_pymem.h" // _PyMem_DefaultRawFree() #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_sysmodule.h" // _PySys_ClearAttrString() #include "pycore_time.h" // _PyTime_AsMicroseconds() #include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() @@ -98,12 +105,21 @@ static struct _inittab *inittab_copy = NULL; #define IMPORT_FUNC(interp) \ (interp)->imports.import_func +#define LAZY_IMPORT_FUNC(interp) \ + (interp)->imports.lazy_import_func + #define IMPORT_LOCK(interp) \ (interp)->imports.lock #define FIND_AND_LOAD(interp) \ (interp)->imports.find_and_load +#define LAZY_IMPORTS_MODE(interp) \ + (interp)->imports.lazy_imports_mode + +#define LAZY_IMPORTS_FILTER(interp) \ + (interp)->imports.lazy_imports_filter + #define _IMPORT_TIME_HEADER(interp) \ do { \ if (FIND_AND_LOAD((interp)).header) { \ @@ -3492,6 +3508,12 @@ _PyImport_InitDefaultImportFunc(PyInterpreterState *interp) return -1; } IMPORT_FUNC(interp) = import_func; + + // Get the __lazy_import__ function + if (PyDict_GetItemStringRef(interp->builtins, "__lazy_import__", &import_func) <= 0) { + return -1; + } + LAZY_IMPORT_FUNC(interp) = import_func; return 0; } @@ -3501,6 +3523,11 @@ _PyImport_IsDefaultImportFunc(PyInterpreterState *interp, PyObject *func) return func == IMPORT_FUNC(interp); } +int +_PyImport_IsDefaultLazyImportFunc(PyInterpreterState *interp, PyObject *func) +{ + return func == LAZY_IMPORT_FUNC(interp); +} /* Import a module, either built-in, frozen, or external, and return its module object WITH INCREMENTED REFERENCE COUNT */ @@ -3764,6 +3791,226 @@ resolve_name(PyThreadState *tstate, PyObject *name, PyObject *globals, int level return NULL; } +PyObject * +_PyImport_ResolveName(PyThreadState *tstate, PyObject *name, PyObject *globals, int level) +{ + return resolve_name(tstate, name, globals, level); +} + +PyObject * +_PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) +{ + PyObject *obj = NULL; + PyObject *fromlist = NULL; + PyObject *import_func = NULL; + assert(lazy_import != NULL); + assert(PyLazyImport_CheckExact(lazy_import)); + + PyLazyImportObject *lz = (PyLazyImportObject *)lazy_import; + PyInterpreterState *interp = tstate->interp; + + // Acquire the global import lock to serialize reification + _PyImport_AcquireLock(interp); + + // Check if we are already importing this module, if so, then we want to return an error + // that indicates we've hit a cycle which will indicate the value isn't yet available. + PyObject *importing = interp->imports.lazy_importing_modules; + if (importing == NULL) { + importing = interp->imports.lazy_importing_modules = PySet_New(NULL); + if (importing == NULL) { + _PyImport_ReleaseLock(interp); + return NULL; + } + } + + assert(PyAnySet_CheckExact(importing)); + int is_loading = _PySet_Contains((PySetObject *)importing, lazy_import); + if (is_loading < 0) { + _PyImport_ReleaseLock(interp); + return NULL; + } else if (is_loading == 1) { + PyObject *name = _PyLazyImport_GetName(lazy_import); + if (name == NULL) { + _PyImport_ReleaseLock(interp); + return NULL; + } + PyObject *errmsg = PyUnicode_FromFormat( + "cannot import name %R (most likely due to a circular import)", + name); + if (errmsg == NULL) { + Py_DECREF(name); + _PyImport_ReleaseLock(interp); + return NULL; + } + PyErr_SetImportErrorSubclass(PyExc_ImportCycleError, errmsg, lz->lz_from, NULL); + Py_DECREF(errmsg); + Py_DECREF(name); + _PyImport_ReleaseLock(interp); + return NULL; + } else if (PySet_Add(importing, lazy_import) < 0) { + _PyImport_ReleaseLock(interp); + goto error; + } + + Py_ssize_t dot = -1; + int full = 0; + if (lz->lz_attr != NULL) { + full = 1; + } + if (!full) { + dot = PyUnicode_FindChar(lz->lz_from, '.', 0, PyUnicode_GET_LENGTH(lz->lz_from), 1); + } + if (dot < 0) { + full = 1; + } + + if (lz->lz_attr != NULL) { + if (PyUnicode_Check(lz->lz_attr)) { + fromlist = PyTuple_New(1); + if (fromlist == NULL) { + goto error; + } + Py_INCREF(lz->lz_attr); + PyTuple_SET_ITEM(fromlist, 0, lz->lz_attr); + } else { + Py_INCREF(lz->lz_attr); + fromlist = lz->lz_attr; + } + } + + PyObject *globals = PyEval_GetGlobals(); + + if (PyMapping_GetOptionalItem(lz->lz_builtins, &_Py_ID(__import__), &import_func) < 0) { + goto error; + } + if (import_func == NULL) { + PyErr_SetString(PyExc_ImportError, "__import__ not found"); + goto error; + } + if (full) { + obj = _PyEval_ImportNameWithImport(tstate, + import_func, + globals, + globals, + lz->lz_from, + fromlist, + _PyLong_GetZero()); + } else { + PyObject *name = PyUnicode_Substring(lz->lz_from, 0, dot); + if (name == NULL) { + goto error; + } + obj = _PyEval_ImportNameWithImport(tstate, + import_func, + globals, + globals, + name, + fromlist, + _PyLong_GetZero()); + Py_DECREF(name); + } + if (obj == NULL) { + goto error; + } + + if (lz->lz_attr != NULL && PyUnicode_Check(lz->lz_attr)) { + PyObject *from = obj; + obj = _PyEval_ImportFrom(tstate, from, lz->lz_attr); + Py_DECREF(from); + if (obj == NULL) { + goto error; + } + } + + assert(!PyLazyImport_CheckExact(obj)); + + goto ok; + +error: + Py_CLEAR(obj); + + /* If an error occurred and we have frame information, add it to the exception */ + if (PyErr_Occurred() && lz->lz_code != NULL && lz->lz_instr_offset >= 0) { + /* Get the current exception - this already has the full traceback from the access point */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + + /* Get import name - this can fail and set an exception */ + PyObject *import_name = _PyLazyImport_GetName(lazy_import); + if (!import_name) { + /* Failed to get import name, just restore original exception */ + _PyErr_SetRaisedException(tstate, exc); + goto ok; + } + + /* Resolve line number from instruction offset on demand */ + int lineno = PyCode_Addr2Line((PyCodeObject *)lz->lz_code, lz->lz_instr_offset*2); + + /* Get strings - these can return NULL on encoding errors */ + const char *filename_str = PyUnicode_AsUTF8(lz->lz_code->co_filename); + if (!filename_str) { + /* Unicode conversion failed - clear error and restore original exception */ + PyErr_Clear(); + Py_DECREF(import_name); + _PyErr_SetRaisedException(tstate, exc); + goto ok; + } + + const char *funcname_str = PyUnicode_AsUTF8(lz->lz_code->co_name); + if (!funcname_str) { + /* Unicode conversion failed - clear error and restore original exception */ + PyErr_Clear(); + Py_DECREF(import_name); + _PyErr_SetRaisedException(tstate, exc); + goto ok; + } + + /* Create a cause exception showing where the lazy import was declared */ + PyObject *msg = PyUnicode_FromFormat( + "deferred import of '%U' raised an exception during resolution", + import_name + ); + Py_DECREF(import_name); /* Done with import_name regardless of what happens next */ + + if (!msg) { + /* Failed to create message - restore original exception */ + _PyErr_SetRaisedException(tstate, exc); + goto ok; + } + + PyObject *cause_exc = PyObject_CallOneArg(PyExc_ImportError, msg); + Py_DECREF(msg); /* Done with msg */ + + if (!cause_exc) { + /* Failed to create exception - restore original */ + _PyErr_SetRaisedException(tstate, exc); + goto ok; + } + + /* Add traceback entry for the lazy import declaration */ + _PyErr_SetRaisedException(tstate, cause_exc); + _PyTraceback_Add(funcname_str, filename_str, lineno); + PyObject *cause_with_tb = _PyErr_GetRaisedException(tstate); + + /* Set the cause on the original exception */ + PyException_SetCause(exc, cause_with_tb); /* Steals ref to cause_with_tb */ + + /* Restore the original exception with its full traceback */ + _PyErr_SetRaisedException(tstate, exc); + } + +ok: + if (PySet_Discard(importing, lazy_import) < 0) { + Py_CLEAR(obj); + } + + // Release the global import lock + _PyImport_ReleaseLock(interp); + + Py_XDECREF(fromlist); + Py_XDECREF(import_func); + return obj; +} + static PyObject * import_find_and_load(PyThreadState *tstate, PyObject *abs_name) { @@ -3818,6 +4065,26 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) #undef accumulated } +static PyObject * +get_abs_name(PyThreadState *tstate, PyObject *name, PyObject *globals, int level) +{ + if (level > 0) { + return resolve_name(tstate, name, globals, level); + } + if (PyUnicode_GET_LENGTH(name) == 0) { + _PyErr_SetString(tstate, PyExc_ValueError, "Empty module name"); + return NULL; + } + return Py_NewRef(name); +} + +PyObject * +_PyImport_GetAbsName(PyThreadState *tstate, PyObject *name, PyObject *globals, int level) +{ + return get_abs_name(tstate, name, globals, level); +} + + PyObject * PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, @@ -3828,6 +4095,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, PyObject *final_mod = NULL; PyObject *mod = NULL; PyObject *package = NULL; + PyObject *lazy_modules = NULL; PyInterpreterState *interp = tstate->interp; int has_from; @@ -3849,17 +4117,9 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, goto error; } - if (level > 0) { - abs_name = resolve_name(tstate, name, globals, level); - if (abs_name == NULL) - goto error; - } - else { /* level == 0 */ - if (PyUnicode_GET_LENGTH(name) == 0) { - _PyErr_SetString(tstate, PyExc_ValueError, "Empty module name"); - goto error; - } - abs_name = Py_NewRef(name); + abs_name = get_abs_name(tstate, name, globals, level); + if (abs_name == NULL) { + goto error; } mod = import_get_module(tstate, abs_name); @@ -3954,6 +4214,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } error: + Py_XDECREF(lazy_modules); Py_XDECREF(abs_name); Py_XDECREF(mod); Py_XDECREF(package); @@ -3963,6 +4224,226 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, return final_mod; } +static PyObject * +get_mod_dict(PyObject *module) +{ + if (PyModule_Check(module)) { + return Py_NewRef(_PyModule_GetDict(module)); + } + + return PyObject_GetAttr(module, &_Py_ID(__dict__)); +} + +static int +register_lazy_on_parent(PyThreadState *tstate, PyObject *name, PyObject *builtins) +{ + int ret = -1; + PyObject *parent = NULL; + PyObject *child = NULL; + PyObject *parent_module = NULL; + PyObject *parent_dict = NULL; + + // Acquire import lock to safely initialize lazy_modules if needed + // This prevents a race where multiple threads could create different dicts + PyInterpreterState *interp = tstate->interp; + _PyImport_AcquireLock(interp); + + PyObject *lazy_modules = interp->imports.lazy_modules; + if (lazy_modules == NULL) { + lazy_modules = interp->imports.lazy_modules = PyDict_New(); + if (lazy_modules == NULL) { + _PyImport_ReleaseLock(interp); + return -1; + } + } + + // Release the lock - we only needed it for initialization. + // The dict operations below use PyDict_SetDefaultRef which uses + // critical sections for thread-safety in free-threaded builds. + _PyImport_ReleaseLock(interp); + + Py_INCREF(name); + while (true) { + Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, PyUnicode_GET_LENGTH(name), -1); + if (dot < 0) { + ret = 0; + goto done; + } + parent = PyUnicode_Substring(name, 0, dot); + /* If `parent` is NULL then this has hit the end of the import, no more + * "parent.child" in the import name. The entire import will be resolved + * lazily. */ + if (parent == NULL) { + goto done; + } + Py_XDECREF(child); + child = PyUnicode_Substring(name, dot + 1, PyUnicode_GET_LENGTH(name)); + if (child == NULL) { + goto done; + } + + /* Add the lazy import for the child to the parent */ + Py_XSETREF(parent_module, PyImport_GetModule(parent)); + if (parent_module == NULL) { + if (PyErr_Occurred()) { + goto done; + } + + // Record the child to be added when the parent is imported. + // Use PyDict_SetDefaultRef to atomically get-or-create the set, + // avoiding TOCTOU races in free-threaded builds. + PyObject *empty_set = PySet_New(NULL); + if (empty_set == NULL) { + goto done; + } + PyObject *lazy_submodules; + int setdefault_result = PyDict_SetDefaultRef(lazy_modules, parent, empty_set, &lazy_submodules); + Py_DECREF(empty_set); + if (setdefault_result < 0) { + goto done; + } + assert(PyAnySet_CheckExact(lazy_submodules)); + if (PySet_Add(lazy_submodules, child) < 0) { + Py_DECREF(lazy_submodules); + goto done; + } + Py_DECREF(lazy_submodules); + } else { + Py_XSETREF(parent_dict, get_mod_dict(parent_module)); + if (parent_dict == NULL) { + goto done; + } + if (PyDict_CheckExact(parent_dict)) { + int contains = PyDict_Contains(parent_dict, child); + if (contains < 0) { + goto done; + } + if (!contains) { + PyObject *lazy_module_attr = _PyLazyImport_New(builtins, parent, child); + if (lazy_module_attr == NULL) { + goto done; + } + if (PyDict_SetItem(parent_dict, child, lazy_module_attr) < 0) { + Py_DECREF(lazy_module_attr); + goto done; + } + Py_DECREF(lazy_module_attr); + } + } + } + + Py_SETREF(name, parent); + parent = NULL; + } + +done: + Py_XDECREF(parent_dict); + Py_XDECREF(parent_module); + Py_XDECREF(child); + Py_XDECREF(parent); + Py_XDECREF(name); + return ret; +} + +PyObject * +_PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, + PyObject *name, PyObject *builtins, + PyObject *globals, PyObject *locals, + PyObject *fromlist, int level) +{ + PyObject *abs_name = get_abs_name(tstate, name, globals, level); + if (abs_name == NULL) { + return NULL; + } + + PyInterpreterState *interp = tstate->interp; + _PyInterpreterFrame *frame = _PyEval_GetFrame(); + if (frame == NULL || frame->f_globals != frame->f_locals) { + Py_DECREF(abs_name); + PyErr_SetString(PyExc_SyntaxError, + "'lazy import' is only allowed at module level"); + return NULL; + } + + // Check if the filter disables the lazy import. + // We must hold a reference to the filter while calling it to prevent + // use-after-free if another thread replaces it via PyImport_SetLazyImportsFilter. + PyObject *filter = FT_ATOMIC_LOAD_PTR_RELAXED(LAZY_IMPORTS_FILTER(interp)); + Py_XINCREF(filter); + if (filter != NULL) { + PyObject *modname; + if (PyDict_GetItemRef(globals, &_Py_ID(__name__), &modname) < 0) { + Py_DECREF(filter); + Py_DECREF(abs_name); + return NULL; + } + if (modname == NULL) { + assert(!PyErr_Occurred()); + modname = Py_NewRef(Py_None); + } + PyObject *args[] = {modname, name, fromlist}; + PyObject *res = PyObject_Vectorcall( + filter, + args, + 3, + NULL + ); + + Py_DECREF(modname); + Py_DECREF(filter); + + if (res == NULL) { + Py_DECREF(abs_name); + return NULL; + } + + int is_true = PyObject_IsTrue(res); + Py_DECREF(res); + + if (is_true < 0) { + Py_DECREF(abs_name); + return NULL; + } + if (!is_true) { + Py_DECREF(abs_name); + return PyImport_ImportModuleLevelObject( + name, globals, locals, fromlist, level + ); + } + } + + // here, 'filter' is either NULL or is equivalent to a borrowed reference + PyObject *res = _PyLazyImport_New(builtins, abs_name, fromlist); + if (res == NULL) { + Py_DECREF(abs_name); + return NULL; + } + if (register_lazy_on_parent(tstate, abs_name, builtins) < 0) { + Py_DECREF(res); + Py_DECREF(abs_name); + return NULL; + } + + // Add the module name to sys.lazy_modules set (PEP 810). + // We must hold a reference to the set while using it to prevent + // use-after-free if another thread clears it during interpreter shutdown. + PyObject *lazy_modules_set = FT_ATOMIC_LOAD_PTR_RELAXED(interp->imports.lazy_modules_set); + Py_XINCREF(lazy_modules_set); + if (lazy_modules_set != NULL) { + assert(PyAnySet_CheckExact(lazy_modules_set)); + if (PySet_Add(lazy_modules_set, abs_name) < 0) { + Py_DECREF(lazy_modules_set); + Py_DECREF(res); + Py_DECREF(abs_name); + return NULL; + } + Py_DECREF(lazy_modules_set); + } + + Py_DECREF(abs_name); + return res; +} + PyObject * PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) @@ -4169,6 +4650,11 @@ _PyImport_ClearCore(PyInterpreterState *interp) Py_CLEAR(MODULES_BY_INDEX(interp)); Py_CLEAR(IMPORTLIB(interp)); Py_CLEAR(IMPORT_FUNC(interp)); + Py_CLEAR(LAZY_IMPORT_FUNC(interp)); + Py_CLEAR(interp->imports.lazy_modules); + Py_CLEAR(interp->imports.lazy_modules_set); + Py_CLEAR(interp->imports.lazy_importing_modules); + Py_CLEAR(interp->imports.lazy_imports_filter); } void @@ -4304,6 +4790,56 @@ PyImport_ImportModuleAttrString(const char *modname, const char *attrname) return result; } +int +PyImport_SetLazyImportsFilter(PyObject *filter) +{ + if (filter == Py_None) { + filter = NULL; + } + if (filter != NULL && !PyCallable_Check(filter)) { + PyErr_SetString(PyExc_ValueError, "filter provided but is not callable"); + return -1; + } + + PyInterpreterState *interp = _PyInterpreterState_GET(); +#ifdef Py_GIL_DISABLED + // Exchange the filter atomically. Use deferred DECREF to prevent + // use-after-free: another thread may have loaded the old filter + // and be about to INCREF it. + PyObject *old = _Py_atomic_exchange_ptr(&LAZY_IMPORTS_FILTER(interp), Py_XNewRef(filter)); + _PyObject_XDecRefDelayed(old); +#else + Py_XSETREF(LAZY_IMPORTS_FILTER(interp), Py_XNewRef(filter)); +#endif + return 0; +} + +/* Return a strong reference to the current lazy imports filter + * or NULL if none exists. This function always succeeds. + */ +PyObject * +PyImport_GetLazyImportsFilter(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + return Py_XNewRef(FT_ATOMIC_LOAD_PTR_RELAXED(LAZY_IMPORTS_FILTER(interp))); +} + +int +PyImport_SetLazyImportsMode(PyImport_LazyImportsMode mode) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + FT_ATOMIC_STORE_INT_RELAXED(LAZY_IMPORTS_MODE(interp), mode); + return 0; +} + +/* Checks if lazy imports is globally enabled or disabled. Return 1 when globally + * forced on, 0 when globally forced off, or -1 when */ +PyImport_LazyImportsMode +PyImport_GetLazyImportsMode(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + return FT_ATOMIC_LOAD_INT_RELAXED(LAZY_IMPORTS_MODE(interp)); +} /**************/ /* the module */ @@ -4903,6 +5439,75 @@ _imp_source_hash_impl(PyObject *module, long key, Py_buffer *source) return PyBytes_FromStringAndSize(hash.data, sizeof(hash.data)); } +/*[clinic input] +_imp._set_lazy_attributes + child_module: object + name: unicode + / +Sets attributes to lazy submodules on the module, as side effects. +[clinic start generated code]*/ + +static PyObject * +_imp__set_lazy_attributes_impl(PyObject *module, PyObject *child_module, + PyObject *name) +/*[clinic end generated code: output=bd34f2e16f215c29 input=d959fbfa236f4d59]*/ +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *child_dict = NULL; + PyObject *ret = NULL; + // Use atomic load and hold a reference to prevent use-after-free + // if another thread clears lazy_modules during interpreter shutdown. + PyObject *lazy_modules = FT_ATOMIC_LOAD_PTR_RELAXED(tstate->interp->imports.lazy_modules); + Py_XINCREF(lazy_modules); + if (lazy_modules != NULL) { + PyObject *lazy_submodules = PyDict_GetItemWithError(lazy_modules, name); + if (lazy_submodules == NULL) { + if (PyErr_Occurred()) { + goto error; + } + goto done; + } + + child_dict = get_mod_dict(child_module); + if (child_dict == NULL) { + goto error; + } + else if (!PyDict_CheckExact(child_dict)) { + goto done; + } + assert(PyAnySet_CheckExact(lazy_submodules)); + Py_ssize_t pos = 0; + PyObject *attr_name; + Py_hash_t hash; + while (_PySet_NextEntry(lazy_submodules, &pos, &attr_name, &hash)) { + if (_PyDict_Contains_KnownHash(child_dict, attr_name, hash)) { + continue; + } + PyObject *builtins = _PyEval_GetBuiltins(tstate); + PyObject *lazy_module_attr = _PyLazyImport_New(builtins, name, attr_name); + if (lazy_module_attr == NULL) { + goto error; + } + + if (_PyModule_ReplaceLazyValue(child_dict, attr_name, lazy_module_attr) < 0) { + Py_DECREF(lazy_module_attr); + goto error; + } + Py_DECREF(lazy_module_attr); + } + if (PyDict_DelItem(lazy_modules, name) < 0) { + goto error; + } + } + +done: + ret = Py_NewRef(Py_None); + +error: + Py_XDECREF(lazy_modules); + Py_XDECREF(child_dict); + return ret; +} PyDoc_STRVAR(doc_imp, "(Extremely) low-level import machinery bits as used by importlib."); @@ -4927,6 +5532,7 @@ static PyMethodDef imp_methods[] = { _IMP_EXEC_BUILTIN_METHODDEF _IMP__FIX_CO_FILENAME_METHODDEF _IMP_SOURCE_HASH_METHODDEF + _IMP__SET_LAZY_ATTRIBUTES_METHODDEF {NULL, NULL} /* sentinel */ }; diff --git a/Python/initconfig.c b/Python/initconfig.c index 7176670c110..ee91704753b 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -111,6 +111,7 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(base_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("base_prefix")), SPEC(bytes_warning, UINT, PUBLIC, SYS_FLAG(9)), SPEC(cpu_count, INT, PUBLIC, NO_SYS), + SPEC(lazy_imports, INT, PUBLIC, NO_SYS), SPEC(exec_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("exec_prefix")), SPEC(executable, WSTR_OPT, PUBLIC, SYS_ATTR("executable")), SPEC(inspect, BOOL, PUBLIC, SYS_FLAG(1)), @@ -317,6 +318,8 @@ The following implementation-specific options are available:\n\ "\ -X importtime[=2]: show how long each import takes; use -X importtime=2 to\n\ log imports of already-loaded modules; also PYTHONPROFILEIMPORTTIME\n\ +-X lazy_imports=[all|none|normal]: control global lazy imports; default is normal;\n\ + also PYTHON_LAZY_IMPORTS\n\ -X int_max_str_digits=N: limit the size of int<->str conversions;\n\ 0 disables the limit; also PYTHONINTMAXSTRDIGITS\n\ -X no_debug_ranges: don't include extra location information in code objects;\n\ @@ -431,6 +434,7 @@ static const char usage_envvars[] = "PYTHON_PRESITE: import this module before site (-X presite)\n" #endif "PYTHONPROFILEIMPORTTIME: show how long each import takes (-X importtime)\n" +"PYTHON_LAZY_IMPORTS: control global lazy imports (-X lazy_imports)\n" "PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files\n" " (-X pycache_prefix)\n" "PYTHONSAFEPATH : don't prepend a potentially unsafe path to sys.path.\n" @@ -939,6 +943,8 @@ config_check_consistency(const PyConfig *config) assert(config->int_max_str_digits >= 0); // cpu_count can be -1 if the user doesn't override it. assert(config->cpu_count != 0); + // lazy_imports can be -1 (default), 0 (off), or 1 (on). + assert(config->lazy_imports >= -1 && config->lazy_imports <= 1); // config->use_frozen_modules is initialized later // by _PyConfig_InitImportConfig(). assert(config->thread_inherit_context >= 0); @@ -1050,6 +1056,7 @@ _PyConfig_InitCompatConfig(PyConfig *config) config->_is_python_build = 0; config->code_debug_ranges = 1; config->cpu_count = -1; + config->lazy_imports = -1; #ifdef Py_GIL_DISABLED config->thread_inherit_context = 1; config->context_aware_warnings = 1; @@ -2284,6 +2291,49 @@ config_init_import_time(PyConfig *config) return _PyStatus_OK(); } +static PyStatus +config_init_lazy_imports(PyConfig *config) +{ + int lazy_imports = -1; + + const char *env = config_get_env(config, "PYTHON_LAZY_IMPORTS"); + if (env) { + if (strcmp(env, "all") == 0) { + lazy_imports = 1; + } + else if (strcmp(env, "none") == 0) { + lazy_imports = 0; + } + else if (strcmp(env, "normal") == 0) { + lazy_imports = -1; + } + else { + return _PyStatus_ERR("PYTHON_LAZY_IMPORTS: invalid value; " + "expected 'all', 'none', or 'normal'"); + } + config->lazy_imports = lazy_imports; + } + + const wchar_t *x_value = config_get_xoption_value(config, L"lazy_imports"); + if (x_value) { + if (wcscmp(x_value, L"all") == 0) { + lazy_imports = 1; + } + else if (wcscmp(x_value, L"none") == 0) { + lazy_imports = 0; + } + else if (wcscmp(x_value, L"normal") == 0) { + lazy_imports = -1; + } + else { + return _PyStatus_ERR("-X lazy_imports: invalid value; " + "expected 'all', 'none', or 'normal'"); + } + config->lazy_imports = lazy_imports; + } + return _PyStatus_OK(); +} + static PyStatus config_read_complex_options(PyConfig *config) { @@ -2307,6 +2357,13 @@ config_read_complex_options(PyConfig *config) } } + if (config->lazy_imports < 0) { + status = config_init_lazy_imports(config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } + if (config->tracemalloc < 0) { status = config_init_tracemalloc(config); if (_PyStatus_EXCEPTION(status)) { @@ -2696,6 +2753,9 @@ config_read(PyConfig *config, int compute_path_config) if (config->tracemalloc < 0) { config->tracemalloc = 0; } + if (config->lazy_imports < 0) { + config->lazy_imports = -1; // Default is auto/unset + } if (config->perf_profiling < 0) { config->perf_profiling = 0; } diff --git a/Python/jit.c b/Python/jit.c index 47d3d7a5d27..18f4ad868f8 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -11,9 +11,11 @@ #include "pycore_floatobject.h" #include "pycore_frame.h" #include "pycore_function.h" +#include "pycore_import.h" #include "pycore_interpframe.h" #include "pycore_interpolation.h" #include "pycore_intrinsics.h" +#include "pycore_lazyimportobject.h" #include "pycore_list.h" #include "pycore_long.h" #include "pycore_opcode_metadata.h" diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 2527dca71d7..96ac33a4b25 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1330,6 +1330,21 @@ init_interp_main(PyThreadState *tstate) } } + // Initialize lazy imports based on configuration + // Do this after site module is imported to avoid circular imports during startup + if (config->lazy_imports != -1) { + PyImport_LazyImportsMode lazy_mode; + if (config->lazy_imports == 1) { + lazy_mode = PyImport_LAZY_ALL; + } else { + lazy_mode = PyImport_LAZY_NONE; + } + if (PyImport_SetLazyImportsMode(lazy_mode) < 0) { + return _PyStatus_ERR("failed to set lazy imports mode"); + } + } + // If config->lazy_imports == -1, use the default mode (no change needed) + if (is_main_interp) { #ifndef MS_WINDOWS emit_stderr_warning_for_legacy_locale(interp->runtime); diff --git a/Python/specialize.c b/Python/specialize.c index 19433bc7a74..a3707eb681f 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -8,6 +8,7 @@ #include "pycore_dict.h" // DICT_KEYS_UNICODE #include "pycore_function.h" // _PyFunction_GetVersionForCurrentState() #include "pycore_interpframe.h" // FRAME_SPECIALS_SIZE +#include "pycore_lazyimportobject.h" // PyLazyImport_CheckExact #include "pycore_list.h" // _PyListIterObject #include "pycore_long.h" // _PyLong_IsNonNegativeCompact() #include "pycore_moduleobject.h" @@ -127,6 +128,7 @@ _PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, int enable_counters #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22 #define SPEC_FAIL_ATTR_CLASS_METHOD_OBJ 23 #define SPEC_FAIL_ATTR_OBJECT_SLOT 24 +#define SPEC_FAIL_ATTR_MODULE_LAZY_VALUE 25 #define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26 #define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27 @@ -384,6 +386,14 @@ specialize_module_load_attr_lock_held(PyDictObject *dict, _Py_CODEUNIT *instr, P SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return -1; } + PyObject *value = NULL; + if (PyDict_GetItemRef((PyObject *)dict, name, &value) < 0 || + (value != NULL && PyLazyImport_CheckExact(value))) { + Py_XDECREF(value); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_MODULE_LAZY_VALUE); + return -1; + } + Py_XDECREF(value); write_u32(cache->version, keys_version); cache->index = (uint16_t)index; specialize(instr, LOAD_ATTR_MODULE); @@ -1284,6 +1294,17 @@ specialize_load_global_lock_held( SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT); goto fail; } + PyObject *value = NULL; + if (PyDict_GetItemRef(globals, name, &value) < 0) { + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_EXPECTED_ERROR); + goto fail; + } + if (value != NULL && PyLazyImport_CheckExact(value)) { + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_ATTR_MODULE_LAZY_VALUE); + Py_DECREF(value); + goto fail; + } + Py_XDECREF(value); Py_ssize_t index = _PyDictKeys_StringLookup(globals_keys, name); if (index == DKIX_ERROR) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_EXPECTED_ERROR); diff --git a/Python/symtable.c b/Python/symtable.c index 29cf9190a4e..05ee50037d8 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -141,6 +141,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_needs_classdict = 0; ste->ste_has_conditional_annotations = 0; ste->ste_in_conditional_block = 0; + ste->ste_in_try_block = 0; ste->ste_in_unevaluated_annotation = 0; ste->ste_annotation_block = NULL; @@ -1747,6 +1748,13 @@ symtable_enter_type_param_block(struct symtable *st, identifier name, #define LEAVE_CONDITIONAL_BLOCK(ST) \ (ST)->st_cur->ste_in_conditional_block = in_conditional_block; +#define ENTER_TRY_BLOCK(ST) \ + int in_try_block = (ST)->st_cur->ste_in_try_block; \ + (ST)->st_cur->ste_in_try_block = 1; + +#define LEAVE_TRY_BLOCK(ST) \ + (ST)->st_cur->ste_in_try_block = in_try_block; + #define ENTER_RECURSIVE() \ if (Py_EnterRecursiveCall(" during compilation")) { \ return 0; \ @@ -1808,6 +1816,36 @@ check_import_from(struct symtable *st, stmt_ty s) return 1; } +static int +check_lazy_import_context(struct symtable *st, stmt_ty s, const char* import_type) +{ + /* Check if inside try/except block */ + if (st->st_cur->ste_in_try_block) { + PyErr_Format(PyExc_SyntaxError, + "lazy %s not allowed inside try/except blocks", import_type); + SET_ERROR_LOCATION(st->st_filename, LOCATION(s)); + return 0; + } + + /* Check if inside function scope */ + if (st->st_cur->ste_type == FunctionBlock) { + PyErr_Format(PyExc_SyntaxError, + "lazy %s not allowed inside functions", import_type); + SET_ERROR_LOCATION(st->st_filename, LOCATION(s)); + return 0; + } + + /* Check if inside class scope */ + if (st->st_cur->ste_type == ClassBlock) { + PyErr_Format(PyExc_SyntaxError, + "lazy %s not allowed inside classes", import_type); + SET_ERROR_LOCATION(st->st_filename, LOCATION(s)); + return 0; + } + + return 1; +} + static bool allows_top_level_await(struct symtable *st) { @@ -2076,19 +2114,23 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) break; case Try_kind: { ENTER_CONDITIONAL_BLOCK(st); + ENTER_TRY_BLOCK(st); VISIT_SEQ(st, stmt, s->v.Try.body); VISIT_SEQ(st, excepthandler, s->v.Try.handlers); VISIT_SEQ(st, stmt, s->v.Try.orelse); VISIT_SEQ(st, stmt, s->v.Try.finalbody); + LEAVE_TRY_BLOCK(st); LEAVE_CONDITIONAL_BLOCK(st); break; } case TryStar_kind: { ENTER_CONDITIONAL_BLOCK(st); + ENTER_TRY_BLOCK(st); VISIT_SEQ(st, stmt, s->v.TryStar.body); VISIT_SEQ(st, excepthandler, s->v.TryStar.handlers); VISIT_SEQ(st, stmt, s->v.TryStar.orelse); VISIT_SEQ(st, stmt, s->v.TryStar.finalbody); + LEAVE_TRY_BLOCK(st); LEAVE_CONDITIONAL_BLOCK(st); break; } @@ -2098,9 +2140,29 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT(st, expr, s->v.Assert.msg); break; case Import_kind: + if (s->v.Import.is_lazy) { + if (!check_lazy_import_context(st, s, "import")) { + return 0; + } + } VISIT_SEQ(st, alias, s->v.Import.names); break; case ImportFrom_kind: + if (s->v.ImportFrom.is_lazy) { + if (!check_lazy_import_context(st, s, "from ... import")) { + return 0; + } + + /* Check for import * */ + for (Py_ssize_t i = 0; i < asdl_seq_LEN(s->v.ImportFrom.names); i++) { + alias_ty alias = (alias_ty)asdl_seq_GET(s->v.ImportFrom.names, i); + if (alias->name && _PyUnicode_EqualToASCIIString(alias->name, "*")) { + PyErr_SetString(PyExc_SyntaxError, "lazy from ... import * is not allowed"); + SET_ERROR_LOCATION(st->st_filename, LOCATION(s)); + return 0; + } + } + } VISIT_SEQ(st, alias, s->v.ImportFrom.names); if (!check_import_from(st, s)) { return 0; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index b4b441bf4d9..9cbcaafab98 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2774,6 +2774,147 @@ PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename) { return 0; } +/*[clinic input] +sys.set_lazy_imports_filter + + filter: object + +Set the lazy imports filter callback. + +The filter is a callable which disables lazy imports when they +would otherwise be enabled. Returns True if the import is still enabled +or False to disable it. The callable is called with: + +(importing_module_name, imported_module_name, [fromlist]) + +Pass None to clear the filter. +[clinic start generated code]*/ + +static PyObject * +sys_set_lazy_imports_filter_impl(PyObject *module, PyObject *filter) +/*[clinic end generated code: output=10251d49469c278c input=2eb48786bdd4ee42]*/ +{ + if (PyImport_SetLazyImportsFilter(filter) < 0) { + return NULL; + } + + Py_RETURN_NONE; +} + +/*[clinic input] +sys.get_lazy_imports_filter + +Get the current lazy imports filter callback. + +Returns the filter callable or None if no filter is set. +[clinic start generated code]*/ + +static PyObject * +sys_get_lazy_imports_filter_impl(PyObject *module) +/*[clinic end generated code: output=3bf73022892165af input=cf1e07cb8e203c94]*/ +{ + PyObject *filter = PyImport_GetLazyImportsFilter(); + if (filter == NULL) { + assert(!PyErr_Occurred()); + Py_RETURN_NONE; + } + return filter; +} + +/*[clinic input] +sys.set_lazy_imports + + mode: object + +Sets the global lazy imports mode. + +The mode parameter must be one of the following strings: +- "all": All top-level imports become potentially lazy +- "none": All lazy imports are suppressed (even explicitly marked ones) +- "normal": Only explicitly marked imports (with 'lazy' keyword) are lazy + +In addition to the mode, lazy imports can be controlled via the filter +provided to sys.set_lazy_imports_filter + +[clinic start generated code]*/ + +static PyObject * +sys_set_lazy_imports_impl(PyObject *module, PyObject *mode) +/*[clinic end generated code: output=1ff34ba6c4feaf73 input=f04e70d8bf9fe4f6]*/ +{ + PyImport_LazyImportsMode lazy_mode; + if (!PyUnicode_Check(mode)) { + PyErr_SetString(PyExc_TypeError, "mode must be a string: 'normal', 'all', or 'none'"); + return NULL; + } + if (PyUnicode_CompareWithASCIIString(mode, "normal") == 0) { + lazy_mode = PyImport_LAZY_NORMAL; + } else if (PyUnicode_CompareWithASCIIString(mode, "all") == 0) { + lazy_mode = PyImport_LAZY_ALL; + } else if (PyUnicode_CompareWithASCIIString(mode, "none") == 0) { + lazy_mode = PyImport_LAZY_NONE; + } else { + PyErr_SetString(PyExc_ValueError, "mode must be 'normal', 'all', or 'none'"); + return NULL; + } + + if (PyImport_SetLazyImportsMode(lazy_mode)) { + return NULL; + } + Py_RETURN_NONE; +} + +/*[clinic input] +sys.get_lazy_imports + +Gets the global lazy imports mode. + +Returns "all" if all top level imports are potentially lazy. +Returns "none" if all explicitly marked lazy imports are suppressed. +Returns "normal" if only explicitly marked imports are lazy. + +[clinic start generated code]*/ + +static PyObject * +sys_get_lazy_imports_impl(PyObject *module) +/*[clinic end generated code: output=4147dec48c51ae99 input=8cb574f1e4e3003c]*/ +{ + switch (PyImport_GetLazyImportsMode()) { + case PyImport_LAZY_NORMAL: + return PyUnicode_FromString("normal"); + case PyImport_LAZY_ALL: + return PyUnicode_FromString("all"); + case PyImport_LAZY_NONE: + return PyUnicode_FromString("none"); + default: + PyErr_SetString(PyExc_RuntimeError, "unknown lazy imports mode"); + return NULL; + } +} + +/*[clinic input] +sys.get_lazy_modules + +Gets the set of module names that have been lazily imported. + +Returns a set of fully-qualified module names that have been lazily +imported at some point (primarily for diagnostics and introspection). +Note that modules are removed from this set when they are reified +(actually loaded). + +[clinic start generated code]*/ + +static PyObject * +sys_get_lazy_modules_impl(PyObject *module) +/*[clinic end generated code: output=4c641f8881ba87c0 input=511b3a9682c09282]*/ +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *lazy_modules_set = FT_ATOMIC_LOAD_PTR_RELAXED(interp->imports.lazy_modules_set); + if (lazy_modules_set == NULL) { + return PySet_New(NULL); + } + return Py_NewRef(lazy_modules_set); +} static PyMethodDef sys_methods[] = { /* Might as well keep this in alphabetic order */ @@ -2839,6 +2980,11 @@ static PyMethodDef sys_methods[] = { SYS_UNRAISABLEHOOK_METHODDEF SYS_GET_INT_MAX_STR_DIGITS_METHODDEF SYS_SET_INT_MAX_STR_DIGITS_METHODDEF + SYS_GET_LAZY_IMPORTS_METHODDEF + SYS_SET_LAZY_IMPORTS_METHODDEF + SYS_GET_LAZY_IMPORTS_FILTER_METHODDEF + SYS_SET_LAZY_IMPORTS_FILTER_METHODDEF + SYS_GET_LAZY_MODULES_METHODDEF SYS__BASEREPL_METHODDEF #ifdef Py_STATS SYS__STATS_ON_METHODDEF @@ -3363,6 +3509,7 @@ static PyStructSequence_Field flags_fields[] = { {"gil", "-X gil"}, {"thread_inherit_context", "-X thread_inherit_context"}, {"context_aware_warnings", "-X context_aware_warnings"}, + {"lazy_imports", "-X lazy_imports"}, {0} }; @@ -3372,7 +3519,7 @@ static PyStructSequence_Desc flags_desc = { "sys.flags", /* name */ flags__doc__, /* doc */ flags_fields, /* fields */ - 18 + 19 }; static void @@ -3465,6 +3612,7 @@ set_flags_from_config(PyInterpreterState *interp, PyObject *flags) #endif SetFlag(config->thread_inherit_context); SetFlag(config->context_aware_warnings); + SetFlag(config->lazy_imports); #undef SetFlagObj #undef SetFlag return 0; @@ -3950,6 +4098,14 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) SET_SYS("path_importer_cache", PyDict_New()); SET_SYS("path_hooks", PyList_New(0)); + /* adding sys.lazy_modules set (PEP 810) */ + PyObject *lazy_modules_set = PySet_New(NULL); + if (lazy_modules_set == NULL) { + goto err_occurred; + } + interp->imports.lazy_modules_set = lazy_modules_set; + SET_SYS("lazy_modules", Py_NewRef(lazy_modules_set)); + if (_PyErr_Occurred(tstate)) { goto err_occurred; } diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 301784f773d..d645d2b6150 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -59,6 +59,7 @@ Objects/interpolationobject.c - _PyInterpolation_Type - Objects/iterobject.c - PyCallIter_Type - Objects/iterobject.c - PySeqIter_Type - Objects/iterobject.c - _PyAnextAwaitable_Type - +Objects/lazyimportobject.c - PyLazyImport_Type - Objects/listobject.c - PyListIter_Type - Objects/listobject.c - PyListRevIter_Type - Objects/listobject.c - PyList_Type - @@ -176,6 +177,7 @@ Objects/exceptions.c - _PyExc_StopIteration - Objects/exceptions.c - _PyExc_GeneratorExit - Objects/exceptions.c - _PyExc_SystemExit - Objects/exceptions.c - _PyExc_KeyboardInterrupt - +Objects/exceptions.c - _PyExc_ImportCycleError - Objects/exceptions.c - _PyExc_ImportError - Objects/exceptions.c - _PyExc_ModuleNotFoundError - Objects/exceptions.c - _PyExc_OSError - @@ -242,6 +244,7 @@ Objects/exceptions.c - PyExc_StopIteration - Objects/exceptions.c - PyExc_GeneratorExit - Objects/exceptions.c - PyExc_SystemExit - Objects/exceptions.c - PyExc_KeyboardInterrupt - +Objects/exceptions.c - PyExc_ImportCycleError - Objects/exceptions.c - PyExc_ImportError - Objects/exceptions.c - PyExc_ModuleNotFoundError - Objects/exceptions.c - PyExc_OSError - diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 0167f1b0ae5..c2f17a89b44 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -12,9 +12,11 @@ #include "pycore_frame.h" #include "pycore_function.h" #include "pycore_genobject.h" +#include "pycore_import.h" #include "pycore_interpframe.h" #include "pycore_interpolation.h" #include "pycore_intrinsics.h" +#include "pycore_lazyimportobject.h" #include "pycore_jit.h" #include "pycore_list.h" #include "pycore_long.h"