gh-139899: Introduce MetaPathFinder.discover and PathEntryFinder.discover (#139900)

* gh-139899: Introduce MetaPathFinder.discover and PathEntryFinder.discover Signed-off-by: Filipe Laíns <lains@riseup.net> * Fix doc reference Signed-off-by: Filipe Laíns <lains@riseup.net> * Remove specific doc references Signed-off-by: Filipe Laíns <lains@riseup.net> * Fix docstrings Signed-off-by: Filipe Laíns <lains@riseup.net> * Revert "Remove specific doc references" This reverts commit 31d1a8f551. Signed-off-by: Filipe Laíns <lains@riseup.net> * Fix news references Signed-off-by: Filipe Laíns <lains@riseup.net> * Add docs warning Signed-off-by: Filipe Laíns <lains@riseup.net> * Raise ValueError on invalid parent Signed-off-by: Filipe Laíns <lains@riseup.net> * Dedupe __path__ in PathFinder.discover Signed-off-by: Filipe Laíns <lains@riseup.net> * Use context manager and add error handling to os.scandir Signed-off-by: Filipe Laíns <lains@riseup.net> * Raise ValueError on invalid parent Signed-off-by: Filipe Laíns <lains@riseup.net> * Dedupe when package exists with multiple suffixes Signed-off-by: Filipe Laíns <lains@riseup.net> * Apply suggestions from code review Co-authored-by: Alyssa Coghlan <ncoghlan@gmail.com> * Add tests Signed-off-by: Filipe Laíns <lains@riseup.net> --------- Signed-off-by: Filipe Laíns <lains@riseup.net> Co-authored-by: Alyssa Coghlan <ncoghlan@gmail.com> Co-authored-by: Brett Cannon <brett@python.org>
2026-02-21 14:41:06 +00:00 · 2026-02-19 15:25:50 +00:00 · 2026-02-19 15:25:50 +00:00 · 157f271de3
commit 157f271de3
parent f1cf762ee7
5 changed files with 235 additions and 0 deletions
--- a/Doc/library/importlib.rst
+++ b/Doc/library/importlib.rst
@ -275,6 +275,28 @@ ABC hierarchy::
      .. versionchanged:: 3.4
         Returns ``None`` when called instead of :data:`NotImplemented`.

+   .. method:: discover(parent=None)
+
+      An optional method which searches for possible specs with given *parent*
+      module spec. If *parent* is *None*, :meth:`MetaPathFinder.discover` will
+      search for top-level modules.
+
+      Returns an iterable of possible specs.
+
+      Raises :exc:`ValueError` if *parent* is not a package module.
+
+      .. warning::
+         This method can potentially yield a very large number of objects, and
+         it may carry out IO operations when computing these values.
+
+         Because of this, it will generaly be desirable to compute the result
+         values on-the-fly, as they are needed. As such, the returned object is
+         only guaranteed to be an :class:`iterable <collections.abc.Iterable>`,
+         instead of a :class:`list` or other
+         :class:`collection <collections.abc.Collection>` type.
+
+      .. versionadded:: next
+

 .. class:: PathEntryFinder

@ -307,6 +329,28 @@ ABC hierarchy::
      :meth:`importlib.machinery.PathFinder.invalidate_caches`
      when invalidating the caches of all cached finders.

+   .. method:: discover(parent=None)
+
+      An optional method which searches for possible specs with given *parent*
+      module spec. If *parent* is *None*, :meth:`PathEntryFinder.discover` will
+      search for top-level modules.
+
+      Returns an iterable of possible specs.
+
+      Raises :exc:`ValueError` if *parent* is not a package module.
+
+      .. warning::
+         This method can potentially yield a very large number of objects, and
+         it may carry out IO operations when computing these values.
+
+         Because of this, it will generaly be desirable to compute the result
+         values on-the-fly, as they are needed. As such, the returned object is
+         only guaranteed to be an :class:`iterable <collections.abc.Iterable>`,
+         instead of a :class:`list` or other
+         :class:`collection <collections.abc.Collection>` type.
+
+      .. versionadded:: next
+

 .. class:: Loader

--- a/Lib/importlib/_bootstrap_external.py
+++ b/Lib/importlib/_bootstrap_external.py
@ -1283,6 +1283,23 @@ def find_spec(cls, fullname, path=None, target=None):
        else:
            return spec

+    @classmethod
+    def discover(cls, parent=None):
+        if parent is None:
+            path = sys.path
+        elif parent.submodule_search_locations is None:
+            raise ValueError(f'{parent} is not a package module')
+        else:
+            path = parent.submodule_search_locations
+
+        for entry in set(path):
+            if not isinstance(entry, str):
+                continue
+            if (finder := cls._path_importer_cache(entry)) is None:
+                continue
+            if discover := getattr(finder, 'discover', None):
+                yield from discover(parent)
+
    @staticmethod
    def find_distributions(*args, **kwargs):
        """
@ -1432,6 +1449,37 @@ def path_hook_for_FileFinder(path):

        return path_hook_for_FileFinder

+    def _find_children(self):
+        with _os.scandir(self.path) as scan_iterator:
+            while True:
+                try:
+                    entry = next(scan_iterator)
+                    if entry.name == _PYCACHE:
+                        continue
+                    # packages
+                    if entry.is_dir() and '.' not in entry.name:
+                        yield entry.name
+                    # files
+                    if entry.is_file():
+                        yield from {
+                            entry.name.removesuffix(suffix)
+                            for suffix, _ in self._loaders
+                            if entry.name.endswith(suffix)
+                        }
+                except OSError:
+                    pass  # ignore exceptions from next(scan_iterator) and os.DirEntry
+                except StopIteration:
+                    break
+
+    def discover(self, parent=None):
+        if parent and parent.submodule_search_locations is None:
+            raise ValueError(f'{parent} is not a package module')
+
+        module_prefix = f'{parent.name}.' if parent else ''
+        for child_name in self._find_children():
+            if spec := self.find_spec(module_prefix + child_name):
+                yield spec
+
    def __repr__(self):
        return f'FileFinder({self.path!r})'

--- a/Lib/importlib/abc.py
+++ b/Lib/importlib/abc.py
@ -45,6 +45,16 @@ def invalidate_caches(self):
        This method is used by importlib.invalidate_caches().
        """

+    def discover(self, parent=None):
+        """An optional method which searches for possible specs with given *parent*
+        module spec. If *parent* is *None*, MetaPathFinder.discover will search
+        for top-level modules.
+
+        Returns an iterable of possible specs.
+        """
+        return ()
+
+
 _register(MetaPathFinder, machinery.BuiltinImporter, machinery.FrozenImporter,
          machinery.PathFinder, machinery.WindowsRegistryFinder)

@ -58,6 +68,15 @@ def invalidate_caches(self):
        This method is used by PathFinder.invalidate_caches().
        """

+    def discover(self, parent=None):
+        """An optional method which searches for possible specs with given
+        *parent* module spec. If *parent* is *None*, PathEntryFinder.discover
+        will search for top-level modules.
+
+        Returns an iterable of possible specs.
+        """
+        return ()
+
 _register(PathEntryFinder, machinery.FileFinder)


--- a/Lib/test/test_importlib/test_discover.py
+++ b/Lib/test/test_importlib/test_discover.py
@ -0,0 +1,121 @@
+from unittest.mock import Mock
+
+from test.test_importlib import util
+
+importlib = util.import_importlib('importlib')
+machinery = util.import_importlib('importlib.machinery')
+
+
+class DiscoverableFinder:
+    def __init__(self, discover=[]):
+        self._discovered_values = discover
+
+    def find_spec(self, fullname, path=None, target=None):
+        raise NotImplemented
+
+    def discover(self, parent=None):
+        yield from self._discovered_values
+
+
+class TestPathFinder:
+    """PathFinder implements MetaPathFinder, which uses the PathEntryFinder(s)
+    registered in sys.path_hooks (and sys.path_importer_cache) to search
+    sys.path or the parent's __path__.
+
+    PathFinder.discover() should redirect to the .discover() method of the
+    PathEntryFinder for each path entry.
+    """
+
+    def test_search_path_hooks_top_level(self):
+        modules = [
+            self.machinery.ModuleSpec(name='example1', loader=None),
+            self.machinery.ModuleSpec(name='example2', loader=None),
+            self.machinery.ModuleSpec(name='example3', loader=None),
+        ]
+
+        with util.import_state(
+            path_importer_cache={
+                'discoverable': DiscoverableFinder(discover=modules),
+            },
+            path=['discoverable'],
+        ):
+            discovered = list(self.machinery.PathFinder.discover())
+
+        self.assertEqual(discovered, modules)
+
+
+    def test_search_path_hooks_parent(self):
+        parent = self.machinery.ModuleSpec(name='example', loader=None, is_package=True)
+        parent.submodule_search_locations.append('discoverable')
+
+        children = [
+            self.machinery.ModuleSpec(name='example.child1', loader=None),
+            self.machinery.ModuleSpec(name='example.child2', loader=None),
+            self.machinery.ModuleSpec(name='example.child3', loader=None),
+        ]
+
+        with util.import_state(
+            path_importer_cache={
+                'discoverable': DiscoverableFinder(discover=children)
+            },
+            path=[],
+        ):
+            discovered = list(self.machinery.PathFinder.discover(parent))
+
+        self.assertEqual(discovered, children)
+
+    def test_invalid_parent(self):
+        parent = self.machinery.ModuleSpec(name='example', loader=None)
+        with self.assertRaises(ValueError):
+            list(self.machinery.PathFinder.discover(parent))
+
+
+(
+    Frozen_TestPathFinder,
+    Source_TestPathFinder,
+) = util.test_both(TestPathFinder, importlib=importlib, machinery=machinery)
+
+
+class TestFileFinder:
+    """FileFinder implements PathEntryFinder and provides the base finder
+    implementation to search the file system.
+    """
+
+    def get_finder(self, path):
+        loader_details = [
+            (self.machinery.SourceFileLoader, self.machinery.SOURCE_SUFFIXES),
+            (self.machinery.SourcelessFileLoader, self.machinery.BYTECODE_SUFFIXES),
+        ]
+        return self.machinery.FileFinder(path, *loader_details)
+
+    def test_discover_top_level(self):
+        modules = {'example1', 'example2', 'example3'}
+        with util.create_modules(*modules) as mapping:
+            finder = self.get_finder(mapping['.root'])
+            discovered = list(finder.discover())
+        self.assertEqual({spec.name for spec in discovered}, modules)
+
+    def test_discover_parent(self):
+        modules = {
+            'example.child1',
+            'example.child2',
+            'example.child3',
+        }
+        with util.create_modules(*modules) as mapping:
+            example = self.get_finder(mapping['.root']).find_spec('example')
+            finder = self.get_finder(example.submodule_search_locations[0])
+            discovered = list(finder.discover(example))
+        self.assertEqual({spec.name for spec in discovered}, modules)
+
+    def test_invalid_parent(self):
+        with util.create_modules('example') as mapping:
+            finder = self.get_finder(mapping['.root'])
+            example = finder.find_spec('example')
+            with self.assertRaises(ValueError):
+                list(finder.discover(example))
+
+
+(
+    Frozen_TestFileFinder,
+    Source_TestFileFinder,
+) = util.test_both(TestFileFinder, importlib=importlib, machinery=machinery)
--- a/Misc/NEWS.d/next/Library/2025-10-10-14-08-58.gh-issue-139899.09leRY.rst
+++ b/Misc/NEWS.d/next/Library/2025-10-10-14-08-58.gh-issue-139899.09leRY.rst
@ -0,0 +1,3 @@
+Introduced :meth:`importlib.abc.MetaPathFinder.discover`
+and :meth:`importlib.abc.PathEntryFinder.discover` to allow module and submodule
+name discovery without assuming the use of traditional filesystem based imports.