Issue #23911: Move path-based bootstrap code to a separate frozen module.

Lib/ctypes/test/test_values.py

@@ -59,8 +59,13 @@ class struct_frozen(Structure):
         items = []
         # _frozen_importlib changes size whenever importlib._bootstrap
         # changes, so it gets a special case.  We should make sure it's
-        # found, but don't worry about its size too much.
+        # found, but don't worry about its size too much.  The same
-        _fzn_implib_seen = False
+        # applies to _frozen_importlib_external.
+        bootstrap_seen = []
+        bootstrap_expected = (
+                b'_frozen_importlib',
+                b'_frozen_importlib_external',
+                )
         for entry in ft:
             # This is dangerous. We *can* iterate over a pointer, but
             # the loop will not terminate (maybe with an access
@@ -68,10 +73,10 @@ class struct_frozen(Structure):
             if entry.name is None:
                 break
 
-            if entry.name == b'_frozen_importlib':
+            if entry.name in bootstrap_expected:
-                _fzn_implib_seen = True
+                bootstrap_seen.append(entry.name)
                 self.assertTrue(entry.size,
-                    "_frozen_importlib was reported as having no size")
+                    "{} was reported as having no size".format(entry.name))
                 continue
             items.append((entry.name, entry.size))
 
@@ -81,8 +86,8 @@ class struct_frozen(Structure):
                     ]
         self.assertEqual(items, expected)
 
-        self.assertTrue(_fzn_implib_seen,
+        self.assertEqual(sorted(bootstrap_seen), bootstrap_expected,
-            "_frozen_importlib wasn't found in PyImport_FrozenModules")
+            "frozen bootstrap modules did not match PyImport_FrozenModules")
 
         from ctypes import _pointer_type_cache
         del _pointer_type_cache[struct_frozen]

Lib/imp.py

@@ -16,7 +16,8 @@
     # Platform doesn't support dynamic loading.
     load_dynamic = None
 
-from importlib._bootstrap import SourcelessFileLoader, _ERR_MSG, _exec, _load
+from importlib._bootstrap import _ERR_MSG, _exec, _load
+from importlib._bootstrap_external import SourcelessFileLoader
 
 from importlib import machinery
 from importlib import util

Lib/importlib/__init__.py

@@ -30,9 +30,25 @@
         pass
     sys.modules['importlib._bootstrap'] = _bootstrap
 
+try:
+    import _frozen_importlib_external as _bootstrap_external
+except ImportError:
+    from . import _bootstrap_external
+    _bootstrap_external._setup(_bootstrap)
+else:
+    _bootstrap_external.__name__ = 'importlib._bootstrap_external'
+    _bootstrap_external.__package__ = 'importlib'
+    try:
+        _bootstrap_external.__file__ = __file__.replace('__init__.py', '_bootstrap_external.py')
+    except NameError:
+        # __file__ is not guaranteed to be defined, e.g. if this code gets
+        # frozen by a tool like cx_Freeze.
+        pass
+    sys.modules['importlib._bootstrap_external'] = _bootstrap_external
+
 # To simplify imports in test code
-_w_long = _bootstrap._w_long
+_w_long = _bootstrap_external._w_long
-_r_long = _bootstrap._r_long
+_r_long = _bootstrap_external._r_long
 
 # Fully bootstrapped at this point, import whatever you like, circular
 # dependencies and startup overhead minimisation permitting :)

Lib/importlib/abc.py

@@ -1,12 +1,17 @@
 """Abstract base classes related to import."""
-from . import _bootstrap
+from . import _bootstrap_external
 from . import machinery
 try:
     import _frozen_importlib
+#    import _frozen_importlib_external
 except ImportError as exc:
     if exc.name != '_frozen_importlib':
         raise
     _frozen_importlib = None
+try:
+    import _frozen_importlib_external
+except ImportError as exc:
+    _frozen_importlib_external = _bootstrap_external
 import abc
 
 
@@ -14,7 +19,10 @@ def _register(abstract_cls, *classes):
     for cls in classes:
         abstract_cls.register(cls)
         if _frozen_importlib is not None:
-            frozen_cls = getattr(_frozen_importlib, cls.__name__)
+            try:
+                frozen_cls = getattr(_frozen_importlib, cls.__name__)
+            except AttributeError:
+                frozen_cls = getattr(_frozen_importlib_external, cls.__name__)
             abstract_cls.register(frozen_cls)
 
 
@@ -102,7 +110,7 @@ def find_loader(self, fullname):
         else:
             return None, []
 
-    find_module = _bootstrap._find_module_shim
+    find_module = _bootstrap_external._find_module_shim
 
     def invalidate_caches(self):
         """An optional method for clearing the finder's cache, if any.
@@ -144,7 +152,7 @@ def load_module(self, fullname):
         """
         if not hasattr(self, 'exec_module'):
             raise ImportError
-        return _bootstrap._load_module_shim(self, fullname)
+        return _bootstrap_external._load_module_shim(self, fullname)
 
     def module_repr(self, module):
         """Return a module's repr.
@@ -222,8 +230,8 @@ def source_to_code(data, path='<string>'):
         argument should be where the data was retrieved (when applicable)."""
         return compile(data, path, 'exec', dont_inherit=True)
 
-    exec_module = _bootstrap._LoaderBasics.exec_module
+    exec_module = _bootstrap_external._LoaderBasics.exec_module
-    load_module = _bootstrap._LoaderBasics.load_module
+    load_module = _bootstrap_external._LoaderBasics.load_module
 
 _register(InspectLoader, machinery.BuiltinImporter, machinery.FrozenImporter)
 
@@ -265,7 +273,7 @@ def get_code(self, fullname):
 _register(ExecutionLoader, machinery.ExtensionFileLoader)
 
 
-class FileLoader(_bootstrap.FileLoader, ResourceLoader, ExecutionLoader):
+class FileLoader(_bootstrap_external.FileLoader, ResourceLoader, ExecutionLoader):
 
     """Abstract base class partially implementing the ResourceLoader and
     ExecutionLoader ABCs."""
@@ -274,7 +282,7 @@ class FileLoader(_bootstrap.FileLoader, ResourceLoader, ExecutionLoader):
             machinery.SourcelessFileLoader)
 
 
-class SourceLoader(_bootstrap.SourceLoader, ResourceLoader, ExecutionLoader):
+class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLoader):
 
     """Abstract base class for loading source code (and optionally any
     corresponding bytecode).

Lib/importlib/machinery.py

@@ -2,18 +2,18 @@
 
 import _imp
 
-from ._bootstrap import (SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES,
-                         OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES,
-                         EXTENSION_SUFFIXES)
 from ._bootstrap import ModuleSpec
 from ._bootstrap import BuiltinImporter
 from ._bootstrap import FrozenImporter
-from ._bootstrap import WindowsRegistryFinder
+from ._bootstrap_external import (SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES,
-from ._bootstrap import PathFinder
+                     OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES,
-from ._bootstrap import FileFinder
+                     EXTENSION_SUFFIXES)
-from ._bootstrap import SourceFileLoader
+from ._bootstrap_external import WindowsRegistryFinder
-from ._bootstrap import SourcelessFileLoader
+from ._bootstrap_external import PathFinder
-from ._bootstrap import ExtensionFileLoader
+from ._bootstrap_external import FileFinder
+from ._bootstrap_external import SourceFileLoader
+from ._bootstrap_external import SourcelessFileLoader
+from ._bootstrap_external import ExtensionFileLoader
 
 
 def all_suffixes():

Lib/importlib/util.py

@@ -1,14 +1,14 @@
 """Utility code for constructing importers, etc."""
 from . import abc
-from ._bootstrap import MAGIC_NUMBER
-from ._bootstrap import cache_from_source
-from ._bootstrap import decode_source
 from ._bootstrap import module_from_spec
-from ._bootstrap import source_from_cache
-from ._bootstrap import spec_from_loader
-from ._bootstrap import spec_from_file_location
 from ._bootstrap import _resolve_name
+from ._bootstrap import spec_from_loader
 from ._bootstrap import _find_spec
+from ._bootstrap_external import MAGIC_NUMBER
+from ._bootstrap_external import cache_from_source
+from ._bootstrap_external import decode_source
+from ._bootstrap_external import source_from_cache
+from ._bootstrap_external import spec_from_file_location
 
 from contextlib import contextmanager
 import functools

Lib/modulefinder.py

@@ -1,7 +1,7 @@
 """Find modules used by a script, using introspection."""
 
 import dis
-import importlib._bootstrap
+import importlib._bootstrap_external
 import importlib.machinery
 import marshal
 import os
@@ -289,7 +289,7 @@ def load_module(self, fqname, fp, pathname, file_info):
             co = compile(fp.read()+'\n', pathname, 'exec')
         elif type == imp.PY_COMPILED:
             try:
-                marshal_data = importlib._bootstrap._validate_bytecode_header(fp.read())
+                marshal_data = importlib._bootstrap_external._validate_bytecode_header(fp.read())
             except ImportError as exc:
                 self.msgout(2, "raise ImportError: " + str(exc), pathname)
                 raise

Lib/py_compile.py

@@ -3,7 +3,7 @@
 This module has intimate knowledge of the format of .pyc files.
 """
 
-import importlib._bootstrap
+import importlib._bootstrap_external
 import importlib.machinery
 import importlib.util
 import os
@@ -137,10 +137,10 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1):
     except FileExistsError:
         pass
     source_stats = loader.path_stats(file)
-    bytecode = importlib._bootstrap._code_to_bytecode(
+    bytecode = importlib._bootstrap_external._code_to_bytecode(
             code, source_stats['mtime'], source_stats['size'])
-    mode = importlib._bootstrap._calc_mode(file)
+    mode = importlib._bootstrap_external._calc_mode(file)
-    importlib._bootstrap._write_atomic(cfile, bytecode, mode)
+    importlib._bootstrap_external._write_atomic(cfile, bytecode, mode)
     return cfile
 
 

Lib/pydoc.py

@@ -53,6 +53,7 @@ class or function within a module or module in a package.  If the
 
 import builtins
 import importlib._bootstrap
+import importlib._bootstrap_external
 import importlib.machinery
 import importlib.util
 import inspect
@@ -292,9 +293,9 @@ def importfile(path):
     filename = os.path.basename(path)
     name, ext = os.path.splitext(filename)
     if is_bytecode:
-        loader = importlib._bootstrap.SourcelessFileLoader(name, path)
+        loader = importlib._bootstrap_external.SourcelessFileLoader(name, path)
     else:
-        loader = importlib._bootstrap.SourceFileLoader(name, path)
+        loader = importlib._bootstrap_external.SourceFileLoader(name, path)
     # XXX We probably don't need to pass in the loader here.
     spec = importlib.util.spec_from_file_location(name, path, loader=loader)
     try:

Lib/runpy.py

@@ -58,7 +58,7 @@ def __exit__(self, *args):
         self.value = self._sentinel
         sys.argv[0] = self._saved_value
 
-# TODO: Replace these helpers with importlib._bootstrap functions
+# TODO: Replace these helpers with importlib._bootstrap_external functions.
 def _run_code(code, run_globals, init_globals=None,
               mod_name=None, mod_spec=None,
               pkg_name=None, script_name=None):

Lib/site.py

@@ -98,8 +98,8 @@ def makepath(*paths):
 def abs_paths():
     """Set all module __file__ and __cached__ attributes to an absolute path"""
     for m in set(sys.modules.values()):
-        if (getattr(getattr(m, '__loader__', None), '__module__', None) !=
+        if (getattr(getattr(m, '__loader__', None), '__module__', None) not in
-                '_frozen_importlib'):
+                ('_frozen_importlib', '_frozen_importlib_external')):
             continue   # don't mess with a PEP 302-supplied __file__
         try:
             m.__file__ = os.path.abspath(m.__file__)

Lib/test/test_import/__init__.py

@@ -1,7 +1,7 @@
 # We import importlib *ASAP* in order to test #15386
 import importlib
 import importlib.util
-from importlib._bootstrap import _get_sourcefile
+from importlib._bootstrap_external import _get_sourcefile
 import builtins
 import marshal
 import os
@@ -845,19 +845,27 @@ def test_frozen_importlib_is_bootstrap(self):
         self.assertEqual(mod.__package__, 'importlib')
         self.assertTrue(mod.__file__.endswith('_bootstrap.py'), mod.__file__)
 
+    def test_frozen_importlib_external_is_bootstrap_external(self):
+        from importlib import _bootstrap_external
+        mod = sys.modules['_frozen_importlib_external']
+        self.assertIs(mod, _bootstrap_external)
+        self.assertEqual(mod.__name__, 'importlib._bootstrap_external')
+        self.assertEqual(mod.__package__, 'importlib')
+        self.assertTrue(mod.__file__.endswith('_bootstrap_external.py'), mod.__file__)
+
     def test_there_can_be_only_one(self):
         # Issue #15386 revealed a tricky loophole in the bootstrapping
         # This test is technically redundant, since the bug caused importing
         # this test module to crash completely, but it helps prove the point
         from importlib import machinery
         mod = sys.modules['_frozen_importlib']
-        self.assertIs(machinery.FileFinder, mod.FileFinder)
+        self.assertIs(machinery.ModuleSpec, mod.ModuleSpec)
 
 
 @cpython_only
 class GetSourcefileTests(unittest.TestCase):
 
-    """Test importlib._bootstrap._get_sourcefile() as used by the C API.
+    """Test importlib._bootstrap_external._get_sourcefile() as used by the C API.
 
     Because of the peculiarities of the need of this function, the tests are
     knowingly whitebox tests.
@@ -867,7 +875,7 @@ class GetSourcefileTests(unittest.TestCase):
     def test_get_sourcefile(self):
         # Given a valid bytecode path, return the path to the corresponding
         # source file if it exists.
-        with mock.patch('importlib._bootstrap._path_isfile') as _path_isfile:
+        with mock.patch('importlib._bootstrap_external._path_isfile') as _path_isfile:
             _path_isfile.return_value = True;
             path = TESTFN + '.pyc'
             expect = TESTFN + '.py'
@@ -876,7 +884,7 @@ def test_get_sourcefile(self):
     def test_get_sourcefile_no_source(self):
         # Given a valid bytecode path without a corresponding source path,
         # return the original bytecode path.
-        with mock.patch('importlib._bootstrap._path_isfile') as _path_isfile:
+        with mock.patch('importlib._bootstrap_external._path_isfile') as _path_isfile:
             _path_isfile.return_value = False;
             path = TESTFN + '.pyc'
             self.assertEqual(_get_sourcefile(path), path)
@@ -1031,7 +1039,7 @@ def test_import_bug(self):
         # We simulate a bug in importlib and check that it's not stripped
         # away from the traceback.
         self.create_module("foo", "")
-        importlib = sys.modules['_frozen_importlib']
+        importlib = sys.modules['_frozen_importlib_external']
         if 'load_module' in vars(importlib.SourceLoader):
             old_exec_module = importlib.SourceLoader.exec_module
         else:

Lib/test/test_importlib/extension/test_case_sensitivity.py

@@ -1,4 +1,4 @@
-from importlib import _bootstrap
+from importlib import _bootstrap_external
 import sys
 from test import support
 import unittest
@@ -26,7 +26,7 @@ def find_module(self):
     def test_case_sensitive(self):
         with support.EnvironmentVarGuard() as env:
             env.unset('PYTHONCASEOK')
-            if b'PYTHONCASEOK' in _bootstrap._os.environ:
+            if b'PYTHONCASEOK' in _bootstrap_external._os.environ:
                 self.skipTest('os.environ changes not reflected in '
                               '_os.environ')
             loader = self.find_module()
@@ -35,7 +35,7 @@ def test_case_sensitive(self):
     def test_case_insensitivity(self):
         with support.EnvironmentVarGuard() as env:
             env.set('PYTHONCASEOK', '1')
-            if b'PYTHONCASEOK' not in _bootstrap._os.environ:
+            if b'PYTHONCASEOK' not in _bootstrap_external._os.environ:
                 self.skipTest('os.environ changes not reflected in '
                               '_os.environ')
             loader = self.find_module()

Lib/test/test_importlib/import_/test_path.py

@@ -99,7 +99,7 @@ def test_None_on_sys_path(self):
         new_path_importer_cache.pop(None, None)
         new_path_hooks = [zipimport.zipimporter,
                           self.machinery.FileFinder.path_hook(
-                              *self.importlib._bootstrap._get_supported_file_loaders())]
+                              *self.importlib._bootstrap_external._get_supported_file_loaders())]
         missing = object()
         email = sys.modules.pop('email', missing)
         try:

Lib/test/test_importlib/source/test_case_sensitivity.py

@@ -42,7 +42,7 @@ def sensitivity_test(self):
     def test_sensitive(self):
         with test_support.EnvironmentVarGuard() as env:
             env.unset('PYTHONCASEOK')
-            if b'PYTHONCASEOK' in self.importlib._bootstrap._os.environ:
+            if b'PYTHONCASEOK' in self.importlib._bootstrap_external._os.environ:
                 self.skipTest('os.environ changes not reflected in '
                               '_os.environ')
             sensitive, insensitive = self.sensitivity_test()
@@ -53,7 +53,7 @@ def test_sensitive(self):
     def test_insensitive(self):
         with test_support.EnvironmentVarGuard() as env:
             env.set('PYTHONCASEOK', '1')
-            if b'PYTHONCASEOK' not in self.importlib._bootstrap._os.environ:
+            if b'PYTHONCASEOK' not in self.importlib._bootstrap_external._os.environ:
                 self.skipTest('os.environ changes not reflected in '
                               '_os.environ')
             sensitive, insensitive = self.sensitivity_test()

Lib/test/test_site.py

@@ -355,8 +355,10 @@ def test_abs_paths(self):
         stdout, stderr = proc.communicate()
         self.assertEqual(proc.returncode, 0)
         os__file__, os__cached__ = stdout.splitlines()[:2]
-        self.assertTrue(os.path.isabs(os__file__))
+        self.assertTrue(os.path.isabs(os__file__),
-        self.assertTrue(os.path.isabs(os__cached__))
+                        "expected absolute path, got {}".format(os__file__))
+        self.assertTrue(os.path.isabs(os__cached__),
+                        "expected absolute path, got {}".format(os__cached__))
 
     def test_no_duplicate_paths(self):
         # No duplicate paths should exist in sys.path

Makefile.pre.in

@@ -533,6 +533,7 @@ coverage-report:
 	: # force rebuilding of parser and importlib
 	@touch $(GRAMMAR_INPUT)
 	@touch $(srcdir)/Lib/importlib/_bootstrap.py
+	@touch $(srcdir)/Lib/importlib/_bootstrap_external.py
 	: # build with coverage info
 	$(MAKE) coverage
 	: # run tests, ignore failures
@@ -694,6 +695,10 @@ Programs/_freeze_importlib.o: Programs/_freeze_importlib.c Makefile
 Programs/_freeze_importlib: Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN)
 	$(LINKCC) $(PY_LDFLAGS) -o $@ Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST)
 
+Python/importlib_external.h: $(srcdir)/Lib/importlib/_bootstrap_external.py Programs/_freeze_importlib
+	./Programs/_freeze_importlib \
+		$(srcdir)/Lib/importlib/_bootstrap_external.py Python/importlib_external.h
+
 Python/importlib.h: $(srcdir)/Lib/importlib/_bootstrap.py Programs/_freeze_importlib
 	./Programs/_freeze_importlib \
 		$(srcdir)/Lib/importlib/_bootstrap.py Python/importlib.h
@@ -841,7 +846,7 @@ $(OPCODETARGETS_H): $(OPCODETARGETGEN_FILES)
 
 Python/ceval.o: $(OPCODETARGETS_H) $(srcdir)/Python/ceval_gil.h
 
-Python/frozen.o: Python/importlib.h
+Python/frozen.o: Python/importlib.h Python/importlib_external.h
 
 Objects/typeobject.o: Objects/typeslots.inc
 Objects/typeslots.inc: $(srcdir)/Include/typeslots.h $(srcdir)/Objects/typeslots.py

Misc/NEWS

@@ -15,6 +15,9 @@ Core and Builtins
 
 - Issue #23910: Optimize property() getter calls.  Patch by Joe Jevnik.
 
+- Issue #23911: Move path-based importlib bootstrap code to a separate
+  frozen module.
+
 - Issue #24022: Fix tokenizer crash when processing undecodable source code.
 
 - Issue #9951: Added a hex() method to bytes, bytearray, and memoryview.

Programs/_freeze_importlib.c

@@ -12,6 +12,7 @@
 #include <unistd.h>
 #endif
 
+int Py_FrozenFlag = 1;  /* Suppress errors from getpath.c */
 
 /* To avoid a circular dependency on frozen.o, we create our own structure
    of frozen modules instead, left deliberately blank so as to avoid
@@ -33,13 +34,14 @@ const char header[] = "/* Auto-generated by Programs/_freeze_importlib.c */";
 int
 main(int argc, char *argv[])
 {
-    char *inpath, *outpath;
+    char *inpath, *outpath, *code_name;
     FILE *infile = NULL, *outfile = NULL;
     struct _Py_stat_struct status;
     size_t text_size, data_size, n;
     char *text = NULL;
     unsigned char *data;
     PyObject *code = NULL, *marshalled = NULL;
+    int is_bootstrap = 1;
 
     PyImport_FrozenModules = _PyImport_FrozenModules;
 
@@ -82,8 +84,14 @@ main(int argc, char *argv[])
     /* Don't install importlib, since it could execute outdated bytecode. */
     _Py_InitializeEx_Private(1, 0);
 
-    code = Py_CompileStringExFlags(text, "<frozen importlib._bootstrap>",
+    if (strstr(inpath, "_external") != NULL) {
-                                   Py_file_input, NULL, 0);
+        is_bootstrap = 0;
+    }
+
+    code_name = is_bootstrap ?
+        "<frozen importlib._bootstrap>" :
+        "<frozen importlib._bootstrap_external>";
+    code = Py_CompileStringExFlags(text, code_name, Py_file_input, NULL, 0);
     if (code == NULL)
         goto error;
     free(text);
@@ -106,7 +114,11 @@ main(int argc, char *argv[])
         goto error;
     }
     fprintf(outfile, "%s\n", header);
-    fprintf(outfile, "const unsigned char _Py_M__importlib[] = {\n");
+    if (is_bootstrap)
+        fprintf(outfile, "const unsigned char _Py_M__importlib[] = {\n");
+    else
+        fprintf(outfile,
+                "const unsigned char _Py_M__importlib_external[] = {\n");
     for (n = 0; n < data_size; n += 16) {
         size_t i, end = Py_MIN(n + 16, data_size);
         fprintf(outfile, "    ");

Python/frozen.c

@@ -3,6 +3,7 @@
 
 #include "Python.h"
 #include "importlib.h"
+#include "importlib_external.h"
 
 /* In order to test the support for frozen modules, by default we
    define a single frozen module, __hello__.  Loading it will print
@@ -31,6 +32,8 @@ static unsigned char M___hello__[] = {
 static const struct _frozen _PyImport_FrozenModules[] = {
     /* importlib */
     {"_frozen_importlib", _Py_M__importlib, (int)sizeof(_Py_M__importlib)},
+    {"_frozen_importlib_external", _Py_M__importlib_external,
+        (int)sizeof(_Py_M__importlib_external)},
     /* Test module */
     {"__hello__", M___hello__, SIZE},
     /* Test package (negative size indicates package-ness) */

Python/import.c

@@ -491,8 +491,13 @@ PyImport_GetMagicNumber(void)
 {
     long res;
     PyInterpreterState *interp = PyThreadState_Get()->interp;
-    PyObject *pyc_magic = PyObject_GetAttrString(interp->importlib,
+    PyObject *external, *pyc_magic;
-                                                 "_RAW_MAGIC_NUMBER");
+
+    external = PyObject_GetAttrString(interp->importlib, "_bootstrap_external");
+    if (external == NULL)
+        return -1;
+    pyc_magic = PyObject_GetAttrString(external, "_RAW_MAGIC_NUMBER");
+    Py_DECREF(external);
     if (pyc_magic == NULL)
         return -1;
     res = PyLong_AsLong(pyc_magic);
@@ -737,7 +742,7 @@ PyImport_ExecCodeModuleWithPathnames(const char *name, PyObject *co,
                                      const char *cpathname)
 {
     PyObject *m = NULL;
-    PyObject *nameobj, *pathobj = NULL, *cpathobj = NULL;
+    PyObject *nameobj, *pathobj = NULL, *cpathobj = NULL, *external= NULL;
 
     nameobj = PyUnicode_FromString(name);
     if (nameobj == NULL)
@@ -765,9 +770,14 @@ PyImport_ExecCodeModuleWithPathnames(const char *name, PyObject *co,
                           "no interpreter!");
         }
 
-        pathobj = _PyObject_CallMethodIdObjArgs(interp->importlib,
+        external= PyObject_GetAttrString(interp->importlib,
-                                                &PyId__get_sourcefile, cpathobj,
+                                         "_bootstrap_external");
-                                                NULL);
+        if (external != NULL) {
+            pathobj = _PyObject_CallMethodIdObjArgs(external,
+                                                    &PyId__get_sourcefile, cpathobj,
+                                                    NULL);
+            Py_DECREF(external);
+        }
         if (pathobj == NULL)
             PyErr_Clear();
     }
@@ -833,7 +843,7 @@ PyObject*
 PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname,
                               PyObject *cpathname)
 {
-    PyObject *d, *res;
+    PyObject *d, *external, *res;
     PyInterpreterState *interp = PyThreadState_GET()->interp;
     _Py_IDENTIFIER(_fix_up_module);
 
@@ -845,9 +855,13 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname,
     if (pathname == NULL) {
         pathname = ((PyCodeObject *)co)->co_filename;
     }
-    res = _PyObject_CallMethodIdObjArgs(interp->importlib,
+    external = PyObject_GetAttrString(interp->importlib, "_bootstrap_external");
+    if (external == NULL)
+        return NULL;
+    res = _PyObject_CallMethodIdObjArgs(external,
                                         &PyId__fix_up_module,
                                         d, name, pathname, cpathname, NULL);
+    Py_DECREF(external);
     if (res != NULL) {
         Py_DECREF(res);
         res = exec_code_in_module(name, d, co);
@@ -1245,6 +1259,7 @@ static void
 remove_importlib_frames(void)
 {
     const char *importlib_filename = "<frozen importlib._bootstrap>";
+    const char *external_filename = "<frozen importlib._bootstrap_external>";
     const char *remove_frames = "_call_with_frames_removed";
     int always_trim = 0;
     int in_importlib = 0;
@@ -1274,7 +1289,10 @@ remove_importlib_frames(void)
         assert(PyTraceBack_Check(tb));
         now_in_importlib = (PyUnicode_CompareWithASCIIString(
                                 code->co_filename,
-                                importlib_filename) == 0);
+                                importlib_filename) == 0) ||
+                           (PyUnicode_CompareWithASCIIString(
+                                code->co_filename,
+                                external_filename) == 0);
         if (now_in_importlib && !in_importlib) {
             /* This is the link to this chunk of importlib tracebacks */
             outer_link = prev_link;

Python/pythonrun.c

@@ -304,7 +304,7 @@ set_main_loader(PyObject *d, const char *filename, const char *loader_name)
 {
     PyInterpreterState *interp;
     PyThreadState *tstate;
-    PyObject *filename_obj, *loader_type, *loader;
+    PyObject *filename_obj, *bootstrap, *loader_type = NULL, *loader;
     int result = 0;
 
     filename_obj = PyUnicode_DecodeFSDefault(filename);
@@ -313,7 +313,12 @@ set_main_loader(PyObject *d, const char *filename, const char *loader_name)
     /* Get current thread state and interpreter pointer */
     tstate = PyThreadState_GET();
     interp = tstate->interp;
-    loader_type = PyObject_GetAttrString(interp->importlib, loader_name);
+    bootstrap = PyObject_GetAttrString(interp->importlib,
+                                       "_bootstrap_external");
+    if (bootstrap != NULL) {
+        loader_type = PyObject_GetAttrString(bootstrap, loader_name);
+        Py_DECREF(bootstrap);
+    }
     if (loader_type == NULL) {
         Py_DECREF(filename_obj);
         return -1;

Tools/freeze/freeze.py

@@ -366,8 +366,10 @@ def main():
             mf.load_file(mod)
 
     # Alias "importlib._bootstrap" to "_frozen_importlib" so that the
-    # import machinery can bootstrap.
+    # import machinery can bootstrap.  Do the same for
+    # importlib._bootstrap_external.
     mf.modules["_frozen_importlib"] = mf.modules["importlib._bootstrap"]
+    mf.modules["_frozen_importlib_external"] = mf.modules["importlib._bootstrap_external"]
 
     # Add the main script as either __main__, or the actual module name.
     if python_entry_is_main: