bpo-45020: Add -X frozen_modules=[on|off] to explicitly control use of frozen modules. (gh-28320)

Currently we freeze several modules into the runtime. For each of these modules it is essential to bootstrapping the runtime that they be frozen. Any other stdlib module that we later freeze into the runtime is not essential. We can just as well import from the .py file.  This PR lets users explicitly choose which should be used, with the new "-X frozen_modules=[on|off]" CLI flag. The default is "off" for now.

https://bugs.python.org/issue45020

Python/clinic/import.c.h

@@ -315,6 +315,37 @@ _imp__frozen_module_names(PyObject *module, PyObject *Py_UNUSED(ignored))
     return _imp__frozen_module_names_impl(module);
 }
 
+PyDoc_STRVAR(_imp__override_frozen_modules_for_tests__doc__,
+"_override_frozen_modules_for_tests($module, override, /)\n"
+"--\n"
+"\n"
+"(internal-only) Override PyConfig.use_frozen_modules.\n"
+"\n"
+"(-1: \"off\", 1: \"on\", 0: no override)\n"
+"See frozen_modules() in Lib/test/support/import_helper.py.");
+
+#define _IMP__OVERRIDE_FROZEN_MODULES_FOR_TESTS_METHODDEF    \
+    {"_override_frozen_modules_for_tests", (PyCFunction)_imp__override_frozen_modules_for_tests, METH_O, _imp__override_frozen_modules_for_tests__doc__},
+
+static PyObject *
+_imp__override_frozen_modules_for_tests_impl(PyObject *module, int override);
+
+static PyObject *
+_imp__override_frozen_modules_for_tests(PyObject *module, PyObject *arg)
+{
+    PyObject *return_value = NULL;
+    int override;
+
+    override = _PyLong_AsInt(arg);
+    if (override == -1 && PyErr_Occurred()) {
+        goto exit;
+    }
+    return_value = _imp__override_frozen_modules_for_tests_impl(module, override);
+
+exit:
+    return return_value;
+}
+
 #if defined(HAVE_DYNAMIC_LOADING)
 
 PyDoc_STRVAR(_imp_create_dynamic__doc__,
@@ -467,4 +498,4 @@ exit:
 #ifndef _IMP_EXEC_DYNAMIC_METHODDEF
     #define _IMP_EXEC_DYNAMIC_METHODDEF
 #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */
-/*[clinic end generated code: output=0ab3fa7c5808bba4 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=96038c277119d6e3 input=a9049054013a1b77]*/

Python/import.c

@@ -1050,17 +1050,58 @@ _imp_create_builtin(PyObject *module, PyObject *spec)
 
 /* Frozen modules */
 
+static bool
+is_essential_frozen_module(const char *name)
+{
+    /* These modules are necessary to bootstrap the import system. */
+    if (strcmp(name, "_frozen_importlib") == 0) {
+        return true;
+    }
+    if (strcmp(name, "_frozen_importlib_external") == 0) {
+        return true;
+    }
+    if (strcmp(name, "zipimport") == 0) {
+        return true;
+    }
+    /* This doesn't otherwise have anywhere to find the module.
+       See frozenmain.c. */
+    if (strcmp(name, "__main__") == 0) {
+        return true;
+    }
+    return false;
+}
+
+static bool
+use_frozen(void)
+{
+    PyInterpreterState *interp = _PyInterpreterState_GET();
+    int override = interp->override_frozen_modules;
+    if (override > 0) {
+        return true;
+    }
+    else if (override < 0) {
+        return false;
+    }
+    else {
+        return interp->config.use_frozen_modules;
+    }
+}
+
 static PyObject *
-list_frozen_module_names(bool force)
+list_frozen_module_names()
 {
     PyObject *names = PyList_New(0);
     if (names == NULL) {
         return NULL;
     }
+    bool enabled = use_frozen();
     for (const struct _frozen *p = PyImport_FrozenModules; ; p++) {
         if (p->name == NULL) {
             break;
         }
+        if (!enabled && !is_essential_frozen_module(p->name)) {
+            continue;
+        }
         PyObject *name = PyUnicode_FromString(p->name);
         if (name == NULL) {
             Py_DECREF(names);
@@ -1077,18 +1118,27 @@ list_frozen_module_names(bool force)
 }
 
 static const struct _frozen *
-find_frozen(PyObject *name)
+find_frozen(PyObject *modname)
 {
-    const struct _frozen *p;
-
-    if (name == NULL)
+    if (modname == NULL) {
         return NULL;
-
+    }
+    const char *name = PyUnicode_AsUTF8(modname);
+    if (name == NULL) {
+        PyErr_Clear();
+        return NULL;
+    }
+    if (!use_frozen() && !is_essential_frozen_module(name)) {
+        return NULL;
+    }
+    const struct _frozen *p;
     for (p = PyImport_FrozenModules; ; p++) {
-        if (p->name == NULL)
+        if (p->name == NULL) {
             return NULL;
-        if (_PyUnicode_EqualToASCIIString(name, p->name))
+        }
+        if (strcmp(name, p->name) == 0) {
             break;
+        }
     }
     return p;
 }
@@ -1991,7 +2041,28 @@ static PyObject *
 _imp__frozen_module_names_impl(PyObject *module)
 /*[clinic end generated code: output=80609ef6256310a8 input=76237fbfa94460d2]*/
 {
-    return list_frozen_module_names(true);
+    return list_frozen_module_names();
+}
+
+/*[clinic input]
+_imp._override_frozen_modules_for_tests
+
+    override: int
+    /
+
+(internal-only) Override PyConfig.use_frozen_modules.
+
+(-1: "off", 1: "on", 0: no override)
+See frozen_modules() in Lib/test/support/import_helper.py.
+[clinic start generated code]*/
+
+static PyObject *
+_imp__override_frozen_modules_for_tests_impl(PyObject *module, int override)
+/*[clinic end generated code: output=36d5cb1594160811 input=8f1f95a3ef21aec3]*/
+{
+    PyInterpreterState *interp = _PyInterpreterState_GET();
+    interp->override_frozen_modules = override;
+    Py_RETURN_NONE;
 }
 
 /* Common implementation for _imp.exec_dynamic and _imp.exec_builtin */
@@ -2155,6 +2226,7 @@ static PyMethodDef imp_methods[] = {
     _IMP_IS_BUILTIN_METHODDEF
     _IMP_IS_FROZEN_METHODDEF
     _IMP__FROZEN_MODULE_NAMES_METHODDEF
+    _IMP__OVERRIDE_FROZEN_MODULES_FOR_TESTS_METHODDEF
     _IMP_CREATE_DYNAMIC_METHODDEF
     _IMP_EXEC_DYNAMIC_METHODDEF
     _IMP_EXEC_BUILTIN_METHODDEF

Python/initconfig.c

@@ -100,6 +100,8 @@ static const char usage_3[] = "\
             instruction in code objects. This is useful when smaller code objects and pyc \n\
             files are desired as well as supressing the extra visual location indicators \n\
             when the interpreter displays tracebacks.\n\
+         -X frozen_modules=[on|off]: whether or not frozen modules should be used.\n\
+            The default is \"on\" (or \"off\" if you are running a local build).\n\
 \n\
 --check-hash-based-pycs always|default|never:\n\
     control how Python invalidates hash-based .pyc files\n\
@@ -949,6 +951,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2)
     COPY_ATTR(pathconfig_warnings);
     COPY_ATTR(_init_main);
     COPY_ATTR(_isolated_interpreter);
+    COPY_ATTR(use_frozen_modules);
     COPY_WSTRLIST(orig_argv);
 
 #undef COPY_ATTR
@@ -1052,6 +1055,7 @@ _PyConfig_AsDict(const PyConfig *config)
     SET_ITEM_INT(_init_main);
     SET_ITEM_INT(_isolated_interpreter);
     SET_ITEM_WSTRLIST(orig_argv);
+    SET_ITEM_INT(use_frozen_modules);
 
     return dict;
 
@@ -1334,6 +1338,7 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict)
     GET_UINT(_install_importlib);
     GET_UINT(_init_main);
     GET_UINT(_isolated_interpreter);
+    GET_UINT(use_frozen_modules);
 
 #undef CHECK_VALUE
 #undef GET_UINT
@@ -1590,6 +1595,17 @@ config_get_xoption(const PyConfig *config, wchar_t *name)
     return _Py_get_xoption(&config->xoptions, name);
 }
 
+static const wchar_t*
+config_get_xoption_value(const PyConfig *config, wchar_t *name)
+{
+    const wchar_t *xoption = config_get_xoption(config, name);
+    if (xoption == NULL) {
+        return NULL;
+    }
+    const wchar_t *sep = wcschr(xoption, L'=');
+    return sep ? sep + 1 : L"";
+}
+
 
 static PyStatus
 config_init_home(PyConfig *config)
@@ -2065,6 +2081,48 @@ config_init_fs_encoding(PyConfig *config, const PyPreConfig *preconfig)
 }
 
 
+static PyStatus
+config_init_import(PyConfig *config, int compute_path_config)
+{
+    PyStatus status;
+
+    status = _PyConfig_InitPathConfig(config, compute_path_config);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+
+    /* -X frozen_modules=[on|off] */
+    const wchar_t *value = config_get_xoption_value(config, L"frozen_modules");
+    if (value == NULL) {
+        // For now we always default to "off".
+        // In the near future we will be factoring in PGO and in-development.
+        config->use_frozen_modules = 0;
+    }
+    else if (wcscmp(value, L"on") == 0) {
+        config->use_frozen_modules = 1;
+    }
+    else if (wcscmp(value, L"off") == 0) {
+        config->use_frozen_modules = 0;
+    }
+    else if (wcslen(value) == 0) {
+        // "-X frozen_modules" and "-X frozen_modules=" both imply "on".
+        config->use_frozen_modules = 1;
+    }
+    else {
+        return PyStatus_Error("bad value for option -X frozen_modules "
+                              "(expected \"on\" or \"off\")");
+    }
+
+    return _PyStatus_OK();
+}
+
+PyStatus
+_PyConfig_InitImportConfig(PyConfig *config)
+{
+    return config_init_import(config, 1);
+}
+
+
 static PyStatus
 config_read(PyConfig *config, int compute_path_config)
 {
@@ -2111,7 +2169,7 @@ config_read(PyConfig *config, int compute_path_config)
     }
 
     if (config->_install_importlib) {
-        status = _PyConfig_InitPathConfig(config, compute_path_config);
+        status = config_init_import(config, compute_path_config);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }

Python/pylifecycle.c

@@ -1078,8 +1078,8 @@ init_interp_main(PyThreadState *tstate)
         return _PyStatus_OK();
     }
 
-    // Compute the path configuration
-    status = _PyConfig_InitPathConfig(&interp->config, 1);
+    // Initialize the import-related configuration.
+    status = _PyConfig_InitImportConfig(&interp->config);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }