[3.12] gh-100238: Use setuptools in peg-generator and reenable tests (GH-104798) (#105135)

(cherry picked from commit afa759fb80)

Co-authored-by: Lysandros Nikolaou <lisandrosnik@gmail.com>

Lib/test/support/__init__.py

@@ -1868,15 +1868,16 @@ def missing_compiler_executable(cmd_names=[]):
     missing.
 
     """
-    # TODO (PEP 632): alternate check without using distutils
-    from distutils import ccompiler, sysconfig, spawn, errors
+    from setuptools._distutils import ccompiler, sysconfig, spawn
+    from setuptools import errors
+
     compiler = ccompiler.new_compiler()
     sysconfig.customize_compiler(compiler)
     if compiler.compiler_type == "msvc":
         # MSVC has no executables, so check whether initialization succeeds
         try:
             compiler.initialize()
-        except errors.DistutilsPlatformError:
+        except errors.PlatformError:
             return "msvc"
     for name in compiler.executables:
         if cmd_names and name not in cmd_names:

Lib/test/test_peg_generator/__init__.py

@@ -3,9 +3,6 @@
 from test import support
 from test.support import load_package_tests
 
-# TODO: gh-92584: peg_generator uses distutils which was removed in Python 3.12
-raise unittest.SkipTest("distutils has been removed in Python 3.12")
-
 
 if support.check_sanitizer(address=True, memory=True):
     # bpo-46633: Skip the test because it is too slow when Python is built

Lib/test/test_peg_generator/test_c_parser.py

@@ -1,3 +1,5 @@
+import contextlib
+import subprocess
 import sysconfig
 import textwrap
 import unittest
@@ -8,7 +10,7 @@
 
 from test import test_tools
 from test import support
-from test.support import os_helper
+from test.support import os_helper, import_helper
 from test.support.script_helper import assert_python_ok
 
 _py_cflags_nodist = sysconfig.get_config_var("PY_CFLAGS_NODIST")
@@ -88,6 +90,16 @@ def setUpClass(cls):
         cls.library_dir = tempfile.mkdtemp(dir=cls.tmp_base)
         cls.addClassCleanup(shutil.rmtree, cls.library_dir)
 
+        with contextlib.ExitStack() as stack:
+            python_exe = stack.enter_context(support.setup_venv_with_pip_setuptools_wheel("venv"))
+            sitepackages = subprocess.check_output(
+                [python_exe, "-c", "import sysconfig; print(sysconfig.get_path('platlib'))"],
+                text=True,
+            ).strip()
+            stack.enter_context(import_helper.DirsOnSysPath(sitepackages))
+            cls.addClassCleanup(stack.pop_all().close)
+
+    @support.requires_venv_with_pip()
     def setUp(self):
         self._backup_config_vars = dict(sysconfig._CONFIG_VARS)
         cmd = support.missing_compiler_executable()

Lib/test/test_peg_generator/test_pegen.py

@@ -794,7 +794,7 @@ def test_soft_keyword(self) -> None:
         start:
             | "number" n=NUMBER { eval(n.string) }
             | "string" n=STRING { n.string }
-            | SOFT_KEYWORD l=NAME n=(NUMBER | NAME | STRING) { f"{l.string} = {n.string}"}
+            | SOFT_KEYWORD l=NAME n=(NUMBER | NAME | STRING) { l.string + " = " + n.string }
         """
         parser_class = make_parser(grammar)
         self.assertEqual(parse_string("number 1", parser_class), 1)

Tools/peg_generator/pegen/build.py

@@ -1,4 +1,5 @@
 import itertools
+import os
 import pathlib
 import sys
 import sysconfig
@@ -27,6 +28,46 @@ def get_extra_flags(compiler_flags: str, compiler_py_flags_nodist: str) -> List[
     return f"{flags} {py_flags_nodist}".split()
 
 
+def fixup_build_ext(cmd):
+    """Function needed to make build_ext tests pass.
+
+    When Python was built with --enable-shared on Unix, -L. is not enough to
+    find libpython<blah>.so, because regrtest runs in a tempdir, not in the
+    source directory where the .so lives.
+
+    When Python was built with in debug mode on Windows, build_ext commands
+    need their debug attribute set, and it is not done automatically for
+    some reason.
+
+    This function handles both of these things.  Example use:
+
+        cmd = build_ext(dist)
+        support.fixup_build_ext(cmd)
+        cmd.ensure_finalized()
+
+    Unlike most other Unix platforms, Mac OS X embeds absolute paths
+    to shared libraries into executables, so the fixup is not needed there.
+
+    Taken from distutils (was part of the CPython stdlib until Python 3.11)
+    """
+    if os.name == 'nt':
+        cmd.debug = sys.executable.endswith('_d.exe')
+    elif sysconfig.get_config_var('Py_ENABLE_SHARED'):
+        # To further add to the shared builds fun on Unix, we can't just add
+        # library_dirs to the Extension() instance because that doesn't get
+        # plumbed through to the final compiler command.
+        runshared = sysconfig.get_config_var('RUNSHARED')
+        if runshared is None:
+            cmd.library_dirs = ['.']
+        else:
+            if sys.platform == 'darwin':
+                cmd.library_dirs = []
+            else:
+                name, equals, value = runshared.partition('=')
+                cmd.library_dirs = [d for d in value.split(os.pathsep) if d]
+
+
+
 def compile_c_extension(
     generated_source_path: str,
     build_dir: Optional[str] = None,
@@ -49,16 +90,15 @@ def compile_c_extension(
     static library of the common parser sources (this is useful in case you are
     creating multiple extensions).
     """
-    import distutils.log
-    from distutils.core import Distribution, Extension
-    from distutils.tests.support import fixup_build_ext  # type: ignore
+    import setuptools.logging
 
-    from distutils.ccompiler import new_compiler
-    from distutils.dep_util import newer_group
-    from distutils.sysconfig import customize_compiler
+    from setuptools import Extension, Distribution
+    from setuptools._distutils.dep_util import newer_group
+    from setuptools._distutils.ccompiler import new_compiler
+    from setuptools._distutils.sysconfig import customize_compiler
 
     if verbose:
-        distutils.log.set_threshold(distutils.log.DEBUG)
+        setuptools.logging.set_threshold(setuptools.logging.logging.DEBUG)
 
     source_file_path = pathlib.Path(generated_source_path)
     extension_name = source_file_path.stem