Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2026-01-06 15:32:22 +00:00
Code Issues Projects Releases Packages Wiki Activity

gh-100238: Use setuptools in peg-generator and reenable tests (#104798)

Browse source
This commit is contained in:
Lysandros Nikolaou 2023-05-26 15:32:03 +02:00 • committed by GitHub
parent e399f46a77
commit afa759fb80
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 163 additions and 102 deletions
Download patch file Download diff file Expand all files Collapse all files

43
Lib/test/support/__init__.py
View file

@ -1865,15 +1865,16 @@ def missing_compiler_executable(cmd_names=[]):
missing.
"""
# TODO (PEP 632): alternate check without using distutils
from distutils import ccompiler, sysconfig, spawn, errors
from setuptools._distutils import ccompiler, sysconfig, spawn
from setuptools import errors
compiler = ccompiler.new_compiler()
sysconfig.customize_compiler(compiler)
if compiler.compiler_type == "msvc":
# MSVC has no executables, so check whether initialization succeeds
try:
compiler.initialize()
except errors.DistutilsPlatformError:
except errors.PlatformError:
return "msvc"
for name in compiler.executables:
if cmd_names and name not in cmd_names:
@ -2270,6 +2271,42 @@ def requires_venv_with_pip():
return unittest.skipUnless(ctypes, 'venv: pip requires ctypes')
# Context manager that creates a virtual environment, install setuptools and wheel in it
# and returns the path to the venv directory and the path to the python executable
@contextlib.contextmanager
def setup_venv_with_pip_setuptools_wheel(venv_dir):
import subprocess
from .os_helper import temp_cwd
with temp_cwd() as temp_dir:
# Create virtual environment to get setuptools
cmd = [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir]
if verbose:
print()
print('Run:', ' '.join(cmd))
subprocess.run(cmd, check=True)
venv = os.path.join(temp_dir, venv_dir)
# Get the Python executable of the venv
python_exe = os.path.basename(sys.executable)
if sys.platform == 'win32':
python = os.path.join(venv, 'Scripts', python_exe)
else:
python = os.path.join(venv, 'bin', python_exe)
cmd = [python, '-X', 'dev',
'-m', 'pip', 'install',
findfile('setuptools-67.6.1-py3-none-any.whl'),
findfile('wheel-0.40.0-py3-none-any.whl')]
if verbose:
print()
print('Run:', ' '.join(cmd))
subprocess.run(cmd, check=True)
yield python
# True if Python is built with the Py_DEBUG macro defined: if
# Python is built in debug mode (./configure --with-pydebug).
Py_DEBUG = hasattr(sys, 'gettotalrefcount')

41
Lib/test/test_cppext.py
View file

@ -35,39 +35,20 @@ def test_build_cpp03(self):
# the test uses venv+pip: skip if it's not available
@support.requires_venv_with_pip()
def check_build(self, std_cpp03, extension_name):
# Build in a temporary directory
with os_helper.temp_cwd():
self._check_build(std_cpp03, extension_name)
venv_dir = 'env'
with support.setup_venv_with_pip_setuptools_wheel(venv_dir) as python_exe:
self._check_build(std_cpp03, extension_name, python_exe)
def _check_build(self, std_cpp03, extension_name):
def _check_build(self, std_cpp03, extension_name, python_exe):
pkg_dir = 'pkg'
os.mkdir(pkg_dir)
shutil.copy(SETUP_TESTCPPEXT, os.path.join(pkg_dir, "setup.py"))
venv_dir = 'env'
verbose = support.verbose
# Create virtual environment to get setuptools
cmd = [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir]
if verbose:
print()
print('Run:', ' '.join(cmd))
subprocess.run(cmd, check=True)
# Get the Python executable of the venv
python_exe = 'python'
if sys.executable.endswith('.exe'):
python_exe += '.exe'
if MS_WINDOWS:
python = os.path.join(venv_dir, 'Scripts', python_exe)
else:
python = os.path.join(venv_dir, 'bin', python_exe)
def run_cmd(operation, cmd):
env = os.environ.copy()
env['CPYTHON_TEST_CPP_STD'] = 'c++03' if std_cpp03 else 'c++11'
env['CPYTHON_TEST_EXT_NAME'] = extension_name
if verbose:
if support.verbose:
print('Run:', ' '.join(cmd))
subprocess.run(cmd, check=True, env=env)
else:
@ -81,14 +62,8 @@ def run_cmd(operation, cmd):
self.fail(
f"{operation} failed with exit code {proc.returncode}")
cmd = [python, '-X', 'dev',
'-m', 'pip', 'install',
support.findfile('setuptools-67.6.1-py3-none-any.whl'),
support.findfile('wheel-0.40.0-py3-none-any.whl')]
run_cmd('Install build dependencies', cmd)
# Build and install the C++ extension
cmd = [python, '-X', 'dev',
cmd = [python_exe, '-X', 'dev',
'-m', 'pip', 'install', '--no-build-isolation',
os.path.abspath(pkg_dir)]
run_cmd('Install', cmd)
@ -96,14 +71,14 @@ def run_cmd(operation, cmd):
# Do a reference run. Until we test that running python
# doesn't leak references (gh-94755), run it so one can manually check
# -X showrefcount results against this baseline.
cmd = [python,
cmd = [python_exe,
'-X', 'dev',
'-X', 'showrefcount',
'-c', 'pass']
run_cmd('Reference run', cmd)
# Import the C++ extension
cmd = [python,
cmd = [python_exe,
'-X', 'dev',
'-X', 'showrefcount',
'-c', f"import {extension_name}"]

3
Lib/test/test_peg_generator/__init__.py
View file

@ -3,9 +3,6 @@
from test import support
from test.support import load_package_tests
# TODO: gh-92584: peg_generator uses distutils which was removed in Python 3.12
raise unittest.SkipTest("distutils has been removed in Python 3.12")
if support.check_sanitizer(address=True, memory=True):
# bpo-46633: Skip the test because it is too slow when Python is built

14
Lib/test/test_peg_generator/test_c_parser.py
View file

@ -1,3 +1,5 @@
import contextlib
import subprocess
import sysconfig
import textwrap
import unittest
@ -8,7 +10,7 @@
from test import test_tools
from test import support
from test.support import os_helper
from test.support import os_helper, import_helper
from test.support.script_helper import assert_python_ok
_py_cflags_nodist = sysconfig.get_config_var("PY_CFLAGS_NODIST")
@ -88,6 +90,16 @@ def setUpClass(cls):
cls.library_dir = tempfile.mkdtemp(dir=cls.tmp_base)
cls.addClassCleanup(shutil.rmtree, cls.library_dir)
with contextlib.ExitStack() as stack:
python_exe = stack.enter_context(support.setup_venv_with_pip_setuptools_wheel("venv"))
sitepackages = subprocess.check_output(
[python_exe, "-c", "import sysconfig; print(sysconfig.get_path('platlib'))"],
text=True,
).strip()
stack.enter_context(import_helper.DirsOnSysPath(sitepackages))
cls.addClassCleanup(stack.pop_all().close)
@support.requires_venv_with_pip()
def setUp(self):
self._backup_config_vars = dict(sysconfig._CONFIG_VARS)
cmd = support.missing_compiler_executable()

110
Lib/test/test_peg_generator/test_pegen.py
View file

@ -96,14 +96,14 @@ def test_gather(self) -> None:
[
[
TokenInfo(
NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2\n"
NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2"
),
TokenInfo(
NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2\n"
NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2"
),
],
TokenInfo(
NEWLINE, string="\n", start=(1, 4), end=(1, 5), line="1, 2\n"
NEWLINE, string="\n", start=(1, 4), end=(1, 5), line="1, 2"
),
],
)
@ -119,8 +119,8 @@ def test_expr_grammar(self) -> None:
self.assertEqual(
node,
[
TokenInfo(NUMBER, string="42", start=(1, 0), end=(1, 2), line="42\n"),
TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="42\n"),
TokenInfo(NUMBER, string="42", start=(1, 0), end=(1, 2), line="42"),
TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="42"),
],
)
@ -137,19 +137,19 @@ def test_optional_operator(self) -> None:
[
[
TokenInfo(
NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2\n"
NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2"
),
[
TokenInfo(
OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2\n"
OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2"
),
TokenInfo(
NUMBER, string="2", start=(1, 4), end=(1, 5), line="1 + 2\n"
NUMBER, string="2", start=(1, 4), end=(1, 5), line="1 + 2"
),
],
],
TokenInfo(
NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 + 2\n"
NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 + 2"
),
],
)
@ -158,10 +158,10 @@ def test_optional_operator(self) -> None:
node,
[
[
TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n"),
TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1"),
None,
],
TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"),
TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1"),
],
)
@ -178,11 +178,11 @@ def test_optional_literal(self) -> None:
[
[
TokenInfo(
NUMBER, string="1", start=(1, 0), end=(1, 1), line="1+\n"
NUMBER, string="1", start=(1, 0), end=(1, 1), line="1+"
),
TokenInfo(OP, string="+", start=(1, 1), end=(1, 2), line="1+\n"),
TokenInfo(OP, string="+", start=(1, 1), end=(1, 2), line="1+"),
],
TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="1+\n"),
TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="1+"),
],
)
node = parse_string("1\n", parser_class)
@ -190,10 +190,10 @@ def test_optional_literal(self) -> None:
node,
[
[
TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n"),
TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1"),
None,
],
TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"),
TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1"),
],
)
@ -210,19 +210,19 @@ def test_alt_optional_operator(self) -> None:
[
[
TokenInfo(
NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2\n"
NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2"
),
[
TokenInfo(
OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2\n"
OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2"
),
TokenInfo(
NUMBER, string="2", start=(1, 4), end=(1, 5), line="1 + 2\n"
NUMBER, string="2", start=(1, 4), end=(1, 5), line="1 + 2"
),
],
],
TokenInfo(
NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 + 2\n"
NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 + 2"
),
],
)
@ -231,10 +231,10 @@ def test_alt_optional_operator(self) -> None:
node,
[
[
TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n"),
TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1"),
None,
],
TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"),
TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1"),
],
)
@ -248,17 +248,17 @@ def test_repeat_0_simple(self) -> None:
self.assertEqual(
node,
[
TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 2 3\n"),
TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 2 3"),
[
TokenInfo(
NUMBER, string="2", start=(1, 2), end=(1, 3), line="1 2 3\n"
NUMBER, string="2", start=(1, 2), end=(1, 3), line="1 2 3"
),
TokenInfo(
NUMBER, string="3", start=(1, 4), end=(1, 5), line="1 2 3\n"
NUMBER, string="3", start=(1, 4), end=(1, 5), line="1 2 3"
),
],
TokenInfo(
NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 2 3\n"
NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 2 3"
),
],
)
@ -266,9 +266,9 @@ def test_repeat_0_simple(self) -> None:
self.assertEqual(
node,
[
TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n"),
TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1"),
[],
TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"),
TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1"),
],
)
@ -283,36 +283,36 @@ def test_repeat_0_complex(self) -> None:
node,
[
TokenInfo(
NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2 + 3\n"
NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2 + 3"
),
[
[
TokenInfo(
OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3\n"
OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3"
),
TokenInfo(
NUMBER,
string="2",
start=(1, 4),
end=(1, 5),
line="1 + 2 + 3\n",
line="1 + 2 + 3",
),
],
[
TokenInfo(
OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3\n"
OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3"
),
TokenInfo(
NUMBER,
string="3",
start=(1, 8),
end=(1, 9),
line="1 + 2 + 3\n",
line="1 + 2 + 3",
),
],
],
TokenInfo(
NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3\n"
NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3"
),
],
)
@ -327,17 +327,17 @@ def test_repeat_1_simple(self) -> None:
self.assertEqual(
node,
[
TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 2 3\n"),
TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 2 3"),
[
TokenInfo(
NUMBER, string="2", start=(1, 2), end=(1, 3), line="1 2 3\n"
NUMBER, string="2", start=(1, 2), end=(1, 3), line="1 2 3"
),
TokenInfo(
NUMBER, string="3", start=(1, 4), end=(1, 5), line="1 2 3\n"
NUMBER, string="3", start=(1, 4), end=(1, 5), line="1 2 3"
),
],
TokenInfo(
NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 2 3\n"
NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 2 3"
),
],
)
@ -355,36 +355,36 @@ def test_repeat_1_complex(self) -> None:
node,
[
TokenInfo(
NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2 + 3\n"
NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2 + 3"
),
[
[
TokenInfo(
OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3\n"
OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3"
),
TokenInfo(
NUMBER,
string="2",
start=(1, 4),
end=(1, 5),
line="1 + 2 + 3\n",
line="1 + 2 + 3",
),
],
[
TokenInfo(
OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3\n"
OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3"
),
TokenInfo(
NUMBER,
string="3",
start=(1, 8),
end=(1, 9),
line="1 + 2 + 3\n",
line="1 + 2 + 3",
),
],
],
TokenInfo(
NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3\n"
NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3"
),
],
)
@ -403,17 +403,17 @@ def test_repeat_with_sep_simple(self) -> None:
[
[
TokenInfo(
NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2, 3\n"
NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2, 3"
),
TokenInfo(
NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2, 3\n"
NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2, 3"
),
TokenInfo(
NUMBER, string="3", start=(1, 6), end=(1, 7), line="1, 2, 3\n"
NUMBER, string="3", start=(1, 6), end=(1, 7), line="1, 2, 3"
),
],
TokenInfo(
NEWLINE, string="\n", start=(1, 7), end=(1, 8), line="1, 2, 3\n"
NEWLINE, string="\n", start=(1, 7), end=(1, 8), line="1, 2, 3"
),
],
)
@ -447,28 +447,28 @@ def test_left_recursive(self) -> None:
string="1",
start=(1, 0),
end=(1, 1),
line="1 + 2 + 3\n",
line="1 + 2 + 3",
),
TokenInfo(
OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3\n"
OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3"
),
TokenInfo(
NUMBER,
string="2",
start=(1, 4),
end=(1, 5),
line="1 + 2 + 3\n",
line="1 + 2 + 3",
),
],
TokenInfo(
OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3\n"
OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3"
),
TokenInfo(
NUMBER, string="3", start=(1, 8), end=(1, 9), line="1 + 2 + 3\n"
NUMBER, string="3", start=(1, 8), end=(1, 9), line="1 + 2 + 3"
),
],
TokenInfo(
NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3\n"
NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3"
),
],
)
@ -794,7 +794,7 @@ def test_soft_keyword(self) -> None:
start:
| "number" n=NUMBER { eval(n.string) }
| "string" n=STRING { n.string }
| SOFT_KEYWORD l=NAME n=(NUMBER | NAME | STRING) { f"{l.string} = {n.string}"}
| SOFT_KEYWORD l=NAME n=(NUMBER | NAME | STRING) { l.string + " = " + n.string }
"""
parser_class = make_parser(grammar)
self.assertEqual(parse_string("number 1", parser_class), 1)

54
Tools/peg_generator/pegen/build.py
View file

@ -1,4 +1,5 @@
import itertools
import os
import pathlib
import sys
import sysconfig
@ -27,6 +28,46 @@ def get_extra_flags(compiler_flags: str, compiler_py_flags_nodist: str) -> List[
return f"{flags} {py_flags_nodist}".split()
def fixup_build_ext(cmd):
"""Function needed to make build_ext tests pass.
When Python was built with --enable-shared on Unix, -L. is not enough to
find libpython<blah>.so, because regrtest runs in a tempdir, not in the
source directory where the .so lives.
When Python was built with in debug mode on Windows, build_ext commands
need their debug attribute set, and it is not done automatically for
some reason.
This function handles both of these things. Example use:
cmd = build_ext(dist)
support.fixup_build_ext(cmd)
cmd.ensure_finalized()
Unlike most other Unix platforms, Mac OS X embeds absolute paths
to shared libraries into executables, so the fixup is not needed there.
Taken from distutils (was part of the CPython stdlib until Python 3.11)
"""
if os.name == 'nt':
cmd.debug = sys.executable.endswith('_d.exe')
elif sysconfig.get_config_var('Py_ENABLE_SHARED'):
# To further add to the shared builds fun on Unix, we can't just add
# library_dirs to the Extension() instance because that doesn't get
# plumbed through to the final compiler command.
runshared = sysconfig.get_config_var('RUNSHARED')
if runshared is None:
cmd.library_dirs = ['.']
else:
if sys.platform == 'darwin':
cmd.library_dirs = []
else:
name, equals, value = runshared.partition('=')
cmd.library_dirs = [d for d in value.split(os.pathsep) if d]
def compile_c_extension(
generated_source_path: str,
build_dir: Optional[str] = None,
@ -49,16 +90,15 @@ def compile_c_extension(
static library of the common parser sources (this is useful in case you are
creating multiple extensions).
"""
import distutils.log
from distutils.core import Distribution, Extension
from distutils.tests.support import fixup_build_ext # type: ignore
import setuptools.logging
from distutils.ccompiler import new_compiler
from distutils.dep_util import newer_group
from distutils.sysconfig import customize_compiler
from setuptools import Extension, Distribution
from setuptools._distutils.dep_util import newer_group
from setuptools._distutils.ccompiler import new_compiler
from setuptools._distutils.sysconfig import customize_compiler
if verbose:
distutils.log.set_threshold(distutils.log.DEBUG)
setuptools.logging.set_threshold(setuptools.logging.logging.DEBUG)
source_file_path = pathlib.Path(generated_source_path)
extension_name = source_file_path.stem