bpo-40334: PEP 617 implementation: New PEG parser for CPython (GH-19503)

Co-authored-by: Guido van Rossum <guido@python.org>
Co-authored-by: Lysandros Nikolaou <lisandrosnik@gmail.com>

Tools/peg_generator/scripts/test_pypi_packages.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3.8
+
+import argparse
+import os
+import glob
+import tarfile
+import zipfile
+import shutil
+import sys
+
+from typing import Generator, Any
+
+sys.path.insert(0, ".")
+from pegen import build
+from scripts import test_parse_directory
+
+argparser = argparse.ArgumentParser(
+    prog="test_pypi_packages", description="Helper program to test parsing PyPI packages",
+)
+argparser.add_argument(
+    "-t", "--tree", action="count", help="Compare parse tree to official AST", default=0
+)
+
+
+def get_packages() -> Generator[str, None, None]:
+    all_packages = (
+        glob.glob("./data/pypi/*.tar.gz")
+        + glob.glob("./data/pypi/*.zip")
+        + glob.glob("./data/pypi/*.tgz")
+    )
+    for package in all_packages:
+        yield package
+
+
+def extract_files(filename: str) -> None:
+    savedir = os.path.join("data", "pypi")
+    if tarfile.is_tarfile(filename):
+        tarfile.open(filename).extractall(savedir)
+    elif zipfile.is_zipfile(filename):
+        zipfile.ZipFile(filename).extractall(savedir)
+    else:
+        raise ValueError(f"Could not identify type of compressed file {filename}")
+
+
+def find_dirname(package_name: str) -> str:
+    for name in os.listdir(os.path.join("data", "pypi")):
+        full_path = os.path.join("data", "pypi", name)
+        if os.path.isdir(full_path) and name in package_name:
+            return full_path
+    assert False  # This is to fix mypy, should never be reached
+
+
+def run_tests(dirname: str, tree: int, extension: Any) -> int:
+    return test_parse_directory.parse_directory(
+        dirname,
+        "data/python.gram",
+        verbose=False,
+        excluded_files=[
+            "*/failset/*",
+            "*/failset/**",
+            "*/failset/**/*",
+            "*/test2to3/*",
+            "*/test2to3/**/*",
+            "*/bad*",
+            "*/lib2to3/tests/data/*",
+        ],
+        skip_actions=False,
+        tree_arg=tree,
+        short=True,
+        extension=extension,
+    )
+
+
+def main() -> None:
+    args = argparser.parse_args()
+    tree = args.tree
+
+    extension = build.build_parser_and_generator(
+        "data/python.gram", "peg_parser/parse.c", compile_extension=True
+    )
+    for package in get_packages():
+        print(f"Extracting files from {package}... ", end="")
+        try:
+            extract_files(package)
+            print("Done")
+        except ValueError as e:
+            print(e)
+            continue
+
+        print(f"Trying to parse all python files ... ")
+        dirname = find_dirname(package)
+        status = run_tests(dirname, tree, extension)
+        if status == 0:
+            print("Done")
+            shutil.rmtree(dirname)
+        else:
+            print(f"Failed to parse {dirname}")
+
+
+if __name__ == "__main__":
+    main()