Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-31 13:41:24 +00:00
Code Issues Projects Releases Packages Wiki Activity

bpo-40334: Refactor peg_generator to receive a Tokens file when building c code (GH-19745)

Browse source
This commit is contained in:
Pablo Galindo 2020-04-28 13:11:55 +01:00 • committed by GitHub
parent 3d53d8756f
commit 5b9f4988c9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 220 additions and 91 deletions
Download patch file Download diff file Expand all files Collapse all files

17
Tools/peg_generator/scripts/test_parse_directory.py
View file

@ -13,7 +13,7 @@
from typing import List, Optional, Any
sys.path.insert(0, os.getcwd())
from pegen.build import build_parser_and_generator
from pegen.build import build_c_parser_and_generator
from pegen.testutil import print_memstats
from scripts import show_parse
@ -26,7 +26,8 @@
description="Helper program to test directories or files for pegen",
)
argparser.add_argument("-d", "--directory", help="Directory path containing files to test")
argparser.add_argument("-g", "--grammar-file", help="Grammar file path")
argparser.add_argument("--grammar-file", help="Grammar file path")
argparser.add_argument("--tokens-file", help="Tokens file path")
argparser.add_argument(
"-e", "--exclude", action="append", default=[], help="Glob(s) for matching files to exclude"
)
@ -114,6 +115,7 @@ def compare_trees(
def parse_directory(
directory: str,
grammar_file: str,
tokens_file: str,
verbose: bool,
excluded_files: List[str],
skip_actions: bool,
@ -131,15 +133,16 @@ def parse_directory(
print("You must specify a directory of files to test.", file=sys.stderr)
return 1
if grammar_file:
if grammar_file and tokens_file:
if not os.path.exists(grammar_file):
print(f"The specified grammar file, {grammar_file}, does not exist.", file=sys.stderr)
return 1
try:
if not extension and parser == "pegen":
build_parser_and_generator(
build_c_parser_and_generator(
grammar_file,
tokens_file,
"peg_extension/parse.c",
compile_extension=True,
skip_actions=skip_actions,
@ -154,7 +157,9 @@ def parse_directory(
return 1
else:
print("A grammar file was not provided - attempting to use existing file...\n")
print(
"A grammar file or a tokens file was not provided - attempting to use existing parser from stdlib...\n"
)
if parser == "pegen":
try:
@ -264,6 +269,7 @@ def main() -> None:
args = argparser.parse_args()
directory = args.directory
grammar_file = args.grammar_file
tokens_file = args.tokens_file
verbose = args.verbose
excluded_files = args.exclude
skip_actions = args.skip_actions
@ -273,6 +279,7 @@ def main() -> None:
parse_directory(
directory,
grammar_file,
tokens_file,
verbose,
excluded_files,
skip_actions,