mirror of
https://github.com/python/cpython.git
synced 2025-10-31 13:41:24 +00:00
42 lines
983 B
Python
42 lines
983 B
Python
import itertools
|
|
|
|
|
|
def generate_tokens(tokens):
|
|
numbers = itertools.count(0)
|
|
for line in tokens:
|
|
line = line.strip()
|
|
|
|
if not line:
|
|
continue
|
|
if line.strip().startswith('#'):
|
|
continue
|
|
|
|
name = line.split()[0]
|
|
yield (name, next(numbers))
|
|
|
|
yield ('N_TOKENS', next(numbers))
|
|
yield ('NT_OFFSET', 256)
|
|
|
|
|
|
def generate_opmap(tokens):
|
|
for line in tokens:
|
|
line = line.strip()
|
|
|
|
if not line:
|
|
continue
|
|
if line.strip().startswith('#'):
|
|
continue
|
|
|
|
pieces = line.split()
|
|
|
|
if len(pieces) != 2:
|
|
continue
|
|
|
|
name, op = pieces
|
|
yield (op.strip("'"), name)
|
|
|
|
# Yield independently <>. This is needed so it does not collide
|
|
# with the token generation in "generate_tokens" because if this
|
|
# symbol is included in Grammar/Tokens, it will collide with !=
|
|
# as it has the same name (NOTEQUAL).
|
|
yield ('<>', 'NOTEQUAL')
|