mirror of
https://github.com/python/cpython.git
synced 2026-01-06 15:32:22 +00:00
compiler.transformer: correct lineno attribute when possible
SF patch #1015989 The basic idea of this patch is to compute lineno attributes for all AST nodes. The actual implementation lead to a lot of restructing and code cleanup. The generated AST nodes now have an optional lineno argument to constructor. Remove the top-level asList(), since it didn't seem to serve any purpose. Add an __iter__ to ast nodes. Use isinstance() instead of explicit type tests. Change transformer to use the new lineno attribute, which replaces three lines of code with one. Use universal newlines so that we can get rid of special-case code for line endings. Use lookup_node() in a few more frequently called, but simple com_xxx methods(). Change string exception to class exception.
This commit is contained in:
Jeremy Hylton
parent
2ad68e69b9
commit
566d934745
6 changed files with 407 additions and 447 deletions
File diff suppressed because it is too large
Load diff
|
|
@ -14,7 +14,10 @@
|
|||
#
|
||||
# Modifications and improvements for Python 2.0 by Jeremy Hylton and
|
||||
# Mark Hammond
|
||||
|
||||
#
|
||||
# Some fixes to try to have correct line number on almost all nodes
|
||||
# (except Module, Discard and Stmt) added by Sylvain Thenault
|
||||
#
|
||||
# Portions of this file are:
|
||||
# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
|
||||
#
|
||||
|
|
@ -22,21 +25,20 @@
|
|||
# http://www.opensource.org/licenses/bsd-license.html
|
||||
# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
|
||||
|
||||
from ast import *
|
||||
from compiler.ast import *
|
||||
import parser
|
||||
# Care must be taken to use only symbols and tokens defined in Python
|
||||
# 1.5.2 for code branches executed in 1.5.2
|
||||
import symbol
|
||||
import token
|
||||
import sys
|
||||
|
||||
error = 'walker.error'
|
||||
class WalkerError(StandardError):
|
||||
pass
|
||||
|
||||
from consts import CO_VARARGS, CO_VARKEYWORDS
|
||||
from consts import OP_ASSIGN, OP_DELETE, OP_APPLY
|
||||
|
||||
def parseFile(path):
|
||||
f = open(path)
|
||||
f = open(path, "U")
|
||||
# XXX The parser API tolerates files without a trailing newline,
|
||||
# but not strings without a trailing newline. Always add an extra
|
||||
# newline to the file contents, since we're going through the string
|
||||
|
|
@ -68,6 +70,16 @@ def asList(nodes):
|
|||
l.append(item)
|
||||
return l
|
||||
|
||||
def extractLineNo(ast):
|
||||
if not isinstance(ast[1], tuple):
|
||||
# get a terminal node
|
||||
return ast[2]
|
||||
for child in ast[1:]:
|
||||
if isinstance(child, tuple):
|
||||
lineno = extractLineNo(child)
|
||||
if lineno is not None:
|
||||
return lineno
|
||||
|
||||
def Node(*args):
|
||||
kind = args[0]
|
||||
if nodes.has_key(kind):
|
||||
|
|
@ -77,7 +89,7 @@ def Node(*args):
|
|||
print nodes[kind], len(args), args
|
||||
raise
|
||||
else:
|
||||
raise error, "Can't find appropriate Node type: %s" % str(args)
|
||||
raise WalkerEror, "Can't find appropriate Node type: %s" % str(args)
|
||||
#return apply(ast.Node, args)
|
||||
|
||||
class Transformer:
|
||||
|
|
@ -108,17 +120,14 @@ def __init__(self):
|
|||
|
||||
def transform(self, tree):
|
||||
"""Transform an AST into a modified parse tree."""
|
||||
if type(tree) != type(()) and type(tree) != type([]):
|
||||
if not (isinstance(tree, tuple) or isinstance(tree, list)):
|
||||
tree = parser.ast2tuple(tree, line_info=1)
|
||||
return self.compile_node(tree)
|
||||
|
||||
def parsesuite(self, text):
|
||||
"""Return a modified parse tree for the given suite text."""
|
||||
# Hack for handling non-native line endings on non-DOS like OSs.
|
||||
# this can go now we have universal newlines?
|
||||
text = text.replace('\x0d', '')
|
||||
return self.transform(parser.suite(text))
|
||||
|
||||
|
||||
def parseexpr(self, text):
|
||||
"""Return a modified parse tree for the given expression text."""
|
||||
return self.transform(parser.expr(text))
|
||||
|
|
@ -156,7 +165,7 @@ def compile_node(self, node):
|
|||
if n == symbol.classdef:
|
||||
return self.classdef(node[1:])
|
||||
|
||||
raise error, ('unexpected node type', n)
|
||||
raise WalkerEror, ('unexpected node type', n)
|
||||
|
||||
def single_input(self, node):
|
||||
### do we want to do anything about being "interactive" ?
|
||||
|
|
@ -254,9 +263,8 @@ def funcdef(self, nodelist):
|
|||
assert isinstance(code, Stmt)
|
||||
assert isinstance(code.nodes[0], Discard)
|
||||
del code.nodes[0]
|
||||
n = Function(decorators, name, names, defaults, flags, doc, code)
|
||||
n.lineno = lineno
|
||||
return n
|
||||
return Function(decorators, name, names, defaults, flags, doc, code,
|
||||
lineno=lineno)
|
||||
|
||||
def lambdef(self, nodelist):
|
||||
# lambdef: 'lambda' [varargslist] ':' test
|
||||
|
|
@ -269,9 +277,7 @@ def lambdef(self, nodelist):
|
|||
# code for lambda
|
||||
code = self.com_node(nodelist[-1])
|
||||
|
||||
n = Lambda(names, defaults, flags, code)
|
||||
n.lineno = nodelist[1][2]
|
||||
return n
|
||||
return Lambda(names, defaults, flags, code, lineno=nodelist[1][2])
|
||||
|
||||
def classdef(self, nodelist):
|
||||
# classdef: 'class' NAME ['(' testlist ')'] ':' suite
|
||||
|
|
@ -291,9 +297,7 @@ def classdef(self, nodelist):
|
|||
assert isinstance(code.nodes[0], Discard)
|
||||
del code.nodes[0]
|
||||
|
||||
n = Class(name, bases, doc, code)
|
||||
n.lineno = nodelist[1][2]
|
||||
return n
|
||||
return Class(name, bases, doc, code, lineno=nodelist[1][2])
|
||||
|
||||
def stmt(self, nodelist):
|
||||
return self.com_stmt(nodelist[0])
|
||||
|
|
@ -310,31 +314,31 @@ def simple_stmt(self, nodelist):
|
|||
return Stmt(stmts)
|
||||
|
||||
def parameters(self, nodelist):
|
||||
raise error
|
||||
raise WalkerEror
|
||||
|
||||
def varargslist(self, nodelist):
|
||||
raise error
|
||||
raise WalkerEror
|
||||
|
||||
def fpdef(self, nodelist):
|
||||
raise error
|
||||
raise WalkerEror
|
||||
|
||||
def fplist(self, nodelist):
|
||||
raise error
|
||||
raise WalkerEror
|
||||
|
||||
def dotted_name(self, nodelist):
|
||||
raise error
|
||||
raise WalkerEror
|
||||
|
||||
def comp_op(self, nodelist):
|
||||
raise error
|
||||
raise WalkerEror
|
||||
|
||||
def trailer(self, nodelist):
|
||||
raise error
|
||||
raise WalkerEror
|
||||
|
||||
def sliceop(self, nodelist):
|
||||
raise error
|
||||
raise WalkerEror
|
||||
|
||||
def argument(self, nodelist):
|
||||
raise error
|
||||
raise WalkerEror
|
||||
|
||||
# --------------------------------------------------------------
|
||||
#
|
||||
|
|
@ -346,21 +350,17 @@ def expr_stmt(self, nodelist):
|
|||
en = nodelist[-1]
|
||||
exprNode = self.lookup_node(en)(en[1:])
|
||||
if len(nodelist) == 1:
|
||||
n = Discard(exprNode)
|
||||
n.lineno = exprNode.lineno
|
||||
return n
|
||||
return Discard(exprNode, lineno=exprNode.lineno)
|
||||
if nodelist[1][0] == token.EQUAL:
|
||||
nodesl = []
|
||||
for i in range(0, len(nodelist) - 2, 2):
|
||||
nodesl.append(self.com_assign(nodelist[i], OP_ASSIGN))
|
||||
n = Assign(nodesl, exprNode)
|
||||
n.lineno = nodelist[1][2]
|
||||
return Assign(nodesl, exprNode, lineno=nodelist[1][2])
|
||||
else:
|
||||
lval = self.com_augassign(nodelist[0])
|
||||
op = self.com_augassign_op(nodelist[1])
|
||||
n = AugAssign(lval, op[1], exprNode)
|
||||
n.lineno = op[2]
|
||||
return n
|
||||
return AugAssign(lval, op[1], exprNode, lineno=op[2])
|
||||
raise WalkerError, "can't get here"
|
||||
|
||||
def print_stmt(self, nodelist):
|
||||
# print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ])
|
||||
|
|
@ -379,45 +379,29 @@ def print_stmt(self, nodelist):
|
|||
for i in range(start, len(nodelist), 2):
|
||||
items.append(self.com_node(nodelist[i]))
|
||||
if nodelist[-1][0] == token.COMMA:
|
||||
n = Print(items, dest)
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
n = Printnl(items, dest)
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return Print(items, dest, lineno=nodelist[0][2])
|
||||
return Printnl(items, dest, lineno=nodelist[0][2])
|
||||
|
||||
def del_stmt(self, nodelist):
|
||||
return self.com_assign(nodelist[1], OP_DELETE)
|
||||
|
||||
def pass_stmt(self, nodelist):
|
||||
n = Pass()
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return Pass(lineno=nodelist[0][2])
|
||||
|
||||
def break_stmt(self, nodelist):
|
||||
n = Break()
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return Break(lineno=nodelist[0][2])
|
||||
|
||||
def continue_stmt(self, nodelist):
|
||||
n = Continue()
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return Continue(lineno=nodelist[0][2])
|
||||
|
||||
def return_stmt(self, nodelist):
|
||||
# return: [testlist]
|
||||
if len(nodelist) < 2:
|
||||
n = Return(Const(None))
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
n = Return(self.com_node(nodelist[1]))
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return Return(Const(None), lineno=nodelist[0][2])
|
||||
return Return(self.com_node(nodelist[1]), lineno=nodelist[0][2])
|
||||
|
||||
def yield_stmt(self, nodelist):
|
||||
n = Yield(self.com_node(nodelist[1]))
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return Yield(self.com_node(nodelist[1]), lineno=nodelist[0][2])
|
||||
|
||||
def raise_stmt(self, nodelist):
|
||||
# raise: [test [',' test [',' test]]]
|
||||
|
|
@ -433,9 +417,7 @@ def raise_stmt(self, nodelist):
|
|||
expr1 = self.com_node(nodelist[1])
|
||||
else:
|
||||
expr1 = None
|
||||
n = Raise(expr1, expr2, expr3)
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return Raise(expr1, expr2, expr3, lineno=nodelist[0][2])
|
||||
|
||||
def import_stmt(self, nodelist):
|
||||
# import_stmt: import_name | import_from
|
||||
|
|
@ -444,9 +426,8 @@ def import_stmt(self, nodelist):
|
|||
|
||||
def import_name(self, nodelist):
|
||||
# import_name: 'import' dotted_as_names
|
||||
n = Import(self.com_dotted_as_names(nodelist[1]))
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return Import(self.com_dotted_as_names(nodelist[1]),
|
||||
lineno=nodelist[0][2])
|
||||
|
||||
def import_from(self, nodelist):
|
||||
# import_from: 'from' dotted_name 'import' ('*' |
|
||||
|
|
@ -456,21 +437,19 @@ def import_from(self, nodelist):
|
|||
assert nodelist[2][1] == 'import'
|
||||
fromname = self.com_dotted_name(nodelist[1])
|
||||
if nodelist[3][0] == token.STAR:
|
||||
n = From(fromname, [('*', None)])
|
||||
# TODO(jhylton): where is the lineno?
|
||||
return From(fromname, [('*', None)])
|
||||
else:
|
||||
node = nodelist[3 + (nodelist[3][0] == token.LPAR)]
|
||||
n = From(fromname, self.com_import_as_names(node))
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return From(fromname, self.com_import_as_names(node),
|
||||
lineno=nodelist[0][2])
|
||||
|
||||
def global_stmt(self, nodelist):
|
||||
# global: NAME (',' NAME)*
|
||||
names = []
|
||||
for i in range(1, len(nodelist), 2):
|
||||
names.append(nodelist[i][1])
|
||||
n = Global(names)
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return Global(names, lineno=nodelist[0][2])
|
||||
|
||||
def exec_stmt(self, nodelist):
|
||||
# exec_stmt: 'exec' expr ['in' expr [',' expr]]
|
||||
|
|
@ -484,9 +463,7 @@ def exec_stmt(self, nodelist):
|
|||
else:
|
||||
expr2 = expr3 = None
|
||||
|
||||
n = Exec(expr1, expr2, expr3)
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return Exec(expr1, expr2, expr3, lineno=nodelist[0][2])
|
||||
|
||||
def assert_stmt(self, nodelist):
|
||||
# 'assert': test, [',' test]
|
||||
|
|
@ -495,9 +472,7 @@ def assert_stmt(self, nodelist):
|
|||
expr2 = self.com_node(nodelist[3])
|
||||
else:
|
||||
expr2 = None
|
||||
n = Assert(expr1, expr2)
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return Assert(expr1, expr2, lineno=nodelist[0][2])
|
||||
|
||||
def if_stmt(self, nodelist):
|
||||
# if: test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
|
||||
|
|
@ -512,9 +487,7 @@ def if_stmt(self, nodelist):
|
|||
## elseNode.lineno = nodelist[-1][1][2]
|
||||
else:
|
||||
elseNode = None
|
||||
n = If(tests, elseNode)
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return If(tests, elseNode, lineno=nodelist[0][2])
|
||||
|
||||
def while_stmt(self, nodelist):
|
||||
# 'while' test ':' suite ['else' ':' suite]
|
||||
|
|
@ -527,9 +500,7 @@ def while_stmt(self, nodelist):
|
|||
else:
|
||||
elseNode = None
|
||||
|
||||
n = While(testNode, bodyNode, elseNode)
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return While(testNode, bodyNode, elseNode, lineno=nodelist[0][2])
|
||||
|
||||
def for_stmt(self, nodelist):
|
||||
# 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
|
||||
|
|
@ -543,9 +514,8 @@ def for_stmt(self, nodelist):
|
|||
else:
|
||||
elseNode = None
|
||||
|
||||
n = For(assignNode, listNode, bodyNode, elseNode)
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return For(assignNode, listNode, bodyNode, elseNode,
|
||||
lineno=nodelist[0][2])
|
||||
|
||||
def try_stmt(self, nodelist):
|
||||
# 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
|
||||
|
|
@ -601,9 +571,7 @@ def not_test(self, nodelist):
|
|||
# 'not' not_test | comparison
|
||||
result = self.com_node(nodelist[-1])
|
||||
if len(nodelist) == 2:
|
||||
n = Not(result)
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return Not(result, lineno=nodelist[0][2])
|
||||
return result
|
||||
|
||||
def comparison(self, nodelist):
|
||||
|
|
@ -637,9 +605,7 @@ def comparison(self, nodelist):
|
|||
# the two have very different semantics and results (note that the
|
||||
# latter form is always true)
|
||||
|
||||
n = Compare(node, results)
|
||||
n.lineno = lineno
|
||||
return n
|
||||
return Compare(node, results, lineno=lineno)
|
||||
|
||||
def expr(self, nodelist):
|
||||
# xor_expr ('|' xor_expr)*
|
||||
|
|
@ -659,11 +625,9 @@ def shift_expr(self, nodelist):
|
|||
for i in range(2, len(nodelist), 2):
|
||||
right = self.com_node(nodelist[i])
|
||||
if nodelist[i-1][0] == token.LEFTSHIFT:
|
||||
node = LeftShift([node, right])
|
||||
node.lineno = nodelist[1][2]
|
||||
node = LeftShift([node, right], lineno=nodelist[1][2])
|
||||
elif nodelist[i-1][0] == token.RIGHTSHIFT:
|
||||
node = RightShift([node, right])
|
||||
node.lineno = nodelist[1][2]
|
||||
node = RightShift([node, right], lineno=nodelist[1][2])
|
||||
else:
|
||||
raise ValueError, "unexpected token: %s" % nodelist[i-1][0]
|
||||
return node
|
||||
|
|
@ -673,11 +637,9 @@ def arith_expr(self, nodelist):
|
|||
for i in range(2, len(nodelist), 2):
|
||||
right = self.com_node(nodelist[i])
|
||||
if nodelist[i-1][0] == token.PLUS:
|
||||
node = Add([node, right])
|
||||
node.lineno = nodelist[1][2]
|
||||
node = Add([node, right], lineno=nodelist[1][2])
|
||||
elif nodelist[i-1][0] == token.MINUS:
|
||||
node = Sub([node, right])
|
||||
node.lineno = nodelist[1][2]
|
||||
node = Sub([node, right], lineno=nodelist[1][2])
|
||||
else:
|
||||
raise ValueError, "unexpected token: %s" % nodelist[i-1][0]
|
||||
return node
|
||||
|
|
@ -703,17 +665,14 @@ def term(self, nodelist):
|
|||
def factor(self, nodelist):
|
||||
elt = nodelist[0]
|
||||
t = elt[0]
|
||||
node = self.com_node(nodelist[-1])
|
||||
node = self.lookup_node(nodelist[-1])(nodelist[-1][1:])
|
||||
# need to handle (unary op)constant here...
|
||||
if t == token.PLUS:
|
||||
node = UnaryAdd(node)
|
||||
node.lineno = elt[2]
|
||||
return UnaryAdd(node, lineno=elt[2])
|
||||
elif t == token.MINUS:
|
||||
node = UnarySub(node)
|
||||
node.lineno = elt[2]
|
||||
return UnarySub(node, lineno=elt[2])
|
||||
elif t == token.TILDE:
|
||||
node = Invert(node)
|
||||
node.lineno = elt[2]
|
||||
node = Invert(node, lineno=elt[2])
|
||||
return node
|
||||
|
||||
def power(self, nodelist):
|
||||
|
|
@ -722,31 +681,26 @@ def power(self, nodelist):
|
|||
for i in range(1, len(nodelist)):
|
||||
elt = nodelist[i]
|
||||
if elt[0] == token.DOUBLESTAR:
|
||||
n = Power([node, self.com_node(nodelist[i+1])])
|
||||
n.lineno = elt[2]
|
||||
return n
|
||||
return Power([node, self.com_node(nodelist[i+1])],
|
||||
lineno=elt[2])
|
||||
|
||||
node = self.com_apply_trailer(node, elt)
|
||||
|
||||
return node
|
||||
|
||||
def atom(self, nodelist):
|
||||
n = self._atom_dispatch[nodelist[0][0]](nodelist)
|
||||
return self._atom_dispatch[nodelist[0][0]](nodelist)
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
|
||||
def atom_lpar(self, nodelist):
|
||||
if nodelist[1][0] == token.RPAR:
|
||||
n = Tuple(())
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return Tuple(())
|
||||
return self.com_node(nodelist[1])
|
||||
|
||||
def atom_lsqb(self, nodelist):
|
||||
if nodelist[1][0] == token.RSQB:
|
||||
n = List(())
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return List(())
|
||||
return self.com_list_constructor(nodelist[1])
|
||||
|
||||
def atom_lbrace(self, nodelist):
|
||||
|
|
@ -755,16 +709,12 @@ def atom_lbrace(self, nodelist):
|
|||
return self.com_dictmaker(nodelist[1])
|
||||
|
||||
def atom_backquote(self, nodelist):
|
||||
n = Backquote(self.com_node(nodelist[1]))
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return Backquote(self.com_node(nodelist[1]))
|
||||
|
||||
def atom_number(self, nodelist):
|
||||
### need to verify this matches compile.c
|
||||
k = eval(nodelist[0][1])
|
||||
n = Const(k)
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return Const(k, lineno=nodelist[0][2])
|
||||
|
||||
def decode_literal(self, lit):
|
||||
if self.encoding:
|
||||
|
|
@ -781,15 +731,10 @@ def atom_string(self, nodelist):
|
|||
k = ''
|
||||
for node in nodelist:
|
||||
k += self.decode_literal(node[1])
|
||||
n = Const(k)
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return Const(k, lineno=nodelist[0][2])
|
||||
|
||||
def atom_name(self, nodelist):
|
||||
### any processing to do?
|
||||
n = Name(nodelist[0][1])
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return Name(nodelist[0][1], lineno=nodelist[0][2])
|
||||
|
||||
# --------------------------------------------------------------
|
||||
#
|
||||
|
|
@ -807,6 +752,8 @@ def atom_name(self, nodelist):
|
|||
def lookup_node(self, node):
|
||||
return self._dispatch[node[0]]
|
||||
|
||||
_callers = {}
|
||||
|
||||
def com_node(self, node):
|
||||
# Note: compile.c has handling in com_node for del_stmt, pass_stmt,
|
||||
# break_stmt, stmt, small_stmt, flow_stmt, simple_stmt,
|
||||
|
|
@ -865,6 +812,7 @@ def com_arglist(self, nodelist):
|
|||
i = i + 2
|
||||
elif len(defaults):
|
||||
# Treat "(a=1, b)" as "(a=1, b=None)"
|
||||
print nodelist[i]
|
||||
defaults.append(Const(None))
|
||||
|
||||
i = i + 1
|
||||
|
|
@ -938,10 +886,9 @@ def com_bases(self, node):
|
|||
|
||||
def com_try_finally(self, nodelist):
|
||||
# try_fin_stmt: "try" ":" suite "finally" ":" suite
|
||||
n = TryFinally(self.com_node(nodelist[2]),
|
||||
self.com_node(nodelist[5]))
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return TryFinally(self.com_node(nodelist[2]),
|
||||
self.com_node(nodelist[5]),
|
||||
lineno=nodelist[0][2])
|
||||
|
||||
def com_try_except(self, nodelist):
|
||||
# try_except: 'try' ':' suite (except_clause ':' suite)* ['else' suite]
|
||||
|
|
@ -965,9 +912,8 @@ def com_try_except(self, nodelist):
|
|||
|
||||
if node[0] == token.NAME:
|
||||
elseNode = self.com_node(nodelist[i+2])
|
||||
n = TryExcept(self.com_node(nodelist[2]), clauses, elseNode)
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
return TryExcept(self.com_node(nodelist[2]), clauses, elseNode,
|
||||
lineno=nodelist[0][2])
|
||||
|
||||
def com_augassign_op(self, node):
|
||||
assert node[0] == symbol.augassign
|
||||
|
|
@ -1031,7 +977,7 @@ def com_assign_tuple(self, node, assigning):
|
|||
assigns = []
|
||||
for i in range(1, len(node), 2):
|
||||
assigns.append(self.com_assign(node[i], assigning))
|
||||
return AssTuple(assigns)
|
||||
return AssTuple(assigns, lineno=extractLineNo(node))
|
||||
|
||||
def com_assign_list(self, node, assigning):
|
||||
assigns = []
|
||||
|
|
@ -1041,12 +987,10 @@ def com_assign_list(self, node, assigning):
|
|||
raise SyntaxError, "can't assign to list comprehension"
|
||||
assert node[i + 1][0] == token.COMMA, node[i + 1]
|
||||
assigns.append(self.com_assign(node[i], assigning))
|
||||
return AssList(assigns)
|
||||
return AssList(assigns, lineno=extractLineNo(node))
|
||||
|
||||
def com_assign_name(self, node, assigning):
|
||||
n = AssName(node[1], assigning)
|
||||
n.lineno = node[2]
|
||||
return n
|
||||
return AssName(node[1], assigning, lineno=node[2])
|
||||
|
||||
def com_assign_trailer(self, primary, node, assigning):
|
||||
t = node[1][0]
|
||||
|
|
@ -1059,7 +1003,7 @@ def com_assign_trailer(self, primary, node, assigning):
|
|||
raise SyntaxError, "unknown trailer type: %s" % t
|
||||
|
||||
def com_assign_attr(self, primary, node, assigning):
|
||||
return AssAttr(primary, node[1], assigning)
|
||||
return AssAttr(primary, node[1], assigning, lineno=node[-1])
|
||||
|
||||
def com_binary(self, constructor, nodelist):
|
||||
"Compile 'NODE (OP NODE)*' into (type, [ node1, ..., nodeN ])."
|
||||
|
|
@ -1071,7 +1015,7 @@ def com_binary(self, constructor, nodelist):
|
|||
for i in range(0, l, 2):
|
||||
n = nodelist[i]
|
||||
items.append(self.lookup_node(n)(n[1:]))
|
||||
return constructor(items)
|
||||
return constructor(items, lineno=extractLineNo(nodelist))
|
||||
|
||||
def com_stmt(self, node):
|
||||
result = self.lookup_node(node)(node[1:])
|
||||
|
|
@ -1081,7 +1025,7 @@ def com_stmt(self, node):
|
|||
return Stmt([result])
|
||||
|
||||
def com_append_stmt(self, stmts, node):
|
||||
result = self.com_node(node)
|
||||
result = self.lookup_node(node)(node[1:])
|
||||
assert result is not None
|
||||
if isinstance(result, Stmt):
|
||||
stmts.extend(result.nodes)
|
||||
|
|
@ -1100,7 +1044,7 @@ def com_list_constructor(self, nodelist):
|
|||
elif nodelist[i][0] == token.COMMA:
|
||||
continue
|
||||
values.append(self.com_node(nodelist[i]))
|
||||
return List(values)
|
||||
return List(values, lineno=values[0].lineno)
|
||||
|
||||
def com_list_comprehension(self, expr, node):
|
||||
# list_iter: list_for | list_if
|
||||
|
|
@ -1125,8 +1069,7 @@ def com_list_comprehension(self, expr, node):
|
|||
node = self.com_list_iter(node[5])
|
||||
elif t == 'if':
|
||||
test = self.com_node(node[2])
|
||||
newif = ListCompIf(test)
|
||||
newif.lineno = node[1][2]
|
||||
newif = ListCompIf(test, lineno=node[1][2])
|
||||
newfor.ifs.append(newif)
|
||||
if len(node) == 3:
|
||||
node = None
|
||||
|
|
@ -1136,9 +1079,7 @@ def com_list_comprehension(self, expr, node):
|
|||
raise SyntaxError, \
|
||||
("unexpected list comprehension element: %s %d"
|
||||
% (node, lineno))
|
||||
n = ListComp(expr, fors)
|
||||
n.lineno = lineno
|
||||
return n
|
||||
return ListComp(expr, fors, lineno=lineno)
|
||||
|
||||
def com_list_iter(self, node):
|
||||
assert node[0] == symbol.list_iter
|
||||
|
|
@ -1163,8 +1104,8 @@ def com_generator_expression(self, expr, node):
|
|||
if t == 'for':
|
||||
assignNode = self.com_assign(node[2], OP_ASSIGN)
|
||||
genNode = self.com_node(node[4])
|
||||
newfor = GenExprFor(assignNode, genNode, [])
|
||||
newfor.lineno = node[1][2]
|
||||
newfor = GenExprFor(assignNode, genNode, [],
|
||||
lineno=node[1][2])
|
||||
fors.append(newfor)
|
||||
if (len(node)) == 5:
|
||||
node = None
|
||||
|
|
@ -1172,8 +1113,7 @@ def com_generator_expression(self, expr, node):
|
|||
node = self.com_gen_iter(node[5])
|
||||
elif t == 'if':
|
||||
test = self.com_node(node[2])
|
||||
newif = GenExprIf(test)
|
||||
newif.lineno = node[1][2]
|
||||
newif = GenExprIf(test, lineno=node[1][2])
|
||||
newfor.ifs.append(newif)
|
||||
if len(node) == 3:
|
||||
node = None
|
||||
|
|
@ -1184,9 +1124,7 @@ def com_generator_expression(self, expr, node):
|
|||
("unexpected generator expression element: %s %d"
|
||||
% (node, lineno))
|
||||
fors[0].is_outmost = True
|
||||
n = GenExpr(GenExprInner(expr, fors))
|
||||
n.lineno = lineno
|
||||
return n
|
||||
return GenExpr(GenExprInner(expr, fors), lineno=lineno)
|
||||
|
||||
def com_gen_iter(self, node):
|
||||
assert node[0] == symbol.gen_iter
|
||||
|
|
@ -1214,13 +1152,11 @@ def com_apply_trailer(self, primaryNode, nodelist):
|
|||
def com_select_member(self, primaryNode, nodelist):
|
||||
if nodelist[0] != token.NAME:
|
||||
raise SyntaxError, "member must be a name"
|
||||
n = Getattr(primaryNode, nodelist[1])
|
||||
n.lineno = nodelist[2]
|
||||
return n
|
||||
return Getattr(primaryNode, nodelist[1], lineno=nodelist[2])
|
||||
|
||||
def com_call_function(self, primaryNode, nodelist):
|
||||
if nodelist[0] == token.RPAR:
|
||||
return CallFunc(primaryNode, [])
|
||||
return CallFunc(primaryNode, [], lineno=extractLineNo(nodelist))
|
||||
args = []
|
||||
kw = 0
|
||||
len_nodelist = len(nodelist)
|
||||
|
|
@ -1253,8 +1189,8 @@ def com_call_function(self, primaryNode, nodelist):
|
|||
dstar_node = self.com_node(ch)
|
||||
else:
|
||||
raise SyntaxError, 'unknown node type: %s' % tok
|
||||
|
||||
return CallFunc(primaryNode, args, star_node, dstar_node)
|
||||
return CallFunc(primaryNode, args, star_node, dstar_node,
|
||||
lineno=extractLineNo(nodelist))
|
||||
|
||||
def com_argument(self, nodelist, kw):
|
||||
if len(nodelist) == 3 and nodelist[2][0] == symbol.gen_for:
|
||||
|
|
@ -1270,8 +1206,7 @@ def com_argument(self, nodelist, kw):
|
|||
n = n[1]
|
||||
if n[0] != token.NAME:
|
||||
raise SyntaxError, "keyword can't be an expression (%s)"%n[0]
|
||||
node = Keyword(n[1], result)
|
||||
node.lineno = n[2]
|
||||
node = Keyword(n[1], result, lineno=n[2])
|
||||
return 1, node
|
||||
|
||||
def com_subscriptlist(self, primary, nodelist, assigning):
|
||||
|
|
@ -1291,8 +1226,8 @@ def com_subscriptlist(self, primary, nodelist, assigning):
|
|||
subscripts = []
|
||||
for i in range(1, len(nodelist), 2):
|
||||
subscripts.append(self.com_subscript(nodelist[i]))
|
||||
|
||||
return Subscript(primary, assigning, subscripts)
|
||||
return Subscript(primary, assigning, subscripts,
|
||||
lineno=extractLineNo(nodelist))
|
||||
|
||||
def com_subscript(self, node):
|
||||
# slice_item: expression | proper_slice | ellipsis
|
||||
|
|
@ -1338,8 +1273,7 @@ def com_sliceobj(self, node):
|
|||
items.append(Const(None))
|
||||
else:
|
||||
items.append(self.com_node(ch[2]))
|
||||
|
||||
return Sliceobj(items)
|
||||
return Sliceobj(items, lineno=extractLineNo(node))
|
||||
|
||||
def com_slice(self, primary, node, assigning):
|
||||
# short_slice: [lower_bound] ":" [upper_bound]
|
||||
|
|
@ -1352,7 +1286,8 @@ def com_slice(self, primary, node, assigning):
|
|||
elif len(node) == 4:
|
||||
lower = self.com_node(node[1])
|
||||
upper = self.com_node(node[3])
|
||||
return Slice(primary, assigning, lower, upper)
|
||||
return Slice(primary, assigning, lower, upper,
|
||||
lineno=extractLineNo(node))
|
||||
|
||||
def get_docstring(self, node, n=None):
|
||||
if n is None:
|
||||
|
|
|
|||
|
|
@ -33,6 +33,65 @@ def testCompileLibrary(self):
|
|||
else:
|
||||
compiler.compile(buf, basename, "exec")
|
||||
|
||||
def testLineNo(self):
|
||||
# Test that all nodes except Module have a correct lineno attribute.
|
||||
filename = __file__
|
||||
if filename.endswith(".pyc") or filename.endswith(".pyo"):
|
||||
filename = filename[:-1]
|
||||
tree = compiler.parseFile(filename)
|
||||
self.check_lineno(tree)
|
||||
|
||||
def check_lineno(self, node):
|
||||
try:
|
||||
self._check_lineno(node)
|
||||
except AssertionError:
|
||||
print node.__class__, node.lineno
|
||||
raise
|
||||
|
||||
def _check_lineno(self, node):
|
||||
if not node.__class__ in NOLINENO:
|
||||
self.assert_(isinstance(node.lineno, int),
|
||||
"lineno=%s on %s" % (node.lineno, node.__class__))
|
||||
self.assert_(node.lineno > 0,
|
||||
"lineno=%s on %s" % (node.lineno, node.__class__))
|
||||
for child in node.getChildNodes():
|
||||
self.check_lineno(child)
|
||||
|
||||
NOLINENO = (compiler.ast.Module, compiler.ast.Stmt, compiler.ast.Discard)
|
||||
|
||||
###############################################################################
|
||||
# code below is just used to trigger some possible errors, for the benefit of
|
||||
# testLineNo
|
||||
###############################################################################
|
||||
|
||||
class Toto:
|
||||
"""docstring"""
|
||||
pass
|
||||
|
||||
a, b = 2, 3
|
||||
[c, d] = 5, 6
|
||||
l = [(x, y) for x, y in zip(range(5), range(5,10))]
|
||||
l[0]
|
||||
l[3:4]
|
||||
if l:
|
||||
pass
|
||||
else:
|
||||
a, b = b, a
|
||||
|
||||
try:
|
||||
print yo
|
||||
except:
|
||||
yo = 3
|
||||
else:
|
||||
yo += 3
|
||||
|
||||
try:
|
||||
a += b
|
||||
finally:
|
||||
b = 0
|
||||
|
||||
###############################################################################
|
||||
|
||||
def test_main():
|
||||
global TEST_ALL
|
||||
TEST_ALL = test.test_support.is_resource_enabled("compiler")
|
||||
|
|
|
|||
|
|
@ -29,9 +29,7 @@ def testMultipleLHS(self):
|
|||
assert vals['b'] == 2
|
||||
|
||||
def test_main():
|
||||
test_support.run_unittest(
|
||||
Tests
|
||||
)
|
||||
test_support.run_unittest(Tests)
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_main()
|
||||
|
|
|
|||
|
|
@ -563,6 +563,7 @@ Michael Stone
|
|||
Ken Stox
|
||||
Daniel Stutzbach
|
||||
Paul Swartz
|
||||
Thenault Sylvain
|
||||
Geoff Talvola
|
||||
William Tanksley
|
||||
Christian Tanzer
|
||||
|
|
|
|||
|
|
@ -94,7 +94,6 @@ def get_argprops(self):
|
|||
def gen_source(self):
|
||||
buf = StringIO()
|
||||
print >> buf, "class %s(Node):" % self.name
|
||||
print >> buf, ' nodes["%s"] = "%s"' % (self.name.lower(), self.name)
|
||||
self._gen_init(buf)
|
||||
print >> buf
|
||||
self._gen_getChildren(buf)
|
||||
|
|
@ -106,12 +105,14 @@ def gen_source(self):
|
|||
return buf.read()
|
||||
|
||||
def _gen_init(self, buf):
|
||||
print >> buf, " def __init__(self, %s):" % self.args
|
||||
if self.args:
|
||||
print >> buf, " def __init__(self, %s, lineno=None):" % self.args
|
||||
else:
|
||||
print >> buf, " def __init__(self, lineno=None):"
|
||||
if self.argnames:
|
||||
for name in self.argnames:
|
||||
print >> buf, " self.%s = %s" % (name, name)
|
||||
else:
|
||||
print >> buf, " pass"
|
||||
print >> buf, " self.lineno = lineno"
|
||||
if self.init:
|
||||
print >> buf, "".join([" " + line for line in self.init])
|
||||
|
||||
|
|
@ -128,15 +129,18 @@ def _gen_getChildren(self, buf):
|
|||
else:
|
||||
print >> buf, " return %s" % clist
|
||||
else:
|
||||
print >> buf, " children = []"
|
||||
template = " children.%s(%sself.%s%s)"
|
||||
for name in self.argnames:
|
||||
if self.argprops[name] == P_NESTED:
|
||||
print >> buf, template % ("extend", "flatten(",
|
||||
name, ")")
|
||||
else:
|
||||
print >> buf, template % ("append", "", name, "")
|
||||
print >> buf, " return tuple(children)"
|
||||
if len(self.argnames) == 1:
|
||||
print >> buf, " return tuple(flatten(self.%s))" % self.argnames[0]
|
||||
else:
|
||||
print >> buf, " children = []"
|
||||
template = " children.%s(%sself.%s%s)"
|
||||
for name in self.argnames:
|
||||
if self.argprops[name] == P_NESTED:
|
||||
print >> buf, template % ("extend", "flatten(",
|
||||
name, ")")
|
||||
else:
|
||||
print >> buf, template % ("append", "", name, "")
|
||||
print >> buf, " return tuple(children)"
|
||||
|
||||
def _gen_getChildNodes(self, buf):
|
||||
print >> buf, " def getChildNodes(self):"
|
||||
|
|
@ -158,7 +162,7 @@ def _gen_getChildNodes(self, buf):
|
|||
template = " nodelist.%s(%sself.%s%s)"
|
||||
for name in self.argnames:
|
||||
if self.argprops[name] == P_NONE:
|
||||
tmp = (" if self.%s is not None:"
|
||||
tmp = (" if self.%s is not None:\n"
|
||||
" nodelist.append(self.%s)")
|
||||
print >> buf, tmp % (name, name)
|
||||
elif self.argprops[name] == P_NESTED:
|
||||
|
|
@ -226,16 +230,15 @@ def main():
|
|||
### PROLOGUE
|
||||
"""Python abstract syntax node definitions
|
||||
|
||||
This file is automatically generated.
|
||||
This file is automatically generated by Tools/compiler/astgen.py
|
||||
"""
|
||||
from types import TupleType, ListType
|
||||
from consts import CO_VARARGS, CO_VARKEYWORDS
|
||||
|
||||
def flatten(list):
|
||||
l = []
|
||||
for elt in list:
|
||||
t = type(elt)
|
||||
if t is TupleType or t is ListType:
|
||||
if t is tuple or t is list:
|
||||
for elt2 in flatten(elt):
|
||||
l.append(elt2)
|
||||
else:
|
||||
|
|
@ -245,29 +248,17 @@ def flatten(list):
|
|||
def flatten_nodes(list):
|
||||
return [n for n in flatten(list) if isinstance(n, Node)]
|
||||
|
||||
def asList(nodearg):
|
||||
l = []
|
||||
for item in nodearg:
|
||||
if hasattr(item, "asList"):
|
||||
l.append(item.asList())
|
||||
else:
|
||||
t = type(item)
|
||||
if t is TupleType or t is ListType:
|
||||
l.append(tuple(asList(item)))
|
||||
else:
|
||||
l.append(item)
|
||||
return l
|
||||
|
||||
nodes = {}
|
||||
|
||||
class Node: # an abstract base class
|
||||
lineno = None # provide a lineno for nodes that don't have one
|
||||
def getType(self):
|
||||
pass # implemented by subclass
|
||||
class Node:
|
||||
"""Abstract base class for ast nodes."""
|
||||
def getChildren(self):
|
||||
pass # implemented by subclasses
|
||||
def asList(self):
|
||||
return tuple(asList(self.getChildren()))
|
||||
def __iter__(self):
|
||||
for n in self.getChildren():
|
||||
yield n
|
||||
def asList(self): # for backwards compatibility
|
||||
return self.getChildren()
|
||||
def getChildNodes(self):
|
||||
pass # implemented by subclasses
|
||||
|
||||
|
|
@ -290,6 +281,6 @@ def __repr__(self):
|
|||
return "Expression(%s)" % (repr(self.node))
|
||||
|
||||
### EPILOGUE
|
||||
klasses = globals()
|
||||
for k in nodes.keys():
|
||||
nodes[k] = klasses[nodes[k]]
|
||||
for name, obj in globals().items():
|
||||
if isinstance(obj, type) and issubclass(obj, Node):
|
||||
nodes[name.lower()] = obj
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue