compiler.transformer: correct lineno attribute when possible

SF patch #1015989

The basic idea of this patch is to compute lineno attributes for all AST nodes.  The actual
implementation lead to a lot of restructing and code cleanup.

The generated AST nodes now have an optional lineno argument to constructor.  Remove the
top-level asList(), since it didn't seem to serve any purpose.  Add an __iter__ to ast nodes.
Use isinstance() instead of explicit type tests.

Change transformer to use the new lineno attribute, which replaces three lines of code with one.
Use universal newlines so that we can get rid of special-case code for line endings.  Use
lookup_node() in a few more frequently called, but simple com_xxx methods().  Change string
exception to class exception.

Tools/compiler/astgen.py

@@ -94,7 +94,6 @@ def get_argprops(self):
     def gen_source(self):
         buf = StringIO()
         print >> buf, "class %s(Node):" % self.name
-        print >> buf, '    nodes["%s"] = "%s"' % (self.name.lower(), self.name)
         self._gen_init(buf)
         print >> buf
         self._gen_getChildren(buf)
@@ -106,12 +105,14 @@ def gen_source(self):
         return buf.read()
 
     def _gen_init(self, buf):
-        print >> buf, "    def __init__(self, %s):" % self.args
+        if self.args:
+            print >> buf, "    def __init__(self, %s, lineno=None):" % self.args
+        else:
+            print >> buf, "    def __init__(self, lineno=None):"
         if self.argnames:
             for name in self.argnames:
                 print >> buf, "        self.%s = %s" % (name, name)
-        else:
-            print >> buf, "        pass"
+        print >> buf, "        self.lineno = lineno"
         if self.init:
             print >> buf, "".join(["    " + line for line in self.init])
 
@@ -128,15 +129,18 @@ def _gen_getChildren(self, buf):
                 else:
                     print >> buf, "        return %s" % clist
             else:
-                print >> buf, "        children = []"
-                template = "        children.%s(%sself.%s%s)"
-                for name in self.argnames:
-                    if self.argprops[name] == P_NESTED:
-                        print >> buf, template % ("extend", "flatten(",
-                                                  name, ")")
-                    else:
-                        print >> buf, template % ("append", "", name, "")
-                print >> buf, "        return tuple(children)"
+                if len(self.argnames) == 1:
+                    print >> buf, "        return tuple(flatten(self.%s))" % self.argnames[0]
+                else:
+                    print >> buf, "        children = []"
+                    template = "        children.%s(%sself.%s%s)"
+                    for name in self.argnames:
+                        if self.argprops[name] == P_NESTED:
+                            print >> buf, template % ("extend", "flatten(",
+                                                      name, ")")
+                        else:
+                            print >> buf, template % ("append", "", name, "")
+                    print >> buf, "        return tuple(children)"
 
     def _gen_getChildNodes(self, buf):
         print >> buf, "    def getChildNodes(self):"
@@ -158,7 +162,7 @@ def _gen_getChildNodes(self, buf):
                 template = "        nodelist.%s(%sself.%s%s)"
                 for name in self.argnames:
                     if self.argprops[name] == P_NONE:
-                        tmp = ("        if self.%s is not None:"
+                        tmp = ("        if self.%s is not None:\n"
                                "            nodelist.append(self.%s)")
                         print >> buf, tmp % (name, name)
                     elif self.argprops[name] == P_NESTED:
@@ -226,16 +230,15 @@ def main():
 ### PROLOGUE
 """Python abstract syntax node definitions
 
-This file is automatically generated.
+This file is automatically generated by Tools/compiler/astgen.py
 """
-from types import TupleType, ListType
 from consts import CO_VARARGS, CO_VARKEYWORDS
 
 def flatten(list):
     l = []
     for elt in list:
         t = type(elt)
-        if t is TupleType or t is ListType:
+        if t is tuple or t is list:
             for elt2 in flatten(elt):
                 l.append(elt2)
         else:
@@ -245,29 +248,17 @@ def flatten(list):
 def flatten_nodes(list):
     return [n for n in flatten(list) if isinstance(n, Node)]
 
-def asList(nodearg):
-    l = []
-    for item in nodearg:
-        if hasattr(item, "asList"):
-            l.append(item.asList())
-        else:
-            t = type(item)
-            if t is TupleType or t is ListType:
-                l.append(tuple(asList(item)))
-            else:
-                l.append(item)
-    return l
-
 nodes = {}
 
-class Node: # an abstract base class
-    lineno = None # provide a lineno for nodes that don't have one
-    def getType(self):
-        pass # implemented by subclass
+class Node:
+    """Abstract base class for ast nodes."""
     def getChildren(self):
         pass # implemented by subclasses
-    def asList(self):
-        return tuple(asList(self.getChildren()))
+    def __iter__(self):
+        for n in self.getChildren():
+            yield n
+    def asList(self): # for backwards compatibility
+        return self.getChildren()
     def getChildNodes(self):
         pass # implemented by subclasses
 
@@ -290,6 +281,6 @@ def __repr__(self):
         return "Expression(%s)" % (repr(self.node))
 
 ### EPILOGUE
-klasses = globals()
-for k in nodes.keys():
-    nodes[k] = klasses[nodes[k]]
+for name, obj in globals().items():
+    if isinstance(obj, type) and issubclass(obj, Node):
+       nodes[name.lower()] = obj